diff options
author | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:17 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:17 +0300 |
commit | 4b11037e5a7d071c63e3c966199fe7102e6462e4 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 17e20fa084178ddcb16255f974dbde74fb93608b (diff) | |
download | ydb-4b11037e5a7d071c63e3c966199fe7102e6462e4.tar.gz |
Restoring authorship annotation for Daniil Cherednik <dcherednik@yandex-team.ru>. Commit 2 of 2.
770 files changed, 41375 insertions, 41375 deletions
diff --git a/library/cpp/actors/core/actor.cpp b/library/cpp/actors/core/actor.cpp index 4a5049a32c..6f9ba6a42b 100644 --- a/library/cpp/actors/core/actor.cpp +++ b/library/cpp/actors/core/actor.cpp @@ -61,17 +61,17 @@ namespace NActors { return TActivationContext::Schedule(delta, new IEventHandle(*this, {}, ev), cookie); } - TActorId TActivationContext::RegisterWithSameMailbox(IActor* actor, TActorId parentId) { + TActorId TActivationContext::RegisterWithSameMailbox(IActor* actor, TActorId parentId) { Y_VERIFY_DEBUG(parentId); auto& ctx = *TlsActivationContext; return ctx.ExecutorThread.RegisterActor(actor, &ctx.Mailbox, parentId.Hint(), parentId); } - TActorId TActorContext::RegisterWithSameMailbox(IActor* actor) const { + TActorId TActorContext::RegisterWithSameMailbox(IActor* actor) const { return ExecutorThread.RegisterActor(actor, &Mailbox, SelfID.Hint(), SelfID); } - TActorId IActor::RegisterWithSameMailbox(IActor* actor) const noexcept { + TActorId IActor::RegisterWithSameMailbox(IActor* actor) const noexcept { return TlsActivationContext->ExecutorThread.RegisterActor(actor, &TlsActivationContext->Mailbox, SelfActorId.Hint(), SelfActorId); } diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index 19aeeef00b..ed29bd14b9 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -71,12 +71,12 @@ namespace NActors { // register new actor in ActorSystem on new fresh mailbox. static TActorId Register(IActor* actor, TActorId parentId = TActorId(), TMailboxType::EType mailboxType = TMailboxType::HTSwap, ui32 poolId = Max<ui32>()); - // Register new actor in ActorSystem on same _mailbox_ as current actor. - // There is one thread per mailbox to execute actor, which mean - // no _cpu core scalability_ for such actors. - // This method of registration can be usefull if multiple actors share - // some memory. - static TActorId RegisterWithSameMailbox(IActor* actor, TActorId parentId); + // Register new actor in ActorSystem on same _mailbox_ as current actor. + // There is one thread per mailbox to execute actor, which mean + // no _cpu core scalability_ for such actors. + // This method of registration can be usefull if multiple actors share + // some memory. + static TActorId RegisterWithSameMailbox(IActor* actor, TActorId parentId); static const TActorContext& AsActorContext(); static TActorContext ActorContextFor(TActorId id); @@ -141,12 +141,12 @@ namespace NActors { // register new actor in ActorSystem on new fresh mailbox. TActorId Register(IActor* actor, TMailboxType::EType mailboxType = TMailboxType::HTSwap, ui32 poolId = Max<ui32>()) const; - // Register new actor in ActorSystem on same _mailbox_ as current actor. - // There is one thread per mailbox to execute actor, which mean - // no _cpu core scalability_ for such actors. - // This method of registration can be usefull if multiple actors share - // some memory. - TActorId RegisterWithSameMailbox(IActor* actor) const; + // Register new actor in ActorSystem on same _mailbox_ as current actor. + // There is one thread per mailbox to execute actor, which mean + // no _cpu core scalability_ for such actors. + // This method of registration can be usefull if multiple actors share + // some memory. + TActorId RegisterWithSameMailbox(IActor* actor) const; std::pair<ui32, ui32> CountMailboxEvents(ui32 maxTraverse = Max<ui32>()) const; }; @@ -204,7 +204,7 @@ namespace NActors { virtual void Schedule(TDuration delta, IEventBase* ev, ISchedulerCookie* cookie = nullptr) const noexcept = 0; virtual TActorId Register(IActor*, TMailboxType::EType mailboxType = TMailboxType::HTSwap, ui32 poolId = Max<ui32>()) const noexcept = 0; - virtual TActorId RegisterWithSameMailbox(IActor*) const noexcept = 0; + virtual TActorId RegisterWithSameMailbox(IActor*) const noexcept = 0; }; class TDecorator; @@ -368,12 +368,12 @@ namespace NActors { // register new actor in ActorSystem on new fresh mailbox. TActorId Register(IActor* actor, TMailboxType::EType mailboxType = TMailboxType::HTSwap, ui32 poolId = Max<ui32>()) const noexcept final; - // Register new actor in ActorSystem on same _mailbox_ as current actor. - // There is one thread per mailbox to execute actor, which mean - // no _cpu core scalability_ for such actors. - // This method of registration can be usefull if multiple actors share - // some memory. - TActorId RegisterWithSameMailbox(IActor* actor) const noexcept final; + // Register new actor in ActorSystem on same _mailbox_ as current actor. + // There is one thread per mailbox to execute actor, which mean + // no _cpu core scalability_ for such actors. + // This method of registration can be usefull if multiple actors share + // some memory. + TActorId RegisterWithSameMailbox(IActor* actor) const noexcept final; std::pair<ui32, ui32> CountMailboxEvents(ui32 maxTraverse = Max<ui32>()) const; @@ -478,7 +478,7 @@ namespace NActors { virtual bool DoBeforeReceiving(TAutoPtr<IEventHandle>& /*ev*/, const TActorContext& /*ctx*/) { return true; } - + virtual void DoAfterReceiving(const TActorContext& /*ctx*/) { } diff --git a/library/cpp/actors/core/actor_coroutine.h b/library/cpp/actors/core/actor_coroutine.h index 8c2f9f37c6..6bcb768eaf 100644 --- a/library/cpp/actors/core/actor_coroutine.h +++ b/library/cpp/actors/core/actor_coroutine.h @@ -134,7 +134,7 @@ namespace NActors { return GetActorContext().Register(actor, mailboxType, poolId); } - TActorId RegisterWithSameMailbox(IActor* actor) { + TActorId RegisterWithSameMailbox(IActor* actor) { return GetActorContext().RegisterWithSameMailbox(actor); } diff --git a/library/cpp/actors/dnsresolver/dnsresolver_ondemand.cpp b/library/cpp/actors/dnsresolver/dnsresolver_ondemand.cpp index 8636ff94c6..2025162e95 100644 --- a/library/cpp/actors/dnsresolver/dnsresolver_ondemand.cpp +++ b/library/cpp/actors/dnsresolver/dnsresolver_ondemand.cpp @@ -32,9 +32,9 @@ namespace NDnsResolver { TActorId GetUpstream() { if (Y_UNLIKELY(!CachingResolverId)) { if (Y_LIKELY(!SimpleResolverId)) { - SimpleResolverId = RegisterWithSameMailbox(CreateSimpleDnsResolver(Options)); + SimpleResolverId = RegisterWithSameMailbox(CreateSimpleDnsResolver(Options)); } - CachingResolverId = RegisterWithSameMailbox(CreateCachingDnsResolver(SimpleResolverId, Options)); + CachingResolverId = RegisterWithSameMailbox(CreateCachingDnsResolver(SimpleResolverId, Options)); } return CachingResolverId; } diff --git a/library/cpp/actors/helpers/flow_controlled_queue.cpp b/library/cpp/actors/helpers/flow_controlled_queue.cpp index d206ebc6f9..d75cc54023 100644 --- a/library/cpp/actors/helpers/flow_controlled_queue.cpp +++ b/library/cpp/actors/helpers/flow_controlled_queue.cpp @@ -94,7 +94,7 @@ class TFlowControlledRequestQueue : public IActor { void RegisterReqActor(THolder<IEventHandle> ev) { TFlowControlledRequestActor *reqActor = new TFlowControlledRequestActor(ActivityType, this, ev->Sender, ev->Cookie, ev->Flags); - const TActorId reqActorId = RegisterWithSameMailbox(reqActor); + const TActorId reqActorId = RegisterWithSameMailbox(reqActor); RegisteredRequests.emplace_back(reqActor); if (!Subscribed && (Target.NodeId() != SelfId().NodeId())) { diff --git a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp index c8d8de482c..7e2d8ccb94 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp @@ -364,7 +364,7 @@ namespace NActors { } // Create new session actor. - SessionID = RegisterWithSameMailbox(Session = new TInterconnectSessionTCP(this, msg->Params)); + SessionID = RegisterWithSameMailbox(Session = new TInterconnectSessionTCP(this, msg->Params)); IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::Init); SessionVirtualId = msg->Self; RemoteSessionVirtualId = msg->Peer; diff --git a/library/cpp/actors/interconnect/interconnect_tcp_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_session.cpp index 01a4e7655a..2ded7f9f53 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_session.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_session.cpp @@ -245,7 +245,7 @@ namespace NActors { auto actor = MakeHolder<TInputSessionTCP>(SelfId(), Socket, ReceiveContext, Proxy->Common, Proxy->Metrics, Proxy->PeerNodeId, nextPacket, GetDeadPeerTimeout(), Params); ReceiveContext->UnlockLastProcessedPacketSerial(); - ReceiverId = Params.Encryption ? RegisterWithSameMailbox(actor.Release()) : Register(actor.Release(), TMailboxType::ReadAsFilled); + ReceiverId = Params.Encryption ? RegisterWithSameMailbox(actor.Release()) : Register(actor.Release(), TMailboxType::ReadAsFilled); // register our socket in poller actor LOG_DEBUG_IC_SESSION("ICS11", "registering socket in PollerActor"); diff --git a/library/cpp/actors/interconnect/mock/ic_mock.cpp b/library/cpp/actors/interconnect/mock/ic_mock.cpp index 0dc1b87ed5..884503e602 100644 --- a/library/cpp/actors/interconnect/mock/ic_mock.cpp +++ b/library/cpp/actors/interconnect/mock/ic_mock.cpp @@ -227,7 +227,7 @@ namespace NActors { CheckSession(); if (!Session) { Session = new TSessionMockActor(this, State.GetValidSessionId()); - RegisterWithSameMailbox(Session); + RegisterWithSameMailbox(Session); } return Session; } diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 6173afc49e..6fa25b9965 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -1791,7 +1791,7 @@ namespace NActors { void Bootstrap(const TActorContext& ctx) { Become(&TStrandingActorDecorator::StateFunc); - ReplyId = ctx.RegisterWithSameMailbox(new TReplyActor(this)); + ReplyId = ctx.RegisterWithSameMailbox(new TReplyActor(this)); DelegateeOptions.OnlyMailboxes.push_back(TEventMailboxId(Delegatee.NodeId(), Delegatee.Hint())); for (const auto& actor : AdditionalActors) { DelegateeOptions.OnlyMailboxes.push_back(TEventMailboxId(actor.NodeId(), actor.Hint())); diff --git a/library/cpp/grpc/client/grpc_client_low.cpp b/library/cpp/grpc/client/grpc_client_low.cpp index ee9e997fa7..73cc908ef8 100644 --- a/library/cpp/grpc/client/grpc_client_low.cpp +++ b/library/cpp/grpc/client/grpc_client_low.cpp @@ -1,164 +1,164 @@ -#include "grpc_client_low.h" -#include <contrib/libs/grpc/src/core/lib/iomgr/socket_mutator.h> -#include <contrib/libs/grpc/include/grpc/support/log.h> - -#include <library/cpp/containers/stack_vector/stack_vec.h> - -#include <util/string/printf.h> +#include "grpc_client_low.h" +#include <contrib/libs/grpc/src/core/lib/iomgr/socket_mutator.h> +#include <contrib/libs/grpc/include/grpc/support/log.h> + +#include <library/cpp/containers/stack_vector/stack_vec.h> + +#include <util/string/printf.h> #include <util/system/thread.h> #include <util/random/random.h> - -#if !defined(_WIN32) && !defined(_WIN64) -#include <sys/types.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <netinet/tcp.h> -#endif - + +#if !defined(_WIN32) && !defined(_WIN64) +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#endif + namespace NGrpc { - -void EnableGRpcTracing() { - grpc_tracer_set_enabled("tcp", true); - grpc_tracer_set_enabled("client_channel", true); - grpc_tracer_set_enabled("channel", true); - grpc_tracer_set_enabled("api", true); - grpc_tracer_set_enabled("connectivity_state", true); - grpc_tracer_set_enabled("handshaker", true); - grpc_tracer_set_enabled("http", true); - grpc_tracer_set_enabled("http2_stream_state", true); - grpc_tracer_set_enabled("op_failure", true); - grpc_tracer_set_enabled("timer", true); - gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); -} - -class TGRpcKeepAliveSocketMutator : public grpc_socket_mutator { -public: - TGRpcKeepAliveSocketMutator(int idle, int count, int interval) - : Idle_(idle) - , Count_(count) - , Interval_(interval) - { - grpc_socket_mutator_init(this, &VTable); - } -private: - static TGRpcKeepAliveSocketMutator* Cast(grpc_socket_mutator* mutator) { - return static_cast<TGRpcKeepAliveSocketMutator*>(mutator); - } - - template<typename TVal> - bool SetOption(int fd, int level, int optname, const TVal& value) { - return setsockopt(fd, level, optname, reinterpret_cast<const char*>(&value), sizeof(value)) == 0; - } - bool SetOption(int fd) { - if (!SetOption(fd, SOL_SOCKET, SO_KEEPALIVE, 1)) { - Cerr << Sprintf("Failed to set SO_KEEPALIVE option: %s", strerror(errno)) << Endl; - return false; - } -#ifdef _linux_ - if (Idle_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPIDLE, Idle_)) { - Cerr << Sprintf("Failed to set TCP_KEEPIDLE option: %s", strerror(errno)) << Endl; - return false; - } - if (Count_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPCNT, Count_)) { - Cerr << Sprintf("Failed to set TCP_KEEPCNT option: %s", strerror(errno)) << Endl; - return false; - } - if (Interval_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPINTVL, Interval_)) { - Cerr << Sprintf("Failed to set TCP_KEEPINTVL option: %s", strerror(errno)) << Endl; - return false; - } -#endif - return true; - } - static bool Mutate(int fd, grpc_socket_mutator* mutator) { - auto self = Cast(mutator); - return self->SetOption(fd); - } - static int Compare(grpc_socket_mutator* a, grpc_socket_mutator* b) { - const auto* selfA = Cast(a); - const auto* selfB = Cast(b); - auto tupleA = std::make_tuple(selfA->Idle_, selfA->Count_, selfA->Interval_); - auto tupleB = std::make_tuple(selfB->Idle_, selfB->Count_, selfB->Interval_); - return tupleA < tupleB ? -1 : tupleA > tupleB ? 1 : 0; - } - static void Destroy(grpc_socket_mutator* mutator) { - delete Cast(mutator); - } - - static grpc_socket_mutator_vtable VTable; - const int Idle_; - const int Count_; - const int Interval_; -}; - -grpc_socket_mutator_vtable TGRpcKeepAliveSocketMutator::VTable = - { - &TGRpcKeepAliveSocketMutator::Mutate, - &TGRpcKeepAliveSocketMutator::Compare, - &TGRpcKeepAliveSocketMutator::Destroy - }; - + +void EnableGRpcTracing() { + grpc_tracer_set_enabled("tcp", true); + grpc_tracer_set_enabled("client_channel", true); + grpc_tracer_set_enabled("channel", true); + grpc_tracer_set_enabled("api", true); + grpc_tracer_set_enabled("connectivity_state", true); + grpc_tracer_set_enabled("handshaker", true); + grpc_tracer_set_enabled("http", true); + grpc_tracer_set_enabled("http2_stream_state", true); + grpc_tracer_set_enabled("op_failure", true); + grpc_tracer_set_enabled("timer", true); + gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); +} + +class TGRpcKeepAliveSocketMutator : public grpc_socket_mutator { +public: + TGRpcKeepAliveSocketMutator(int idle, int count, int interval) + : Idle_(idle) + , Count_(count) + , Interval_(interval) + { + grpc_socket_mutator_init(this, &VTable); + } +private: + static TGRpcKeepAliveSocketMutator* Cast(grpc_socket_mutator* mutator) { + return static_cast<TGRpcKeepAliveSocketMutator*>(mutator); + } + + template<typename TVal> + bool SetOption(int fd, int level, int optname, const TVal& value) { + return setsockopt(fd, level, optname, reinterpret_cast<const char*>(&value), sizeof(value)) == 0; + } + bool SetOption(int fd) { + if (!SetOption(fd, SOL_SOCKET, SO_KEEPALIVE, 1)) { + Cerr << Sprintf("Failed to set SO_KEEPALIVE option: %s", strerror(errno)) << Endl; + return false; + } +#ifdef _linux_ + if (Idle_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPIDLE, Idle_)) { + Cerr << Sprintf("Failed to set TCP_KEEPIDLE option: %s", strerror(errno)) << Endl; + return false; + } + if (Count_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPCNT, Count_)) { + Cerr << Sprintf("Failed to set TCP_KEEPCNT option: %s", strerror(errno)) << Endl; + return false; + } + if (Interval_ && !SetOption(fd, IPPROTO_TCP, TCP_KEEPINTVL, Interval_)) { + Cerr << Sprintf("Failed to set TCP_KEEPINTVL option: %s", strerror(errno)) << Endl; + return false; + } +#endif + return true; + } + static bool Mutate(int fd, grpc_socket_mutator* mutator) { + auto self = Cast(mutator); + return self->SetOption(fd); + } + static int Compare(grpc_socket_mutator* a, grpc_socket_mutator* b) { + const auto* selfA = Cast(a); + const auto* selfB = Cast(b); + auto tupleA = std::make_tuple(selfA->Idle_, selfA->Count_, selfA->Interval_); + auto tupleB = std::make_tuple(selfB->Idle_, selfB->Count_, selfB->Interval_); + return tupleA < tupleB ? -1 : tupleA > tupleB ? 1 : 0; + } + static void Destroy(grpc_socket_mutator* mutator) { + delete Cast(mutator); + } + + static grpc_socket_mutator_vtable VTable; + const int Idle_; + const int Count_; + const int Interval_; +}; + +grpc_socket_mutator_vtable TGRpcKeepAliveSocketMutator::VTable = + { + &TGRpcKeepAliveSocketMutator::Mutate, + &TGRpcKeepAliveSocketMutator::Compare, + &TGRpcKeepAliveSocketMutator::Destroy + }; + TChannelPool::TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, const TDuration& expireTime) - : TcpKeepAliveSettings_(tcpKeepAliveSettings) + : TcpKeepAliveSettings_(tcpKeepAliveSettings) , ExpireTime_(expireTime) , UpdateReUseTime_(ExpireTime_ * 0.3 < TDuration::Seconds(20) ? ExpireTime_ * 0.3 : TDuration::Seconds(20)) -{} - -void TChannelPool::GetStubsHolderLocked( - const TString& channelId, - const TGRpcClientConfig& config, - std::function<void(TStubsHolder&)> cb) -{ - { - std::shared_lock readGuard(RWMutex_); - const auto it = Pool_.find(channelId); - if (it != Pool_.end()) { +{} + +void TChannelPool::GetStubsHolderLocked( + const TString& channelId, + const TGRpcClientConfig& config, + std::function<void(TStubsHolder&)> cb) +{ + { + std::shared_lock readGuard(RWMutex_); + const auto it = Pool_.find(channelId); + if (it != Pool_.end()) { if (!it->second.IsChannelBroken() && !(Now() > it->second.GetLastUseTime() + UpdateReUseTime_)) { - return cb(it->second); - } - } - } - { - std::unique_lock writeGuard(RWMutex_); - { - auto it = Pool_.find(channelId); - if (it != Pool_.end()) { - if (!it->second.IsChannelBroken()) { + return cb(it->second); + } + } + } + { + std::unique_lock writeGuard(RWMutex_); + { + auto it = Pool_.find(channelId); + if (it != Pool_.end()) { + if (!it->second.IsChannelBroken()) { EraseFromQueueByTime(it->second.GetLastUseTime(), channelId); auto now = Now(); LastUsedQueue_.emplace(now, channelId); it->second.SetLastUseTime(now); - return cb(it->second); - } else { - // This channel can't be used. Remove from pool to create new one + return cb(it->second); + } else { + // This channel can't be used. Remove from pool to create new one EraseFromQueueByTime(it->second.GetLastUseTime(), channelId); - Pool_.erase(it); - } - } - } - TGRpcKeepAliveSocketMutator* mutator = nullptr; - // will be destroyed inside grpc - if (TcpKeepAliveSettings_.Enabled) { - mutator = new TGRpcKeepAliveSocketMutator( - TcpKeepAliveSettings_.Idle, - TcpKeepAliveSettings_.Count, - TcpKeepAliveSettings_.Interval - ); - } - cb(Pool_.emplace(channelId, CreateChannelInterface(config, mutator)).first->second); + Pool_.erase(it); + } + } + } + TGRpcKeepAliveSocketMutator* mutator = nullptr; + // will be destroyed inside grpc + if (TcpKeepAliveSettings_.Enabled) { + mutator = new TGRpcKeepAliveSocketMutator( + TcpKeepAliveSettings_.Idle, + TcpKeepAliveSettings_.Count, + TcpKeepAliveSettings_.Interval + ); + } + cb(Pool_.emplace(channelId, CreateChannelInterface(config, mutator)).first->second); LastUsedQueue_.emplace(Pool_.at(channelId).GetLastUseTime(), channelId); - } -} - -void TChannelPool::DeleteChannel(const TString& channelId) { - std::unique_lock writeLock(RWMutex_); + } +} + +void TChannelPool::DeleteChannel(const TString& channelId) { + std::unique_lock writeLock(RWMutex_); auto poolIt = Pool_.find(channelId); if (poolIt != Pool_.end()) { EraseFromQueueByTime(poolIt->second.GetLastUseTime(), channelId); Pool_.erase(poolIt); } -} - +} + void TChannelPool::DeleteExpiredStubsHolders() { std::unique_lock writeLock(RWMutex_); auto lastExpired = LastUsedQueue_.lower_bound(Now() - ExpireTime_); @@ -392,7 +392,7 @@ private: // Some children are stored inline, others are in a set std::array<TContextImpl*, 2> InlineChildren{ { nullptr, nullptr } }; - std::unordered_set<TContextImpl*> Children; + std::unordered_set<TContextImpl*> Children; // Single callback is stored without extra allocations TStackVec<TCallback, 1> Callbacks; @@ -404,10 +404,10 @@ private: TGRpcClientLow::TGRpcClientLow(size_t numWorkerThread, bool useCompletionQueuePerThread) : UseCompletionQueuePerThread_(useCompletionQueuePerThread) { - Init(numWorkerThread); -} - -void TGRpcClientLow::Init(size_t numWorkerThread) { + Init(numWorkerThread); +} + +void TGRpcClientLow::Init(size_t numWorkerThread) { SetCqState(WORKING); if (UseCompletionQueuePerThread_) { for (size_t i = 0; i < numWorkerThread; i++) { @@ -425,9 +425,9 @@ void TGRpcClientLow::Init(size_t numWorkerThread) { PullEvents(cq); }).Release()); } - } -} - + } +} + void TGRpcClientLow::AddWorkerThreadForTest() { if (UseCompletionQueuePerThread_) { CQS_.push_back(std::make_unique<grpc::CompletionQueue>()); @@ -453,17 +453,17 @@ void TGRpcClientLow::Stop(bool wait) { if (wait) { WaitInternal(); - } -} - + } +} + void TGRpcClientLow::StopInternal(bool silent) { bool shutdown; TVector<TContextImpl::TContextPtr> cancelQueue; - { - std::unique_lock<std::mutex> guard(Mtx_); - + { + std::unique_lock<std::mutex> guard(Mtx_); + auto allowStateChange = [&]() { switch (GetCqState()) { case WORKING: @@ -484,7 +484,7 @@ void TGRpcClientLow::StopInternal(bool silent) { SetCqState(silent ? STOP_SILENT : STOP_EXPLICIT); - if (!silent && !Contexts_.empty()) { + if (!silent && !Contexts_.empty()) { cancelQueue.reserve(Contexts_.size()); for (auto* ptr : Contexts_) { // N.B. some contexts may be stuck in destructors @@ -494,7 +494,7 @@ void TGRpcClientLow::StopInternal(bool silent) { } } - shutdown = Contexts_.empty(); + shutdown = Contexts_.empty(); } for (auto& context : cancelQueue) { @@ -506,62 +506,62 @@ void TGRpcClientLow::StopInternal(bool silent) { for (auto& cq : CQS_) { cq->Shutdown(); } - } + } } void TGRpcClientLow::WaitInternal() { - std::unique_lock<std::mutex> guard(JoinMutex_); - - for (auto& ti : WorkerThreads_) { - ti->Join(); - } -} - + std::unique_lock<std::mutex> guard(JoinMutex_); + + for (auto& ti : WorkerThreads_) { + ti->Join(); + } +} + void TGRpcClientLow::WaitIdle() { - std::unique_lock<std::mutex> guard(Mtx_); - - while (!Contexts_.empty()) { - ContextsEmpty_.wait(guard); - } -} - + std::unique_lock<std::mutex> guard(Mtx_); + + while (!Contexts_.empty()) { + ContextsEmpty_.wait(guard); + } +} + std::shared_ptr<IQueueClientContext> TGRpcClientLow::CreateContext() { - std::unique_lock<std::mutex> guard(Mtx_); - - auto allowCreateContext = [&]() { - switch (GetCqState()) { - case WORKING: - return true; - case STOP_SILENT: - case STOP_EXPLICIT: - return false; + std::unique_lock<std::mutex> guard(Mtx_); + + auto allowCreateContext = [&]() { + switch (GetCqState()) { + case WORKING: + return true; + case STOP_SILENT: + case STOP_EXPLICIT: + return false; } - Y_UNREACHABLE(); - }; - - if (!allowCreateContext()) { - // New context creation is forbidden - return nullptr; - } - - auto context = std::make_shared<TContextImpl>(); - Contexts_.insert(context.get()); - context->Owner = this; - if (UseCompletionQueuePerThread_) { - context->CQ = CQS_[RandomNumber(CQS_.size())].get(); - } else { - context->CQ = CQS_[0].get(); - } - return context; + Y_UNREACHABLE(); + }; + + if (!allowCreateContext()) { + // New context creation is forbidden + return nullptr; + } + + auto context = std::make_shared<TContextImpl>(); + Contexts_.insert(context.get()); + context->Owner = this; + if (UseCompletionQueuePerThread_) { + context->CQ = CQS_[RandomNumber(CQS_.size())].get(); + } else { + context->CQ = CQS_[0].get(); + } + return context; } void TGRpcClientLow::ForgetContext(TContextImpl* context) { bool shutdown = false; - { - std::unique_lock<std::mutex> guard(Mtx_); - + { + std::unique_lock<std::mutex> guard(Mtx_); + if (!Contexts_.erase(context)) { Y_FAIL("Unexpected ForgetContext(%p)", context); } @@ -571,7 +571,7 @@ void TGRpcClientLow::ForgetContext(TContextImpl* context) { shutdown = true; } - ContextsEmpty_.notify_all(); + ContextsEmpty_.notify_all(); } } @@ -580,7 +580,7 @@ void TGRpcClientLow::ForgetContext(TContextImpl* context) { for (auto& cq : CQS_) { cq->Shutdown(); } - } -} - + } +} + } // namespace NGRpc diff --git a/library/cpp/grpc/client/grpc_client_low.h b/library/cpp/grpc/client/grpc_client_low.h index e452be45a1..ab0a0627be 100644 --- a/library/cpp/grpc/client/grpc_client_low.h +++ b/library/cpp/grpc/client/grpc_client_low.h @@ -1,57 +1,57 @@ -#pragma once - -#include "grpc_common.h" - +#pragma once + +#include "grpc_common.h" + #include <util/thread/factory.h> -#include <grpc++/grpc++.h> +#include <grpc++/grpc++.h> #include <grpc++/support/async_stream.h> #include <grpc++/support/async_unary_call.h> - -#include <deque> -#include <typeindex> -#include <typeinfo> + +#include <deque> +#include <typeindex> +#include <typeinfo> #include <variant> -#include <vector> -#include <unordered_map> -#include <unordered_set> -#include <mutex> -#include <shared_mutex> - -/* - * This file contains low level logic for grpc - * This file should not be used in high level code without special reason - */ +#include <vector> +#include <unordered_map> +#include <unordered_set> +#include <mutex> +#include <shared_mutex> + +/* + * This file contains low level logic for grpc + * This file should not be used in high level code without special reason + */ namespace NGrpc { - -const size_t DEFAULT_NUM_THREADS = 2; - -//////////////////////////////////////////////////////////////////////////////// - -void EnableGRpcTracing(); - -//////////////////////////////////////////////////////////////////////////////// - -struct TTcpKeepAliveSettings { - bool Enabled; - size_t Idle; - size_t Count; - size_t Interval; -}; - -//////////////////////////////////////////////////////////////////////////////// - -// Common interface used to execute action from grpc cq routine -class IQueueClientEvent { -public: - virtual ~IQueueClientEvent() = default; - + +const size_t DEFAULT_NUM_THREADS = 2; + +//////////////////////////////////////////////////////////////////////////////// + +void EnableGRpcTracing(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TTcpKeepAliveSettings { + bool Enabled; + size_t Idle; + size_t Count; + size_t Interval; +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Common interface used to execute action from grpc cq routine +class IQueueClientEvent { +public: + virtual ~IQueueClientEvent() = default; + //! Execute an action defined by implementation - virtual bool Execute(bool ok) = 0; - - //! Finish and destroy event - virtual void Destroy() = 0; -}; - + virtual bool Execute(bool ok) = 0; + + //! Finish and destroy event + virtual void Destroy() = 0; +}; + // Implementation of IQueueClientEvent that reduces allocations template<class TSelf> class TQueueClientFixedEvent : private IQueueClientEvent { @@ -123,8 +123,8 @@ public: } }; -// Represents grpc status and error message string -struct TGrpcStatus { +// Represents grpc status and error message string +struct TGrpcStatus { TString Msg; TString Details; int GRpcStatusCode; @@ -167,36 +167,36 @@ struct TGrpcStatus { bool Ok() const { return !InternalError && GRpcStatusCode == grpc::StatusCode::OK; } -}; - -bool inline IsGRpcStatusGood(const TGrpcStatus& status) { +}; + +bool inline IsGRpcStatusGood(const TGrpcStatus& status) { return status.Ok(); -} - +} + // Response callback type - this callback will be called when request is finished -// (or after getting each chunk in case of streaming mode) -template<typename TResponse> -using TResponseCallback = std::function<void (TGrpcStatus&&, TResponse&&)>; - +// (or after getting each chunk in case of streaming mode) +template<typename TResponse> +using TResponseCallback = std::function<void (TGrpcStatus&&, TResponse&&)>; + template<typename TResponse> using TAdvancedResponseCallback = std::function<void (const grpc::ClientContext&, TGrpcStatus&&, TResponse&&)>; -// Call associated metadata -struct TCallMeta { - std::shared_ptr<grpc::CallCredentials> CallCredentials; - std::vector<std::pair<TString, TString>> Aux; +// Call associated metadata +struct TCallMeta { + std::shared_ptr<grpc::CallCredentials> CallCredentials; + std::vector<std::pair<TString, TString>> Aux; std::variant<TDuration, TInstant> Timeout; // timeout as duration from now or time point in future -}; - -class TGRpcRequestProcessorCommon { -protected: - void ApplyMeta(const TCallMeta& meta) { - for (const auto& rec : meta.Aux) { - Context.AddMetadata(rec.first, rec.second); - } - if (meta.CallCredentials) { - Context.set_credentials(meta.CallCredentials); - } +}; + +class TGRpcRequestProcessorCommon { +protected: + void ApplyMeta(const TCallMeta& meta) { + for (const auto& rec : meta.Aux) { + Context.AddMetadata(rec.first, rec.second); + } + if (meta.CallCredentials) { + Context.set_credentials(meta.CallCredentials); + } if (const TDuration* timeout = std::get_if<TDuration>(&meta.Timeout)) { if (*timeout) { auto deadline = gpr_time_add( @@ -208,10 +208,10 @@ protected: if (*deadline) { Context.set_deadline(gpr_time_from_micros(deadline->MicroSeconds(), GPR_CLOCK_MONOTONIC)); } - } - } + } + } - void GetInitialMetadata(std::unordered_multimap<TString, TString>* metadata) { + void GetInitialMetadata(std::unordered_multimap<TString, TString>* metadata) { for (const auto& [key, value] : Context.GetServerInitialMetadata()) { metadata->emplace( TString(key.begin(), key.end()), @@ -220,61 +220,61 @@ protected: } } - grpc::Status Status; - grpc::ClientContext Context; - std::shared_ptr<IQueueClientContext> LocalContext; -}; - -template<typename TStub, typename TRequest, typename TResponse> -class TSimpleRequestProcessor - : public TThrRefBase - , public IQueueClientEvent - , public TGRpcRequestProcessorCommon { - using TAsyncReaderPtr = std::unique_ptr<grpc::ClientAsyncResponseReader<TResponse>>; - template<typename> friend class TServiceConnection; -public: - using TPtr = TIntrusivePtr<TSimpleRequestProcessor>; - using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); - + grpc::Status Status; + grpc::ClientContext Context; + std::shared_ptr<IQueueClientContext> LocalContext; +}; + +template<typename TStub, typename TRequest, typename TResponse> +class TSimpleRequestProcessor + : public TThrRefBase + , public IQueueClientEvent + , public TGRpcRequestProcessorCommon { + using TAsyncReaderPtr = std::unique_ptr<grpc::ClientAsyncResponseReader<TResponse>>; + template<typename> friend class TServiceConnection; +public: + using TPtr = TIntrusivePtr<TSimpleRequestProcessor>; + using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); + explicit TSimpleRequestProcessor(TResponseCallback<TResponse>&& callback) : Callback_(std::move(callback)) - { } - - ~TSimpleRequestProcessor() { + { } + + ~TSimpleRequestProcessor() { if (!Replied_ && Callback_) { Callback_(TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); Callback_ = nullptr; // free resources as early as possible - } - } - - bool Execute(bool ok) override { - { - std::unique_lock<std::mutex> guard(Mutex_); - LocalContext.reset(); + } + } + + bool Execute(bool ok) override { + { + std::unique_lock<std::mutex> guard(Mutex_); + LocalContext.reset(); } TGrpcStatus status; if (ok) { - status = Status; + status = Status; } else { status = TGrpcStatus::Internal("Unexpected error"); - } - Replied_ = true; - Callback_(std::move(status), std::move(Reply_)); + } + Replied_ = true; + Callback_(std::move(status), std::move(Reply_)); Callback_ = nullptr; // free resources as early as possible - return false; - } - - void Destroy() override { + return false; + } + + void Destroy() override { UnRef(); - } - -private: + } + +private: IQueueClientEvent* FinishedEvent() { Ref(); return this; - } - - void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { + } + + void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { auto context = provider->CreateContext(); if (!context) { Replied_ = true; @@ -282,11 +282,11 @@ private: Callback_ = nullptr; return; } - { - std::unique_lock<std::mutex> guard(Mutex_); - LocalContext = context; - Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); - Reader_->Finish(&Reply_, &Status, FinishedEvent()); + { + std::unique_lock<std::mutex> guard(Mutex_); + LocalContext = context; + Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); + Reader_->Finish(&Reply_, &Status, FinishedEvent()); } context->SubscribeStop([self = TPtr(this)] { self->Stop(); @@ -294,17 +294,17 @@ private: } void Stop() { - Context.TryCancel(); + Context.TryCancel(); } TResponseCallback<TResponse> Callback_; - TResponse Reply_; - std::mutex Mutex_; - TAsyncReaderPtr Reader_; - - bool Replied_ = false; -}; - + TResponse Reply_; + std::mutex Mutex_; + TAsyncReaderPtr Reader_; + + bool Replied_ = false; +}; + template<typename TStub, typename TRequest, typename TResponse> class TAdvancedRequestProcessor : public TThrRefBase @@ -328,8 +328,8 @@ public: } bool Execute(bool ok) override { - { - std::unique_lock<std::mutex> guard(Mutex_); + { + std::unique_lock<std::mutex> guard(Mutex_); LocalContext.reset(); } TGrpcStatus status; @@ -362,8 +362,8 @@ private: Callback_ = nullptr; return; } - { - std::unique_lock<std::mutex> guard(Mutex_); + { + std::unique_lock<std::mutex> guard(Mutex_); LocalContext = context; Reader_ = (stub.*asyncRequest)(&Context, request, context->CompletionQueue()); Reader_->Finish(&Reply_, &Status, FinishedEvent()); @@ -379,16 +379,16 @@ private: TAdvancedResponseCallback<TResponse> Callback_; TResponse Reply_; - std::mutex Mutex_; + std::mutex Mutex_; TAsyncReaderPtr Reader_; bool Replied_ = false; }; -template<class TResponse> -class IStreamRequestReadProcessor : public TThrRefBase { +template<class TResponse> +class IStreamRequestReadProcessor : public TThrRefBase { public: - using TPtr = TIntrusivePtr<IStreamRequestReadProcessor>; + using TPtr = TIntrusivePtr<IStreamRequestReadProcessor>; using TReadCallback = std::function<void(TGrpcStatus&&)>; /** @@ -399,7 +399,7 @@ public: /** * Scheduled initial server metadata read from the stream */ - virtual void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) = 0; + virtual void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) = 0; /** * Scheduled response read from the stream @@ -420,72 +420,72 @@ public: virtual void AddFinishedCallback(TReadCallback callback) = 0; }; -template<class TRequest, class TResponse> -class IStreamRequestReadWriteProcessor : public IStreamRequestReadProcessor<TResponse> { -public: - using TPtr = TIntrusivePtr<IStreamRequestReadWriteProcessor>; +template<class TRequest, class TResponse> +class IStreamRequestReadWriteProcessor : public IStreamRequestReadProcessor<TResponse> { +public: + using TPtr = TIntrusivePtr<IStreamRequestReadWriteProcessor>; using TWriteCallback = std::function<void(TGrpcStatus&&)>; - - /** - * Scheduled request write to the stream - */ + + /** + * Scheduled request write to the stream + */ virtual void Write(TRequest&& request, TWriteCallback callback = { }) = 0; -}; - -class TGRpcKeepAliveSocketMutator; - -// Class to hold stubs allocated on channel. -// It is poor documented part of grpc. See KIKIMR-6109 and comment to this commit - -// Stub holds shared_ptr<ChannelInterface>, so we can destroy this holder even if -// request processor using stub -class TStubsHolder : public TNonCopyable { - using TypeInfoRef = std::reference_wrapper<const std::type_info>; - - struct THasher { - std::size_t operator()(TypeInfoRef code) const { - return code.get().hash_code(); - } - }; - - struct TEqualTo { - bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { - return lhs.get() == rhs.get(); - } - }; -public: - TStubsHolder(std::shared_ptr<grpc::ChannelInterface> channel) - : ChannelInterface_(channel) - {} - - // Returns true if channel can't be used to perform request now - bool IsChannelBroken() const { - auto state = ChannelInterface_->GetState(false); - return state == GRPC_CHANNEL_SHUTDOWN || - state == GRPC_CHANNEL_TRANSIENT_FAILURE; - } - - template<typename TStub> - std::shared_ptr<TStub> GetOrCreateStub() { - const auto& stubId = typeid(TStub); - { - std::shared_lock readGuard(RWMutex_); - const auto it = Stubs_.find(stubId); - if (it != Stubs_.end()) { - return std::static_pointer_cast<TStub>(it->second); - } - } - { - std::unique_lock writeGuard(RWMutex_); - auto it = Stubs_.emplace(stubId, nullptr); - if (!it.second) { - return std::static_pointer_cast<TStub>(it.first->second); - } else { - it.first->second = std::make_shared<TStub>(ChannelInterface_); - return std::static_pointer_cast<TStub>(it.first->second); - } - } - } +}; + +class TGRpcKeepAliveSocketMutator; + +// Class to hold stubs allocated on channel. +// It is poor documented part of grpc. See KIKIMR-6109 and comment to this commit + +// Stub holds shared_ptr<ChannelInterface>, so we can destroy this holder even if +// request processor using stub +class TStubsHolder : public TNonCopyable { + using TypeInfoRef = std::reference_wrapper<const std::type_info>; + + struct THasher { + std::size_t operator()(TypeInfoRef code) const { + return code.get().hash_code(); + } + }; + + struct TEqualTo { + bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { + return lhs.get() == rhs.get(); + } + }; +public: + TStubsHolder(std::shared_ptr<grpc::ChannelInterface> channel) + : ChannelInterface_(channel) + {} + + // Returns true if channel can't be used to perform request now + bool IsChannelBroken() const { + auto state = ChannelInterface_->GetState(false); + return state == GRPC_CHANNEL_SHUTDOWN || + state == GRPC_CHANNEL_TRANSIENT_FAILURE; + } + + template<typename TStub> + std::shared_ptr<TStub> GetOrCreateStub() { + const auto& stubId = typeid(TStub); + { + std::shared_lock readGuard(RWMutex_); + const auto it = Stubs_.find(stubId); + if (it != Stubs_.end()) { + return std::static_pointer_cast<TStub>(it->second); + } + } + { + std::unique_lock writeGuard(RWMutex_); + auto it = Stubs_.emplace(stubId, nullptr); + if (!it.second) { + return std::static_pointer_cast<TStub>(it.first->second); + } else { + it.first->second = std::make_shared<TStub>(ChannelInterface_); + return std::static_pointer_cast<TStub>(it.first->second); + } + } + } const TInstant& GetLastUseTime() const { return LastUsed_; @@ -494,76 +494,76 @@ public: void SetLastUseTime(const TInstant& time) { LastUsed_ = time; } -private: +private: TInstant LastUsed_ = Now(); - std::shared_mutex RWMutex_; - std::unordered_map<TypeInfoRef, std::shared_ptr<void>, THasher, TEqualTo> Stubs_; - std::shared_ptr<grpc::ChannelInterface> ChannelInterface_; -}; - -class TChannelPool { -public: + std::shared_mutex RWMutex_; + std::unordered_map<TypeInfoRef, std::shared_ptr<void>, THasher, TEqualTo> Stubs_; + std::shared_ptr<grpc::ChannelInterface> ChannelInterface_; +}; + +class TChannelPool { +public: TChannelPool(const TTcpKeepAliveSettings& tcpKeepAliveSettings, const TDuration& expireTime = TDuration::Minutes(6)); - //Allows to CreateStub from TStubsHolder under lock - //The callback will be called just during GetStubsHolderLocked call - void GetStubsHolderLocked(const TString& channelId, const TGRpcClientConfig& config, std::function<void(TStubsHolder&)> cb); - void DeleteChannel(const TString& channelId); + //Allows to CreateStub from TStubsHolder under lock + //The callback will be called just during GetStubsHolderLocked call + void GetStubsHolderLocked(const TString& channelId, const TGRpcClientConfig& config, std::function<void(TStubsHolder&)> cb); + void DeleteChannel(const TString& channelId); void DeleteExpiredStubsHolders(); -private: - std::shared_mutex RWMutex_; - std::unordered_map<TString, TStubsHolder> Pool_; +private: + std::shared_mutex RWMutex_; + std::unordered_map<TString, TStubsHolder> Pool_; std::multimap<TInstant, TString> LastUsedQueue_; - TTcpKeepAliveSettings TcpKeepAliveSettings_; + TTcpKeepAliveSettings TcpKeepAliveSettings_; TDuration ExpireTime_; TDuration UpdateReUseTime_; void EraseFromQueueByTime(const TInstant& lastUseTime, const TString& channelId); -}; - -template<class TResponse> -using TStreamReaderCallback = std::function<void(TGrpcStatus&&, typename IStreamRequestReadProcessor<TResponse>::TPtr)>; - -template<typename TStub, typename TRequest, typename TResponse> -class TStreamRequestReadProcessor - : public IStreamRequestReadProcessor<TResponse> - , public TGRpcRequestProcessorCommon { - template<typename> friend class TServiceConnection; -public: - using TSelf = TStreamRequestReadProcessor; - using TAsyncReaderPtr = std::unique_ptr<grpc::ClientAsyncReader<TResponse>>; - using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*, void*); - using TReaderCallback = TStreamReaderCallback<TResponse>; - using TPtr = TIntrusivePtr<TSelf>; - using TBase = IStreamRequestReadProcessor<TResponse>; - using TReadCallback = typename TBase::TReadCallback; - - explicit TStreamRequestReadProcessor(TReaderCallback&& callback) - : Callback(std::move(callback)) - { - Y_VERIFY(Callback, "Missing connected callback"); - } - - void Cancel() override { - Context.TryCancel(); - - { - std::unique_lock<std::mutex> guard(Mutex); - Cancelled = true; +}; + +template<class TResponse> +using TStreamReaderCallback = std::function<void(TGrpcStatus&&, typename IStreamRequestReadProcessor<TResponse>::TPtr)>; + +template<typename TStub, typename TRequest, typename TResponse> +class TStreamRequestReadProcessor + : public IStreamRequestReadProcessor<TResponse> + , public TGRpcRequestProcessorCommon { + template<typename> friend class TServiceConnection; +public: + using TSelf = TStreamRequestReadProcessor; + using TAsyncReaderPtr = std::unique_ptr<grpc::ClientAsyncReader<TResponse>>; + using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*, void*); + using TReaderCallback = TStreamReaderCallback<TResponse>; + using TPtr = TIntrusivePtr<TSelf>; + using TBase = IStreamRequestReadProcessor<TResponse>; + using TReadCallback = typename TBase::TReadCallback; + + explicit TStreamRequestReadProcessor(TReaderCallback&& callback) + : Callback(std::move(callback)) + { + Y_VERIFY(Callback, "Missing connected callback"); + } + + void Cancel() override { + Context.TryCancel(); + + { + std::unique_lock<std::mutex> guard(Mutex); + Cancelled = true; if (Started && !ReadFinished) { - if (!ReadActive) { - ReadFinished = true; - } - if (ReadFinished) { - Stream->Finish(&Status, OnFinishedTag.Prepare()); - } - } - } - } - - void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { + if (!ReadActive) { + ReadFinished = true; + } + if (ReadFinished) { + Stream->Finish(&Status, OnFinishedTag.Prepare()); + } + } + } + } + + void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); if (!Finished && !HasInitialMetadata) { ReadActive = true; @@ -588,66 +588,66 @@ public: callback(std::move(status)); } - void Read(TResponse* message, TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock<std::mutex> guard(Mutex); - Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - ReadCallback = std::move(callback); - if (!ReadFinished) { - Stream->Read(message, OnReadDoneTag.Prepare()); - } - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - } - - callback(std::move(status)); - } - - void Finish(TReadCallback callback) override { - TGrpcStatus status; - - { - std::unique_lock<std::mutex> guard(Mutex); - Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); - if (!Finished) { - ReadActive = true; - FinishCallback = std::move(callback); - if (!ReadFinished) { - ReadFinished = true; - } - Stream->Finish(&Status, OnFinishedTag.Prepare()); - return; - } - if (FinishedOk) { - status = Status; - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - } - - callback(std::move(status)); - } + void Read(TResponse* message, TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock<std::mutex> guard(Mutex); + Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + ReadCallback = std::move(callback); + if (!ReadFinished) { + Stream->Read(message, OnReadDoneTag.Prepare()); + } + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + if (status.Ok()) { + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + } + + callback(std::move(status)); + } + + void Finish(TReadCallback callback) override { + TGrpcStatus status; + + { + std::unique_lock<std::mutex> guard(Mutex); + Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); + if (!Finished) { + ReadActive = true; + FinishCallback = std::move(callback); + if (!ReadFinished) { + ReadFinished = true; + } + Stream->Finish(&Status, OnFinishedTag.Prepare()); + return; + } + if (FinishedOk) { + status = Status; + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + } + + callback(std::move(status)); + } void AddFinishedCallback(TReadCallback callback) override { Y_VERIFY(callback, "Unexpected empty callback"); TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); if (!Finished) { FinishedCallbacks.emplace_back().swap(callback); return; @@ -665,103 +665,103 @@ public: callback(std::move(status)); } -private: - void Start(TStub& stub, const TRequest& request, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - auto callback = std::move(Callback); - TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); - callback(std::move(status), nullptr); - return; - } - - { - std::unique_lock<std::mutex> guard(Mutex); - LocalContext = context; - Stream = (stub.*asyncRequest)(&Context, request, context->CompletionQueue(), OnStartDoneTag.Prepare()); - } - - context->SubscribeStop([self = TPtr(this)] { - self->Cancel(); - }); - } - - void OnReadDone(bool ok) { - TGrpcStatus status; - TReadCallback callback; - std::unordered_multimap<TString, TString>* initialMetadata = nullptr; - - { - std::unique_lock<std::mutex> guard(Mutex); - Y_VERIFY(ReadActive, "Unexpected Read done callback"); - Y_VERIFY(!ReadFinished, "Unexpected ReadFinished flag"); - - if (!ok || Cancelled) { - ReadFinished = true; - - Stream->Finish(&Status, OnFinishedTag.Prepare()); - if (!ok) { - // Keep ReadActive=true, so callback is called - // after the call is finished with an error - return; - } - } - - callback = std::move(ReadCallback); - ReadCallback = nullptr; - ReadActive = false; +private: + void Start(TStub& stub, const TRequest& request, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + auto callback = std::move(Callback); + TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); + callback(std::move(status), nullptr); + return; + } + + { + std::unique_lock<std::mutex> guard(Mutex); + LocalContext = context; + Stream = (stub.*asyncRequest)(&Context, request, context->CompletionQueue(), OnStartDoneTag.Prepare()); + } + + context->SubscribeStop([self = TPtr(this)] { + self->Cancel(); + }); + } + + void OnReadDone(bool ok) { + TGrpcStatus status; + TReadCallback callback; + std::unordered_multimap<TString, TString>* initialMetadata = nullptr; + + { + std::unique_lock<std::mutex> guard(Mutex); + Y_VERIFY(ReadActive, "Unexpected Read done callback"); + Y_VERIFY(!ReadFinished, "Unexpected ReadFinished flag"); + + if (!ok || Cancelled) { + ReadFinished = true; + + Stream->Finish(&Status, OnFinishedTag.Prepare()); + if (!ok) { + // Keep ReadActive=true, so callback is called + // after the call is finished with an error + return; + } + } + + callback = std::move(ReadCallback); + ReadCallback = nullptr; + ReadActive = false; initialMetadata = InitialMetadata; InitialMetadata = nullptr; HasInitialMetadata = true; - } - + } + if (initialMetadata) { GetInitialMetadata(initialMetadata); } - callback(std::move(status)); - } - - void OnStartDone(bool ok) { - TReaderCallback callback; - - { - std::unique_lock<std::mutex> guard(Mutex); + callback(std::move(status)); + } + + void OnStartDone(bool ok) { + TReaderCallback callback; + + { + std::unique_lock<std::mutex> guard(Mutex); Started = true; if (!ok || Cancelled) { - ReadFinished = true; - Stream->Finish(&Status, OnFinishedTag.Prepare()); - return; - } - callback = std::move(Callback); - Callback = nullptr; - } - - callback({ }, typename TBase::TPtr(this)); - } - - void OnFinished(bool ok) { - TGrpcStatus status; - std::vector<TReadCallback> finishedCallbacks; + ReadFinished = true; + Stream->Finish(&Status, OnFinishedTag.Prepare()); + return; + } + callback = std::move(Callback); + Callback = nullptr; + } + + callback({ }, typename TBase::TPtr(this)); + } + + void OnFinished(bool ok) { + TGrpcStatus status; + std::vector<TReadCallback> finishedCallbacks; TReaderCallback startCallback; - TReadCallback readCallback; - TReadCallback finishCallback; - - { - std::unique_lock<std::mutex> guard(Mutex); - - Finished = true; - FinishedOk = ok; - LocalContext.reset(); - - if (ok) { - status = Status; + TReadCallback readCallback; + TReadCallback finishCallback; + + { + std::unique_lock<std::mutex> guard(Mutex); + + Finished = true; + FinishedOk = ok; + LocalContext.reset(); + + if (ok) { + status = Status; } else if (Cancelled) { status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Stream cancelled"); - } else { - status = TGrpcStatus::Internal("Unexpected error"); - } - + } else { + status = TGrpcStatus::Internal("Unexpected error"); + } + finishedCallbacks.swap(FinishedCallbacks); if (Callback) { @@ -769,68 +769,68 @@ private: startCallback = std::move(Callback); Callback = nullptr; } else if (ReadActive) { - if (ReadCallback) { - readCallback = std::move(ReadCallback); - ReadCallback = nullptr; - } else { - finishCallback = std::move(FinishCallback); - FinishCallback = nullptr; - } - ReadActive = false; - } - } - + if (ReadCallback) { + readCallback = std::move(ReadCallback); + ReadCallback = nullptr; + } else { + finishCallback = std::move(FinishCallback); + FinishCallback = nullptr; + } + ReadActive = false; + } + } + for (auto& finishedCallback : finishedCallbacks) { auto statusCopy = status; finishedCallback(std::move(statusCopy)); } if (startCallback) { - if (status.Ok()) { + if (status.Ok()) { status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); } startCallback(std::move(status), nullptr); } else if (readCallback) { if (status.Ok()) { - status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); - } - readCallback(std::move(status)); - } else if (finishCallback) { - finishCallback(std::move(status)); - } - } - - TReaderCallback Callback; - TAsyncReaderPtr Stream; - using TFixedEvent = TQueueClientFixedEvent<TSelf>; - std::mutex Mutex; - TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; - TFixedEvent OnStartDoneTag = { this, &TSelf::OnStartDone }; - TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; - - TReadCallback ReadCallback; - TReadCallback FinishCallback; - std::vector<TReadCallback> FinishedCallbacks; - std::unordered_multimap<TString, TString>* InitialMetadata = nullptr; + status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); + } + readCallback(std::move(status)); + } else if (finishCallback) { + finishCallback(std::move(status)); + } + } + + TReaderCallback Callback; + TAsyncReaderPtr Stream; + using TFixedEvent = TQueueClientFixedEvent<TSelf>; + std::mutex Mutex; + TFixedEvent OnReadDoneTag = { this, &TSelf::OnReadDone }; + TFixedEvent OnStartDoneTag = { this, &TSelf::OnStartDone }; + TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; + + TReadCallback ReadCallback; + TReadCallback FinishCallback; + std::vector<TReadCallback> FinishedCallbacks; + std::unordered_multimap<TString, TString>* InitialMetadata = nullptr; bool Started = false; bool HasInitialMetadata = false; - bool ReadActive = false; - bool ReadFinished = false; - bool Finished = false; - bool Cancelled = false; - bool FinishedOk = false; -}; - + bool ReadActive = false; + bool ReadFinished = false; + bool Finished = false; + bool Cancelled = false; + bool FinishedOk = false; +}; + template<class TRequest, class TResponse> -using TStreamConnectedCallback = std::function<void(TGrpcStatus&&, typename IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr)>; +using TStreamConnectedCallback = std::function<void(TGrpcStatus&&, typename IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr)>; template<class TStub, class TRequest, class TResponse> -class TStreamRequestReadWriteProcessor - : public IStreamRequestReadWriteProcessor<TRequest, TResponse> - , public TGRpcRequestProcessorCommon { +class TStreamRequestReadWriteProcessor + : public IStreamRequestReadWriteProcessor<TRequest, TResponse> + , public TGRpcRequestProcessorCommon { public: - using TSelf = TStreamRequestReadWriteProcessor; - using TBase = IStreamRequestReadWriteProcessor<TRequest, TResponse>; + using TSelf = TStreamRequestReadWriteProcessor; + using TBase = IStreamRequestReadWriteProcessor<TRequest, TResponse>; using TPtr = TIntrusivePtr<TSelf>; using TConnectedCallback = TStreamConnectedCallback<TRequest, TResponse>; using TReadCallback = typename TBase::TReadCallback; @@ -838,7 +838,7 @@ public: using TAsyncReaderWriterPtr = std::unique_ptr<grpc::ClientAsyncReaderWriter<TRequest, TResponse>>; using TAsyncRequest = TAsyncReaderWriterPtr (TStub::*)(grpc::ClientContext*, grpc::CompletionQueue*, void*); - explicit TStreamRequestReadWriteProcessor(TConnectedCallback&& callback) + explicit TStreamRequestReadWriteProcessor(TConnectedCallback&& callback) : ConnectedCallback(std::move(callback)) { Y_VERIFY(ConnectedCallback, "Missing connected callback"); @@ -847,8 +847,8 @@ public: void Cancel() override { Context.TryCancel(); - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Cancelled = true; if (Started && !(ReadFinished && WriteFinished)) { if (!ReadActive) { @@ -867,8 +867,8 @@ public: void Write(TRequest&& request, TWriteCallback callback) override { TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); if (Cancelled || ReadFinished || WriteFinished) { status = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); } else if (WriteActive) { @@ -887,11 +887,11 @@ public: } } - void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { + void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); if (!Finished && !HasInitialMetadata) { ReadActive = true; @@ -919,8 +919,8 @@ public: void Read(TResponse* message, TReadCallback callback) override { TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); if (!Finished) { ReadActive = true; @@ -947,8 +947,8 @@ public: void Finish(TReadCallback callback) override { TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(!ReadActive, "Multiple Read/Finish calls detected"); if (!Finished) { ReadActive = true; @@ -979,8 +979,8 @@ public: TGrpcStatus status; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); if (!Finished) { FinishedCallbacks.emplace_back().swap(callback); return; @@ -1010,8 +1010,8 @@ private: return; } - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); LocalContext = context; Stream = (stub.*asyncRequest)(&Context, context->CompletionQueue(), OnConnectedTag.Prepare()); } @@ -1025,8 +1025,8 @@ private: void OnConnected(bool ok) { TConnectedCallback callback; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Started = true; if (!ok || Cancelled) { ReadFinished = true; @@ -1045,10 +1045,10 @@ private: void OnReadDone(bool ok) { TGrpcStatus status; TReadCallback callback; - std::unordered_multimap<TString, TString>* initialMetadata = nullptr; + std::unordered_multimap<TString, TString>* initialMetadata = nullptr; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(ReadActive, "Unexpected Read done callback"); Y_VERIFY(!ReadFinished, "Unexpected ReadFinished flag"); @@ -1085,8 +1085,8 @@ private: void OnWriteDone(bool ok) { TWriteCallback okCallback; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Y_VERIFY(WriteActive, "Unexpected Write done callback"); Y_VERIFY(!WriteFinished, "Unexpected WriteFinished flag"); @@ -1107,7 +1107,7 @@ private: if (ReadFinished) { Stream->Finish(&Status, OnFinishedTag.Prepare()); } - } else if (!WriteQueue.empty()) { + } else if (!WriteQueue.empty()) { WriteCallback.swap(WriteQueue.front().Callback); Stream->Write(WriteQueue.front().Request, OnWriteDoneTag.Prepare()); WriteQueue.pop_front(); @@ -1127,14 +1127,14 @@ private: void OnFinished(bool ok) { TGrpcStatus status; - std::deque<TWriteItem> writesDropped; - std::vector<TReadCallback> finishedCallbacks; + std::deque<TWriteItem> writesDropped; + std::vector<TReadCallback> finishedCallbacks; TConnectedCallback connectedCallback; TReadCallback readCallback; TReadCallback finishCallback; - { - std::unique_lock<std::mutex> guard(Mutex); + { + std::unique_lock<std::mutex> guard(Mutex); Finished = true; FinishedOk = ok; LocalContext.reset(); @@ -1211,15 +1211,15 @@ private: TFixedEvent OnFinishedTag = { this, &TSelf::OnFinished }; private: - std::mutex Mutex; + std::mutex Mutex; TAsyncReaderWriterPtr Stream; TConnectedCallback ConnectedCallback; TReadCallback ReadCallback; TReadCallback FinishCallback; - std::vector<TReadCallback> FinishedCallbacks; - std::deque<TWriteItem> WriteQueue; + std::vector<TReadCallback> FinishedCallbacks; + std::deque<TWriteItem> WriteQueue; TWriteCallback WriteCallback; - std::unordered_multimap<TString, TString>* InitialMetadata = nullptr; + std::unordered_multimap<TString, TString>* InitialMetadata = nullptr; bool Started = false; bool HasInitialMetadata = false; bool ReadActive = false; @@ -1231,30 +1231,30 @@ private: bool FinishedOk = false; }; -class TGRpcClientLow; - -template<typename TGRpcService> -class TServiceConnection { - using TStub = typename TGRpcService::Stub; - friend class TGRpcClientLow; - -public: - /* - * Start simple request - */ - template<typename TRequest, typename TResponse> - void DoRequest(const TRequest& request, +class TGRpcClientLow; + +template<typename TGRpcService> +class TServiceConnection { + using TStub = typename TGRpcService::Stub; + friend class TGRpcClientLow; + +public: + /* + * Start simple request + */ + template<typename TRequest, typename TResponse> + void DoRequest(const TRequest& request, TResponseCallback<TResponse> callback, typename TSimpleRequestProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, + const TCallMeta& metas = { }, IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive<TSimpleRequestProcessor<TStub, TRequest, TResponse>>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); - } - - /* + { + auto processor = MakeIntrusive<TSimpleRequestProcessor<TStub, TRequest, TResponse>>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, asyncRequest, request, provider ? provider : Provider_); + } + + /* * Start simple request */ template<typename TRequest, typename TResponse> @@ -1270,58 +1270,58 @@ public: } /* - * Start bidirectional streamming - */ - template<typename TRequest, typename TResponse> - void DoStreamRequest(TStreamConnectedCallback<TRequest, TResponse> callback, - typename TStreamRequestReadWriteProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, - const TCallMeta& metas = { }, - IQueueClientContextProvider* provider = nullptr) - { - auto processor = MakeIntrusive<TStreamRequestReadWriteProcessor<TStub, TRequest, TResponse>>(std::move(callback)); - processor->ApplyMeta(metas); - processor->Start(*Stub_, std::move(asyncRequest), provider ? provider : Provider_); - } - - /* - * Start streaming response reading (one request, many responses) - */ + * Start bidirectional streamming + */ + template<typename TRequest, typename TResponse> + void DoStreamRequest(TStreamConnectedCallback<TRequest, TResponse> callback, + typename TStreamRequestReadWriteProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, + const TCallMeta& metas = { }, + IQueueClientContextProvider* provider = nullptr) + { + auto processor = MakeIntrusive<TStreamRequestReadWriteProcessor<TStub, TRequest, TResponse>>(std::move(callback)); + processor->ApplyMeta(metas); + processor->Start(*Stub_, std::move(asyncRequest), provider ? provider : Provider_); + } + + /* + * Start streaming response reading (one request, many responses) + */ template<typename TRequest, typename TResponse> - void DoStreamRequest(const TRequest& request, - TStreamReaderCallback<TResponse> callback, - typename TStreamRequestReadProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, + void DoStreamRequest(const TRequest& request, + TStreamReaderCallback<TResponse> callback, + typename TStreamRequestReadProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, const TCallMeta& metas = { }, IQueueClientContextProvider* provider = nullptr) { - auto processor = MakeIntrusive<TStreamRequestReadProcessor<TStub, TRequest, TResponse>>(std::move(callback)); + auto processor = MakeIntrusive<TStreamRequestReadProcessor<TStub, TRequest, TResponse>>(std::move(callback)); processor->ApplyMeta(metas); - processor->Start(*Stub_, request, std::move(asyncRequest), provider ? provider : Provider_); + processor->Start(*Stub_, request, std::move(asyncRequest), provider ? provider : Provider_); } -private: - TServiceConnection(std::shared_ptr<grpc::ChannelInterface> ci, +private: + TServiceConnection(std::shared_ptr<grpc::ChannelInterface> ci, + IQueueClientContextProvider* provider) + : Stub_(TGRpcService::NewStub(ci)) + , Provider_(provider) + { + Y_VERIFY(Provider_, "Connection does not have a queue provider"); + } + + TServiceConnection(TStubsHolder& holder, IQueueClientContextProvider* provider) - : Stub_(TGRpcService::NewStub(ci)) + : Stub_(holder.GetOrCreateStub<TStub>()) , Provider_(provider) { Y_VERIFY(Provider_, "Connection does not have a queue provider"); } - TServiceConnection(TStubsHolder& holder, - IQueueClientContextProvider* provider) - : Stub_(holder.GetOrCreateStub<TStub>()) - , Provider_(provider) - { - Y_VERIFY(Provider_, "Connection does not have a queue provider"); - } - - std::shared_ptr<TStub> Stub_; + std::shared_ptr<TStub> Stub_; IQueueClientContextProvider* Provider_; -}; - -class TGRpcClientLow +}; + +class TGRpcClientLow : public IQueueClientContextProvider -{ +{ class TContextImpl; friend class TContextImpl; @@ -1331,10 +1331,10 @@ class TGRpcClientLow STOP_EXPLICIT = 2, }; -public: +public: explicit TGRpcClientLow(size_t numWorkerThread = DEFAULT_NUM_THREADS, bool useCompletionQueuePerThread = false); - ~TGRpcClientLow(); - + ~TGRpcClientLow(); + // Tries to stop all currently running requests (via their stop callbacks) // Will shutdown CQ and drain events once all requests have finished // No new requests may be started after this call @@ -1357,24 +1357,24 @@ public: IQueueClientContextPtr CreateContext() override; - template<typename TGRpcService> - std::unique_ptr<TServiceConnection<TGRpcService>> CreateGRpcServiceConnection(const TGRpcClientConfig& config) { - return std::unique_ptr<TServiceConnection<TGRpcService>>(new TServiceConnection<TGRpcService>(CreateChannelInterface(config), this)); - } - - template<typename TGRpcService> - std::unique_ptr<TServiceConnection<TGRpcService>> CreateGRpcServiceConnection(TStubsHolder& holder) { - return std::unique_ptr<TServiceConnection<TGRpcService>>(new TServiceConnection<TGRpcService>(holder, this)); - } - + template<typename TGRpcService> + std::unique_ptr<TServiceConnection<TGRpcService>> CreateGRpcServiceConnection(const TGRpcClientConfig& config) { + return std::unique_ptr<TServiceConnection<TGRpcService>>(new TServiceConnection<TGRpcService>(CreateChannelInterface(config), this)); + } + + template<typename TGRpcService> + std::unique_ptr<TServiceConnection<TGRpcService>> CreateGRpcServiceConnection(TStubsHolder& holder) { + return std::unique_ptr<TServiceConnection<TGRpcService>>(new TServiceConnection<TGRpcService>(holder, this)); + } + // Tests only, not thread-safe void AddWorkerThreadForTest(); -private: +private: using IThreadRef = std::unique_ptr<IThreadFactory::IThread>; using CompletionQueueRef = std::unique_ptr<grpc::CompletionQueue>; - void Init(size_t numWorkerThread); - + void Init(size_t numWorkerThread); + inline ECqState GetCqState() const { return (ECqState) AtomicGet(CqState_); } inline void SetCqState(ECqState state) { AtomicSet(CqState_, state); } @@ -1385,15 +1385,15 @@ private: private: bool UseCompletionQueuePerThread_; - std::vector<CompletionQueueRef> CQS_; - std::vector<IThreadRef> WorkerThreads_; + std::vector<CompletionQueueRef> CQS_; + std::vector<IThreadRef> WorkerThreads_; TAtomic CqState_ = -1; - std::mutex Mtx_; - std::condition_variable ContextsEmpty_; - std::unordered_set<TContextImpl*> Contexts_; + std::mutex Mtx_; + std::condition_variable ContextsEmpty_; + std::unordered_set<TContextImpl*> Contexts_; + + std::mutex JoinMutex_; +}; - std::mutex JoinMutex_; -}; - } // namespace NGRpc diff --git a/library/cpp/grpc/client/grpc_common.h b/library/cpp/grpc/client/grpc_common.h index ac62e8b331..ffcdafe045 100644 --- a/library/cpp/grpc/client/grpc_common.h +++ b/library/cpp/grpc/client/grpc_common.h @@ -1,53 +1,53 @@ -#pragma once - -#include <grpc++/grpc++.h> +#pragma once + +#include <grpc++/grpc++.h> #include <grpc++/resource_quota.h> - + #include <util/datetime/base.h> -#include <unordered_map> -#include <util/generic/string.h> - -constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; - +#include <unordered_map> +#include <util/generic/string.h> + +constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; + namespace NGrpc { - -struct TGRpcClientConfig { - TString Locator; // format host:port - TDuration Timeout = TDuration::Max(); // request timeout - ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size + +struct TGRpcClientConfig { + TString Locator; // format host:port + TDuration Timeout = TDuration::Max(); // request timeout + ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests - ui32 MaxInFlight = 0; + ui32 MaxInFlight = 0; bool EnableSsl = false; TString SslCaCert; //Implicitly enables Ssl if not empty grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; ui64 MemQuota = 0; - std::unordered_map<TString, TString> StringChannelParams; - std::unordered_map<TString, int> IntChannelParams; + std::unordered_map<TString, TString> StringChannelParams; + std::unordered_map<TString, int> IntChannelParams; TString LoadBalancingPolicy = { }; TString SslTargetNameOverride = { }; - - TGRpcClientConfig() = default; - TGRpcClientConfig(const TGRpcClientConfig&) = default; - TGRpcClientConfig(TGRpcClientConfig&&) = default; - TGRpcClientConfig& operator=(const TGRpcClientConfig&) = default; - TGRpcClientConfig& operator=(TGRpcClientConfig&&) = default; - - TGRpcClientConfig(const TString& locator, TDuration timeout = TDuration::Max(), + + TGRpcClientConfig() = default; + TGRpcClientConfig(const TGRpcClientConfig&) = default; + TGRpcClientConfig(TGRpcClientConfig&&) = default; + TGRpcClientConfig& operator=(const TGRpcClientConfig&) = default; + TGRpcClientConfig& operator=(TGRpcClientConfig&&) = default; + + TGRpcClientConfig(const TString& locator, TDuration timeout = TDuration::Max(), ui64 maxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, TString caCert = "", grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) - : Locator(locator) - , Timeout(timeout) - , MaxMessageSize(maxMessageSize) - , MaxInFlight(maxInFlight) + : Locator(locator) + , Timeout(timeout) + , MaxMessageSize(maxMessageSize) + , MaxInFlight(maxInFlight) , EnableSsl(enableSsl) - , SslCaCert(caCert) + , SslCaCert(caCert) , CompressionAlgoritm(compressionAlgorithm) - {} -}; - -inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRpcClientConfig& config, grpc_socket_mutator* mutator = nullptr){ - grpc::ChannelArguments args; + {} +}; + +inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRpcClientConfig& config, grpc_socket_mutator* mutator = nullptr){ + grpc::ChannelArguments args; args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); args.SetCompressionAlgorithm(config.CompressionAlgoritm); @@ -65,9 +65,9 @@ inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRp quota.Resize(config.MemQuota); args.SetResourceQuota(quota); } - if (mutator) { - args.SetSocketMutator(mutator); - } + if (mutator) { + args.SetSocketMutator(mutator); + } if (!config.LoadBalancingPolicy.empty()) { args.SetLoadBalancingPolicyName(config.LoadBalancingPolicy); } @@ -75,10 +75,10 @@ inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRp args.SetSslTargetNameOverride(config.SslTargetNameOverride); } if (config.EnableSsl || config.SslCaCert) { - return grpc::CreateCustomChannel(config.Locator, grpc::SslCredentials(grpc::SslCredentialsOptions{config.SslCaCert, "", ""}), args); - } else { - return grpc::CreateCustomChannel(config.Locator, grpc::InsecureChannelCredentials(), args); - } -} - + return grpc::CreateCustomChannel(config.Locator, grpc::SslCredentials(grpc::SslCredentialsOptions{config.SslCaCert, "", ""}), args); + } else { + return grpc::CreateCustomChannel(config.Locator, grpc::InsecureChannelCredentials(), args); + } +} + } // namespace NGRpc diff --git a/library/cpp/grpc/client/ya.make b/library/cpp/grpc/client/ya.make index 11f36aa94f..a4e74b067c 100644 --- a/library/cpp/grpc/client/ya.make +++ b/library/cpp/grpc/client/ya.make @@ -6,11 +6,11 @@ OWNER( ) SRCS( - grpc_client_low.cpp + grpc_client_low.cpp ) PEERDIR( - contrib/libs/grpc + contrib/libs/grpc ) END() diff --git a/library/cpp/grpc/server/event_callback.cpp b/library/cpp/grpc/server/event_callback.cpp index 559a472807..f423836bd6 100644 --- a/library/cpp/grpc/server/event_callback.cpp +++ b/library/cpp/grpc/server/event_callback.cpp @@ -1 +1 @@ -#include "event_callback.h" +#include "event_callback.h" diff --git a/library/cpp/grpc/server/event_callback.h b/library/cpp/grpc/server/event_callback.h index c0e16ee504..d0b700b3c9 100644 --- a/library/cpp/grpc/server/event_callback.h +++ b/library/cpp/grpc/server/event_callback.h @@ -1,80 +1,80 @@ -#pragma once - -#include "grpc_server.h" - +#pragma once + +#include "grpc_server.h" + namespace NGrpc { - -enum class EQueueEventStatus { - OK, - ERROR -}; - -template<class TCallback> + +enum class EQueueEventStatus { + OK, + ERROR +}; + +template<class TCallback> class TQueueEventCallback: public IQueueEvent { -public: - TQueueEventCallback(const TCallback& callback) - : Callback(callback) - {} - - TQueueEventCallback(TCallback&& callback) - : Callback(std::move(callback)) - {} - - bool Execute(bool ok) override { - Callback(ok ? EQueueEventStatus::OK : EQueueEventStatus::ERROR); - return false; - } - - void DestroyRequest() override { - delete this; - } - -private: - TCallback Callback; -}; - +public: + TQueueEventCallback(const TCallback& callback) + : Callback(callback) + {} + + TQueueEventCallback(TCallback&& callback) + : Callback(std::move(callback)) + {} + + bool Execute(bool ok) override { + Callback(ok ? EQueueEventStatus::OK : EQueueEventStatus::ERROR); + return false; + } + + void DestroyRequest() override { + delete this; + } + +private: + TCallback Callback; +}; + // Implementation of IQueueEvent that reduces allocations -template<class TSelf> +template<class TSelf> class TQueueFixedEvent: private IQueueEvent { - using TCallback = void (TSelf::*)(EQueueEventStatus); - -public: - TQueueFixedEvent(TSelf* self, TCallback callback) - : Self(self) - , Callback(callback) - { } - + using TCallback = void (TSelf::*)(EQueueEventStatus); + +public: + TQueueFixedEvent(TSelf* self, TCallback callback) + : Self(self) + , Callback(callback) + { } + IQueueEvent* Prepare() { - Self->Ref(); - return this; - } - -private: - bool Execute(bool ok) override { - ((*Self).*Callback)(ok ? EQueueEventStatus::OK : EQueueEventStatus::ERROR); - return false; - } - - void DestroyRequest() override { - Self->UnRef(); - } - -private: - TSelf* const Self; - TCallback const Callback; -}; - -template<class TCallback> + Self->Ref(); + return this; + } + +private: + bool Execute(bool ok) override { + ((*Self).*Callback)(ok ? EQueueEventStatus::OK : EQueueEventStatus::ERROR); + return false; + } + + void DestroyRequest() override { + Self->UnRef(); + } + +private: + TSelf* const Self; + TCallback const Callback; +}; + +template<class TCallback> inline IQueueEvent* MakeQueueEventCallback(TCallback&& callback) { - return new TQueueEventCallback<TCallback>(std::forward<TCallback>(callback)); -} - -template<class T> + return new TQueueEventCallback<TCallback>(std::forward<TCallback>(callback)); +} + +template<class T> inline IQueueEvent* MakeQueueEventCallback(T* self, void (T::*method)(EQueueEventStatus)) { - using TPtr = TIntrusivePtr<T>; - return MakeQueueEventCallback([self = TPtr(self), method] (EQueueEventStatus status) { - ((*self).*method)(status); - }); -} - + using TPtr = TIntrusivePtr<T>; + return MakeQueueEventCallback([self = TPtr(self), method] (EQueueEventStatus status) { + ((*self).*method)(status); + }); +} + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_async_ctx_base.h b/library/cpp/grpc/server/grpc_async_ctx_base.h index d170e2ef34..51356d4ce5 100644 --- a/library/cpp/grpc/server/grpc_async_ctx_base.h +++ b/library/cpp/grpc/server/grpc_async_ctx_base.h @@ -1,48 +1,48 @@ -#pragma once - -#include "grpc_server.h" - -#include <util/generic/vector.h> -#include <util/generic/string.h> -#include <util/system/yassert.h> +#pragma once + +#include "grpc_server.h" + +#include <util/generic/vector.h> +#include <util/generic/string.h> +#include <util/system/yassert.h> #include <util/generic/set.h> - -#include <grpc++/server.h> -#include <grpc++/server_context.h> - -#include <chrono> - + +#include <grpc++/server.h> +#include <grpc++/server_context.h> + +#include <chrono> + namespace NGrpc { - -template<typename TService> + +template<typename TService> class TBaseAsyncContext: public ICancelableContext { -public: - TBaseAsyncContext(typename TService::TCurrentGRpcService::AsyncService* service, grpc::ServerCompletionQueue* cq) - : Service(service) - , CQ(cq) - { - } - - TString GetPeerName() const { - return TString(Context.peer()); - } - - TInstant Deadline() const { - // The timeout transferred in "grpc-timeout" header [1] and calculated from the deadline +public: + TBaseAsyncContext(typename TService::TCurrentGRpcService::AsyncService* service, grpc::ServerCompletionQueue* cq) + : Service(service) + , CQ(cq) + { + } + + TString GetPeerName() const { + return TString(Context.peer()); + } + + TInstant Deadline() const { + // The timeout transferred in "grpc-timeout" header [1] and calculated from the deadline // right before the request is getting to be send. // 1. https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md // - // After this timeout calculated back to the deadline on the server side - // using server grpc GPR_CLOCK_MONOTONIC time (raw_deadline() method). - // deadline() method convert this to epoch related deadline GPR_CLOCK_REALTIME - // - - std::chrono::system_clock::time_point t = Context.deadline(); - if (t == std::chrono::system_clock::time_point::max()) { - return TInstant::Max(); + // After this timeout calculated back to the deadline on the server side + // using server grpc GPR_CLOCK_MONOTONIC time (raw_deadline() method). + // deadline() method convert this to epoch related deadline GPR_CLOCK_REALTIME + // + + std::chrono::system_clock::time_point t = Context.deadline(); + if (t == std::chrono::system_clock::time_point::max()) { + return TInstant::Max(); } - auto us = std::chrono::time_point_cast<std::chrono::microseconds>(t); - return TInstant::MicroSeconds(us.time_since_epoch().count()); + auto us = std::chrono::time_point_cast<std::chrono::microseconds>(t); + return TInstant::MicroSeconds(us.time_since_epoch().count()); } TSet<TStringBuf> GetPeerMetaKeys() const { @@ -54,7 +54,7 @@ public: } TVector<TStringBuf> GetPeerMetaValues(TStringBuf key) const { - const auto& clientMetadata = Context.client_metadata(); + const auto& clientMetadata = Context.client_metadata(); const auto range = clientMetadata.equal_range(grpc::string_ref{key.data(), key.size()}); if (range.first == range.second) { return {}; @@ -63,32 +63,32 @@ public: TVector<TStringBuf> values; values.reserve(std::distance(range.first, range.second)); - for (auto it = range.first; it != range.second; ++it) { + for (auto it = range.first; it != range.second; ++it) { values.emplace_back(it->second.data(), it->second.size()); - } + } return values; - } - + } + grpc_compression_level GetCompressionLevel() const { return Context.compression_level(); } - void Shutdown() override { - // Shutdown may only be called after request has started successfully - if (Context.c_call()) - Context.TryCancel(); - } - -protected: - //! The means of communication with the gRPC runtime for an asynchronous - //! server. - typename TService::TCurrentGRpcService::AsyncService* const Service; - //! The producer-consumer queue where for asynchronous server notifications. - grpc::ServerCompletionQueue* const CQ; - //! Context for the rpc, allowing to tweak aspects of it such as the use - //! of compression, authentication, as well as to send metadata back to the - //! client. - grpc::ServerContext Context; -}; - + void Shutdown() override { + // Shutdown may only be called after request has started successfully + if (Context.c_call()) + Context.TryCancel(); + } + +protected: + //! The means of communication with the gRPC runtime for an asynchronous + //! server. + typename TService::TCurrentGRpcService::AsyncService* const Service; + //! The producer-consumer queue where for asynchronous server notifications. + grpc::ServerCompletionQueue* const CQ; + //! Context for the rpc, allowing to tweak aspects of it such as the use + //! of compression, authentication, as well as to send metadata back to the + //! client. + grpc::ServerContext Context; +}; + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_counters.cpp b/library/cpp/grpc/server/grpc_counters.cpp index f3a0b2b5e9..fa96e0100b 100644 --- a/library/cpp/grpc/server/grpc_counters.cpp +++ b/library/cpp/grpc/server/grpc_counters.cpp @@ -1,4 +1,4 @@ -#include "grpc_counters.h" +#include "grpc_counters.h" namespace NGrpc { namespace { diff --git a/library/cpp/grpc/server/grpc_counters.h b/library/cpp/grpc/server/grpc_counters.h index 25b7c28369..0b6c36c84c 100644 --- a/library/cpp/grpc/server/grpc_counters.h +++ b/library/cpp/grpc/server/grpc_counters.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/dynamic_counters/percentile/percentile.h> #include <library/cpp/monlib/dynamic_counters/counters.h> -#include <util/generic/ptr.h> - +#include <util/generic/ptr.h> + namespace NGrpc { - + struct ICounterBlock : public TThrRefBase { virtual void CountNotOkRequest() = 0; virtual void CountNotOkResponse() = 0; @@ -15,9 +15,9 @@ struct ICounterBlock : public TThrRefBase { virtual void CountResponseBytes(ui32 responseSize) = 0; virtual void StartProcessing(ui32 requestSize) = 0; virtual void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, TDuration requestDuration) = 0; - virtual void CountRequestsWithoutDatabase() {} - virtual void CountRequestsWithoutToken() {} - virtual void CountRequestWithoutTls() {} + virtual void CountRequestsWithoutDatabase() {} + virtual void CountRequestsWithoutToken() {} + virtual void CountRequestWithoutTls() {} virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } @@ -26,62 +26,62 @@ struct ICounterBlock : public TThrRefBase { using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; class TCounterBlock final : public ICounterBlock { - NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; - NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; - NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter; - NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter; - NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes; - NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; - NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated; - NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted; + NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; + NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; + NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter; + NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter; + NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes; + NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes; + NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated; + NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted; bool Percentile = false; NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs; - std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters; - -public: - TCounterBlock(NMonitoring::TDynamicCounters::TCounterPtr totalCounter, - NMonitoring::TDynamicCounters::TCounterPtr inflyCounter, - NMonitoring::TDynamicCounters::TCounterPtr notOkRequestCounter, - NMonitoring::TDynamicCounters::TCounterPtr notOkResponseCounter, - NMonitoring::TDynamicCounters::TCounterPtr requestBytes, - NMonitoring::TDynamicCounters::TCounterPtr inflyRequestBytes, + std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters; + +public: + TCounterBlock(NMonitoring::TDynamicCounters::TCounterPtr totalCounter, + NMonitoring::TDynamicCounters::TCounterPtr inflyCounter, + NMonitoring::TDynamicCounters::TCounterPtr notOkRequestCounter, + NMonitoring::TDynamicCounters::TCounterPtr notOkResponseCounter, + NMonitoring::TDynamicCounters::TCounterPtr requestBytes, + NMonitoring::TDynamicCounters::TCounterPtr inflyRequestBytes, NMonitoring::TDynamicCounters::TCounterPtr responseBytes, - NMonitoring::TDynamicCounters::TCounterPtr notAuthenticated, - NMonitoring::TDynamicCounters::TCounterPtr resourceExhausted, + NMonitoring::TDynamicCounters::TCounterPtr notAuthenticated, + NMonitoring::TDynamicCounters::TCounterPtr resourceExhausted, TIntrusivePtr<NMonitoring::TDynamicCounters> group) - : TotalCounter(std::move(totalCounter)) - , InflyCounter(std::move(inflyCounter)) - , NotOkRequestCounter(std::move(notOkRequestCounter)) - , NotOkResponseCounter(std::move(notOkResponseCounter)) - , RequestBytes(std::move(requestBytes)) - , InflyRequestBytes(std::move(inflyRequestBytes)) - , ResponseBytes(std::move(responseBytes)) - , NotAuthenticated(std::move(notAuthenticated)) - , ResourceExhausted(std::move(resourceExhausted)) + : TotalCounter(std::move(totalCounter)) + , InflyCounter(std::move(inflyCounter)) + , NotOkRequestCounter(std::move(notOkRequestCounter)) + , NotOkResponseCounter(std::move(notOkResponseCounter)) + , RequestBytes(std::move(requestBytes)) + , InflyRequestBytes(std::move(inflyRequestBytes)) + , ResponseBytes(std::move(responseBytes)) + , NotAuthenticated(std::move(notAuthenticated)) + , ResourceExhausted(std::move(resourceExhausted)) { if (group) { RequestHistMs.Initialize(group, "event", "request", "ms", {0.5f, 0.9f, 0.99f, 0.999f, 1.0f}); Percentile = true; } } - + void CountNotOkRequest() override { - NotOkRequestCounter->Inc(); - } - + NotOkRequestCounter->Inc(); + } + void CountNotOkResponse() override { - NotOkResponseCounter->Inc(); - } - + NotOkResponseCounter->Inc(); + } + void CountNotAuthenticated() override { - NotAuthenticated->Inc(); - } - + NotAuthenticated->Inc(); + } + void CountResourceExhausted() override { - ResourceExhausted->Inc(); - } - + ResourceExhausted->Inc(); + } + void CountRequestBytes(ui32 requestSize) override { *RequestBytes += requestSize; } @@ -91,27 +91,27 @@ public: } void StartProcessing(ui32 requestSize) override { - TotalCounter->Inc(); - InflyCounter->Inc(); - *RequestBytes += requestSize; - *InflyRequestBytes += requestSize; - } - + TotalCounter->Inc(); + InflyCounter->Inc(); + *RequestBytes += requestSize; + *InflyRequestBytes += requestSize; + } + void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, TDuration requestDuration) override { Y_UNUSED(status); - InflyCounter->Dec(); - *InflyRequestBytes -= requestSize; - *ResponseBytes += responseSize; - if (!ok) { - NotOkResponseCounter->Inc(); - } + InflyCounter->Dec(); + *InflyRequestBytes -= requestSize; + *ResponseBytes += responseSize; + if (!ok) { + NotOkResponseCounter->Inc(); + } if (Percentile) { RequestHistMs.Increment(requestDuration.MilliSeconds()); } - } + } ICounterBlockPtr Clone() override { return this; @@ -122,10 +122,10 @@ public: RequestHistMs.Update(); } } -}; - -using TCounterBlockPtr = TIntrusivePtr<TCounterBlock>; - +}; + +using TCounterBlockPtr = TIntrusivePtr<TCounterBlock>; + /** * Creates new instance of ICounterBlock implementation which does nothing. * diff --git a/library/cpp/grpc/server/grpc_request.cpp b/library/cpp/grpc/server/grpc_request.cpp index 33264fe6f2..d18a32776f 100644 --- a/library/cpp/grpc/server/grpc_request.cpp +++ b/library/cpp/grpc/server/grpc_request.cpp @@ -1,59 +1,59 @@ -#include "grpc_request.h" - +#include "grpc_request.h" + namespace NGrpc { - -const char* GRPC_USER_AGENT_HEADER = "user-agent"; - + +const char* GRPC_USER_AGENT_HEADER = "user-agent"; + class TStreamAdaptor: public IStreamAdaptor { -public: - TStreamAdaptor() - : StreamIsReady_(true) - {} - - void Enqueue(std::function<void()>&& fn, bool urgent) override { - with_lock(Mtx_) { - if (!UrgentQueue_.empty() || !NormalQueue_.empty()) { - Y_VERIFY(!StreamIsReady_); - } - auto& queue = urgent ? UrgentQueue_ : NormalQueue_; - if (StreamIsReady_ && queue.empty()) { - StreamIsReady_ = false; - } else { - queue.push_back(std::move(fn)); - return; - } - } - fn(); - } - - size_t ProcessNext() override { - size_t left = 0; - std::function<void()> fn; - with_lock(Mtx_) { - Y_VERIFY(!StreamIsReady_); - auto& queue = UrgentQueue_.empty() ? NormalQueue_ : UrgentQueue_; - if (queue.empty()) { - // Both queues are empty - StreamIsReady_ = true; - } else { - fn = std::move(queue.front()); - queue.pop_front(); - left = UrgentQueue_.size() + NormalQueue_.size(); - } - } - if (fn) - fn(); - return left; - } -private: - bool StreamIsReady_; - TList<std::function<void()>> NormalQueue_; - TList<std::function<void()>> UrgentQueue_; - TMutex Mtx_; -}; - -IStreamAdaptor::TPtr CreateStreamAdaptor() { - return std::make_unique<TStreamAdaptor>(); -} - +public: + TStreamAdaptor() + : StreamIsReady_(true) + {} + + void Enqueue(std::function<void()>&& fn, bool urgent) override { + with_lock(Mtx_) { + if (!UrgentQueue_.empty() || !NormalQueue_.empty()) { + Y_VERIFY(!StreamIsReady_); + } + auto& queue = urgent ? UrgentQueue_ : NormalQueue_; + if (StreamIsReady_ && queue.empty()) { + StreamIsReady_ = false; + } else { + queue.push_back(std::move(fn)); + return; + } + } + fn(); + } + + size_t ProcessNext() override { + size_t left = 0; + std::function<void()> fn; + with_lock(Mtx_) { + Y_VERIFY(!StreamIsReady_); + auto& queue = UrgentQueue_.empty() ? NormalQueue_ : UrgentQueue_; + if (queue.empty()) { + // Both queues are empty + StreamIsReady_ = true; + } else { + fn = std::move(queue.front()); + queue.pop_front(); + left = UrgentQueue_.size() + NormalQueue_.size(); + } + } + if (fn) + fn(); + return left; + } +private: + bool StreamIsReady_; + TList<std::function<void()>> NormalQueue_; + TList<std::function<void()>> UrgentQueue_; + TMutex Mtx_; +}; + +IStreamAdaptor::TPtr CreateStreamAdaptor() { + return std::make_unique<TStreamAdaptor>(); +} + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h index 2841069d62..5bd8d3902b 100644 --- a/library/cpp/grpc/server/grpc_request.h +++ b/library/cpp/grpc/server/grpc_request.h @@ -1,118 +1,118 @@ -#pragma once - +#pragma once + #include <google/protobuf/text_format.h> #include <google/protobuf/arena.h> #include <google/protobuf/message.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> #include <library/cpp/logger/priority.h> - + #include "grpc_response.h" -#include "event_callback.h" +#include "event_callback.h" #include "grpc_async_ctx_base.h" #include "grpc_counters.h" -#include "grpc_request_base.h" -#include "grpc_server.h" +#include "grpc_request_base.h" +#include "grpc_server.h" #include "logger.h" - + #include <util/system/hp_timer.h> -#include <grpc++/server.h> -#include <grpc++/server_context.h> +#include <grpc++/server.h> +#include <grpc++/server_context.h> #include <grpc++/support/async_stream.h> #include <grpc++/support/async_unary_call.h> -#include <grpc++/support/byte_buffer.h> -#include <grpc++/impl/codegen/async_stream.h> - +#include <grpc++/support/byte_buffer.h> +#include <grpc++/impl/codegen/async_stream.h> + namespace NGrpc { - -class IStreamAdaptor { -public: - using TPtr = std::unique_ptr<IStreamAdaptor>; - virtual void Enqueue(std::function<void()>&& fn, bool urgent) = 0; - virtual size_t ProcessNext() = 0; - virtual ~IStreamAdaptor() = default; -}; - -IStreamAdaptor::TPtr CreateStreamAdaptor(); - -/////////////////////////////////////////////////////////////////////////////// + +class IStreamAdaptor { +public: + using TPtr = std::unique_ptr<IStreamAdaptor>; + virtual void Enqueue(std::function<void()>&& fn, bool urgent) = 0; + virtual size_t ProcessNext() = 0; + virtual ~IStreamAdaptor() = default; +}; + +IStreamAdaptor::TPtr CreateStreamAdaptor(); + +/////////////////////////////////////////////////////////////////////////////// template<typename TIn, typename TOut, typename TService, typename TInProtoPrinter, typename TOutProtoPrinter> class TGRpcRequestImpl - : public TBaseAsyncContext<TService> - , public IQueueEvent - , public IRequestContextBase -{ + : public TBaseAsyncContext<TService> + , public IQueueEvent + , public IRequestContextBase +{ using TThis = TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter>; -public: - using TOnRequest = std::function<void (IRequestContextBase* ctx)>; - using TRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, - grpc::ServerAsyncResponseWriter<TOut>*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); - using TStreamRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, - grpc::ServerAsyncWriter<TOut>*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); - +public: + using TOnRequest = std::function<void (IRequestContextBase* ctx)>; + using TRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, + grpc::ServerAsyncResponseWriter<TOut>*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); + using TStreamRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, + grpc::ServerAsyncWriter<TOut>*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); + TGRpcRequestImpl(TService* server, - typename TService::TCurrentGRpcService::AsyncService* service, - grpc::ServerCompletionQueue* cq, - TOnRequest cb, - TRequestCallback requestCallback, + typename TService::TCurrentGRpcService::AsyncService* service, + grpc::ServerCompletionQueue* cq, + TOnRequest cb, + TRequestCallback requestCallback, const char* name, TLoggerPtr logger, ICounterBlockPtr counters, IGRpcRequestLimiterPtr limiter) - : TBaseAsyncContext<TService>(service, cq) - , Server_(server) - , Cb_(cb) - , RequestCallback_(requestCallback) - , StreamRequestCallback_(nullptr) - , Name_(name) + : TBaseAsyncContext<TService>(service, cq) + , Server_(server) + , Cb_(cb) + , RequestCallback_(requestCallback) + , StreamRequestCallback_(nullptr) + , Name_(name) , Logger_(std::move(logger)) - , Counters_(std::move(counters)) + , Counters_(std::move(counters)) , RequestLimiter_(std::move(limiter)) , Writer_(new grpc::ServerAsyncResponseWriter<TUniversalResponseRef<TOut>>(&this->Context)) , StateFunc_(&TThis::SetRequestDone) - { - AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); - Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); - Y_VERIFY(Request_); + { + AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); + Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); + Y_VERIFY(Request_); GRPC_LOG_DEBUG(Logger_, "[%p] created request Name# %s", this, Name_); - FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); - } - + FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); + } + TGRpcRequestImpl(TService* server, - typename TService::TCurrentGRpcService::AsyncService* service, - grpc::ServerCompletionQueue* cq, - TOnRequest cb, - TStreamRequestCallback requestCallback, + typename TService::TCurrentGRpcService::AsyncService* service, + grpc::ServerCompletionQueue* cq, + TOnRequest cb, + TStreamRequestCallback requestCallback, const char* name, TLoggerPtr logger, ICounterBlockPtr counters, IGRpcRequestLimiterPtr limiter) - : TBaseAsyncContext<TService>(service, cq) - , Server_(server) - , Cb_(cb) - , RequestCallback_(nullptr) - , StreamRequestCallback_(requestCallback) - , Name_(name) + : TBaseAsyncContext<TService>(service, cq) + , Server_(server) + , Cb_(cb) + , RequestCallback_(nullptr) + , StreamRequestCallback_(requestCallback) + , Name_(name) , Logger_(std::move(logger)) - , Counters_(std::move(counters)) + , Counters_(std::move(counters)) , RequestLimiter_(std::move(limiter)) - , StreamWriter_(new grpc::ServerAsyncWriter<TUniversalResponse<TOut>>(&this->Context)) + , StreamWriter_(new grpc::ServerAsyncWriter<TUniversalResponse<TOut>>(&this->Context)) , StateFunc_(&TThis::SetRequestDone) - { - AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); - Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); - Y_VERIFY(Request_); + { + AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); + Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); + Y_VERIFY(Request_); GRPC_LOG_DEBUG(Logger_, "[%p] created streaming request Name# %s", this, Name_); - FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); - StreamAdaptor_ = CreateStreamAdaptor(); - } - - TAsyncFinishResult GetFinishFuture() override { - return FinishPromise_.GetFuture(); - } - + FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); + StreamAdaptor_ = CreateStreamAdaptor(); + } + + TAsyncFinishResult GetFinishFuture() override { + return FinishPromise_.GetFuture(); + } + TString GetPeer() const override { return TString(this->Context.peer()); } @@ -121,7 +121,7 @@ public: return Server_->SslServer(); } - void Run() { + void Run() { // Start request unless server is shutting down if (auto guard = Server_->ProtectShutdown()) { Ref(); //For grpc c runtime @@ -135,28 +135,28 @@ public: (&this->Context, Request_, reinterpret_cast<grpc::ServerAsyncWriter<TOut>*>(StreamWriter_.Get()), this->CQ, this->CQ, GetGRpcTag()); } - } - } - + } + } + ~TGRpcRequestImpl() { - // No direct dtor call allowed - Y_ASSERT(RefCount() == 0); - } - - bool Execute(bool ok) override { - return (this->*StateFunc_)(ok); - } - - void DestroyRequest() override { + // No direct dtor call allowed + Y_ASSERT(RefCount() == 0); + } + + bool Execute(bool ok) override { + return (this->*StateFunc_)(ok); + } + + void DestroyRequest() override { if (RequestRegistered_) { Server_->DeregisterRequestCtx(this); RequestRegistered_ = false; } - UnRef(); - } - - TInstant Deadline() const override { - return TBaseAsyncContext<TService>::Deadline(); + UnRef(); + } + + TInstant Deadline() const override { + return TBaseAsyncContext<TService>::Deadline(); } TSet<TStringBuf> GetPeerMetaKeys() const override { @@ -164,299 +164,299 @@ public: } TVector<TStringBuf> GetPeerMetaValues(TStringBuf key) const override { - return TBaseAsyncContext<TService>::GetPeerMetaValues(key); - } - + return TBaseAsyncContext<TService>::GetPeerMetaValues(key); + } + grpc_compression_level GetCompressionLevel() const override { return TBaseAsyncContext<TService>::GetCompressionLevel(); } - //! Get pointer to the request's message. - const NProtoBuf::Message* GetRequest() const override { - return Request_; - } - - TAuthState& GetAuthState() override { - return AuthState_; - } - + //! Get pointer to the request's message. + const NProtoBuf::Message* GetRequest() const override { + return Request_; + } + + TAuthState& GetAuthState() override { + return AuthState_; + } + void Reply(NProtoBuf::Message* resp, ui32 status) override { ResponseStatus = status; - WriteDataOk(resp); - } - + WriteDataOk(resp); + } + void Reply(grpc::ByteBuffer* resp, ui32 status) override { ResponseStatus = status; - WriteByteDataOk(resp); - } - + WriteByteDataOk(resp); + } + void ReplyError(grpc::StatusCode code, const TString& msg) override { - FinishGrpcStatus(code, msg, false); - } - - void ReplyUnauthenticated(const TString& in) override { - const TString message = in.empty() ? TString("unauthenticated") : TString("unauthenticated, ") + in; - FinishGrpcStatus(grpc::StatusCode::UNAUTHENTICATED, message, false); - } - - void SetNextReplyCallback(TOnNextReply&& cb) override { - NextReplyCb_ = cb; - } - + FinishGrpcStatus(code, msg, false); + } + + void ReplyUnauthenticated(const TString& in) override { + const TString message = in.empty() ? TString("unauthenticated") : TString("unauthenticated, ") + in; + FinishGrpcStatus(grpc::StatusCode::UNAUTHENTICATED, message, false); + } + + void SetNextReplyCallback(TOnNextReply&& cb) override { + NextReplyCb_ = cb; + } + void AddTrailingMetadata(const TString& key, const TString& value) override { - this->Context.AddTrailingMetadata(key, value); - } - - void FinishStreamingOk() override { - GRPC_LOG_DEBUG(Logger_, "[%p] finished streaming Name# %s peer# %s (enqueued)", this, Name_, - this->Context.peer().c_str()); - auto cb = [this]() { - StateFunc_ = &TThis::SetFinishDone; - GRPC_LOG_DEBUG(Logger_, "[%p] finished streaming Name# %s peer# %s (pushed to grpc)", this, Name_, - this->Context.peer().c_str()); - - StreamWriter_->Finish(grpc::Status::OK, GetGRpcTag()); - }; - StreamAdaptor_->Enqueue(std::move(cb), false); - } - - google::protobuf::Arena* GetArena() override { - return &Arena_; - } - + this->Context.AddTrailingMetadata(key, value); + } + + void FinishStreamingOk() override { + GRPC_LOG_DEBUG(Logger_, "[%p] finished streaming Name# %s peer# %s (enqueued)", this, Name_, + this->Context.peer().c_str()); + auto cb = [this]() { + StateFunc_ = &TThis::SetFinishDone; + GRPC_LOG_DEBUG(Logger_, "[%p] finished streaming Name# %s peer# %s (pushed to grpc)", this, Name_, + this->Context.peer().c_str()); + + StreamWriter_->Finish(grpc::Status::OK, GetGRpcTag()); + }; + StreamAdaptor_->Enqueue(std::move(cb), false); + } + + google::protobuf::Arena* GetArena() override { + return &Arena_; + } + void UseDatabase(const TString& database) override { Counters_->UseDatabase(database); } -private: - void Clone() { - if (!Server_->IsShuttingDown()) { - if (RequestCallback_) { +private: + void Clone() { + if (!Server_->IsShuttingDown()) { + if (RequestCallback_) { MakeIntrusive<TThis>( Server_, this->Service, this->CQ, Cb_, RequestCallback_, Name_, Logger_, Counters_->Clone(), RequestLimiter_)->Run(); - } else { + } else { MakeIntrusive<TThis>( Server_, this->Service, this->CQ, Cb_, StreamRequestCallback_, Name_, Logger_, Counters_->Clone(), RequestLimiter_)->Run(); - } - } - } - - void WriteDataOk(NProtoBuf::Message* resp) { - auto makeResponseString = [&] { - TString x; + } + } + } + + void WriteDataOk(NProtoBuf::Message* resp) { + auto makeResponseString = [&] { + TString x; TOutProtoPrinter printer; - printer.SetSingleLineMode(true); - printer.PrintToString(*resp, &x); - return x; - }; - - auto sz = (size_t)resp->ByteSize(); - if (Writer_) { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s", this, Name_, - makeResponseString().data(), this->Context.peer().c_str()); + printer.SetSingleLineMode(true); + printer.PrintToString(*resp, &x); + return x; + }; + + auto sz = (size_t)resp->ByteSize(); + if (Writer_) { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s", this, Name_, + makeResponseString().data(), this->Context.peer().c_str()); StateFunc_ = &TThis::SetFinishDone; - ResponseSize = sz; - Y_VERIFY(this->Context.c_call()); + ResponseSize = sz; + Y_VERIFY(this->Context.c_call()); Writer_->Finish(TUniversalResponseRef<TOut>(resp), grpc::Status::OK, GetGRpcTag()); - } else { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s (enqueued)", - this, Name_, makeResponseString().data(), this->Context.peer().c_str()); + } else { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s (enqueued)", + this, Name_, makeResponseString().data(), this->Context.peer().c_str()); // because of std::function cannot hold move-only captured object // we allocate shared object on heap to avoid message copy auto uResp = MakeIntrusive<TUniversalResponse<TOut>>(resp); auto cb = [this, uResp = std::move(uResp), sz, &makeResponseString]() { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s (pushed to grpc)", - this, Name_, makeResponseString().data(), this->Context.peer().c_str()); - StateFunc_ = &TThis::NextReply; - ResponseSize += sz; + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s (pushed to grpc)", + this, Name_, makeResponseString().data(), this->Context.peer().c_str()); + StateFunc_ = &TThis::NextReply; + ResponseSize += sz; StreamWriter_->Write(*uResp, GetGRpcTag()); - }; - StreamAdaptor_->Enqueue(std::move(cb), false); - } - } - - void WriteByteDataOk(grpc::ByteBuffer* resp) { - auto sz = resp->Length(); - if (Writer_) { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s", this, Name_, - this->Context.peer().c_str()); + }; + StreamAdaptor_->Enqueue(std::move(cb), false); + } + } + + void WriteByteDataOk(grpc::ByteBuffer* resp) { + auto sz = resp->Length(); + if (Writer_) { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s", this, Name_, + this->Context.peer().c_str()); StateFunc_ = &TThis::SetFinishDone; - ResponseSize = sz; + ResponseSize = sz; Writer_->Finish(TUniversalResponseRef<TOut>(resp), grpc::Status::OK, GetGRpcTag()); - } else { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s (enqueued)", this, Name_, - this->Context.peer().c_str()); + } else { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s (enqueued)", this, Name_, + this->Context.peer().c_str()); // because of std::function cannot hold move-only captured object // we allocate shared object on heap to avoid buffer copy auto uResp = MakeIntrusive<TUniversalResponse<TOut>>(resp); auto cb = [this, uResp = std::move(uResp), sz]() { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s (pushed to grpc)", - this, Name_, this->Context.peer().c_str()); - StateFunc_ = &TThis::NextReply; - ResponseSize += sz; + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s (pushed to grpc)", + this, Name_, this->Context.peer().c_str()); + StateFunc_ = &TThis::NextReply; + ResponseSize += sz; StreamWriter_->Write(*uResp, GetGRpcTag()); - }; - StreamAdaptor_->Enqueue(std::move(cb), false); - } - } - - void FinishGrpcStatus(grpc::StatusCode code, const TString& msg, bool urgent) { - Y_VERIFY(code != grpc::OK); - if (code == grpc::StatusCode::UNAUTHENTICATED) { - Counters_->CountNotAuthenticated(); - } else if (code == grpc::StatusCode::RESOURCE_EXHAUSTED) { - Counters_->CountResourceExhausted(); - } - - if (Writer_) { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)", this, - Name_, msg.c_str(), this->Context.peer().c_str(), (int)code); - StateFunc_ = &TThis::SetFinishError; + }; + StreamAdaptor_->Enqueue(std::move(cb), false); + } + } + + void FinishGrpcStatus(grpc::StatusCode code, const TString& msg, bool urgent) { + Y_VERIFY(code != grpc::OK); + if (code == grpc::StatusCode::UNAUTHENTICATED) { + Counters_->CountNotAuthenticated(); + } else if (code == grpc::StatusCode::RESOURCE_EXHAUSTED) { + Counters_->CountResourceExhausted(); + } + + if (Writer_) { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)", this, + Name_, msg.c_str(), this->Context.peer().c_str(), (int)code); + StateFunc_ = &TThis::SetFinishError; TOut resp; Writer_->Finish(TUniversalResponseRef<TOut>(&resp), grpc::Status(code, msg), GetGRpcTag()); - } else { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)" - " (enqueued)", this, Name_, msg.c_str(), this->Context.peer().c_str(), (int)code); - auto cb = [this, code, msg]() { - GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)" - " (pushed to grpc)", this, Name_, msg.c_str(), - this->Context.peer().c_str(), (int)code); - StateFunc_ = &TThis::SetFinishError; - StreamWriter_->Finish(grpc::Status(code, msg), GetGRpcTag()); - }; - StreamAdaptor_->Enqueue(std::move(cb), urgent); - } - } - - bool SetRequestDone(bool ok) { - auto makeRequestString = [&] { - TString resp; - if (ok) { + } else { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)" + " (enqueued)", this, Name_, msg.c_str(), this->Context.peer().c_str(), (int)code); + auto cb = [this, code, msg]() { + GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s nodata (%s) peer# %s, grpc status# (%d)" + " (pushed to grpc)", this, Name_, msg.c_str(), + this->Context.peer().c_str(), (int)code); + StateFunc_ = &TThis::SetFinishError; + StreamWriter_->Finish(grpc::Status(code, msg), GetGRpcTag()); + }; + StreamAdaptor_->Enqueue(std::move(cb), urgent); + } + } + + bool SetRequestDone(bool ok) { + auto makeRequestString = [&] { + TString resp; + if (ok) { TInProtoPrinter printer; - printer.SetSingleLineMode(true); - printer.PrintToString(*Request_, &resp); - } else { - resp = "<not ok>"; - } - return resp; - }; + printer.SetSingleLineMode(true); + printer.PrintToString(*Request_, &resp); + } else { + resp = "<not ok>"; + } + return resp; + }; GRPC_LOG_DEBUG(Logger_, "[%p] received request Name# %s ok# %s data# %s peer# %s", this, Name_, ok ? "true" : "false", makeRequestString().data(), this->Context.peer().c_str()); - - if (this->Context.c_call() == nullptr) { + + if (this->Context.c_call() == nullptr) { Y_VERIFY(!ok); - // One ref by OnFinishTag, grpc will not call this tag if no request received - UnRef(); + // One ref by OnFinishTag, grpc will not call this tag if no request received + UnRef(); } else if (!(RequestRegistered_ = Server_->RegisterRequestCtx(this))) { // Request cannot be registered due to shutdown // It's unsafe to continue, so drop this request without processing GRPC_LOG_DEBUG(Logger_, "[%p] dropping request Name# %s due to shutdown", this, Name_); this->Context.TryCancel(); return false; - } - - Clone(); // TODO: Request pool? - if (!ok) { - Counters_->CountNotOkRequest(); - return false; - } - + } + + Clone(); // TODO: Request pool? + if (!ok) { + Counters_->CountNotOkRequest(); + return false; + } + if (IncRequest()) { - // Adjust counters. - RequestSize = Request_->ByteSize(); - Counters_->StartProcessing(RequestSize); + // Adjust counters. + RequestSize = Request_->ByteSize(); + Counters_->StartProcessing(RequestSize); RequestTimer.Reset(); - - if (!SslServer()) { - Counters_->CountRequestWithoutTls(); - } - - //TODO: Move this in to grpc_request_proxy + + if (!SslServer()) { + Counters_->CountRequestWithoutTls(); + } + + //TODO: Move this in to grpc_request_proxy auto maybeDatabase = GetPeerMetaValues(TStringBuf("x-ydb-database")); - if (maybeDatabase.empty()) { - Counters_->CountRequestsWithoutDatabase(); - } + if (maybeDatabase.empty()) { + Counters_->CountRequestsWithoutDatabase(); + } auto maybeToken = GetPeerMetaValues(TStringBuf("x-ydb-auth-ticket")); - if (maybeToken.empty() || maybeToken[0].empty()) { + if (maybeToken.empty() || maybeToken[0].empty()) { TString db{maybeDatabase ? maybeDatabase[0] : TStringBuf{}}; - Counters_->CountRequestsWithoutToken(); - GRPC_LOG_DEBUG(Logger_, "[%p] received request without user token " - "Name# %s data# %s peer# %s database# %s", this, Name_, - makeRequestString().data(), this->Context.peer().c_str(), db.c_str()); - } - - // Handle current request. - Cb_(this); - } else { - //This request has not been counted - SkipUpdateCountersOnError = true; - FinishGrpcStatus(grpc::StatusCode::RESOURCE_EXHAUSTED, "no resource", true); - } - return true; - } - - bool NextReply(bool ok) { - auto logCb = [this, ok](int left) { - GRPC_LOG_DEBUG(Logger_, "[%p] ready for next reply Name# %s ok# %s peer# %s left# %d", this, Name_, - ok ? "true" : "false", this->Context.peer().c_str(), left); - }; - - if (!ok) { - logCb(-1); + Counters_->CountRequestsWithoutToken(); + GRPC_LOG_DEBUG(Logger_, "[%p] received request without user token " + "Name# %s data# %s peer# %s database# %s", this, Name_, + makeRequestString().data(), this->Context.peer().c_str(), db.c_str()); + } + + // Handle current request. + Cb_(this); + } else { + //This request has not been counted + SkipUpdateCountersOnError = true; + FinishGrpcStatus(grpc::StatusCode::RESOURCE_EXHAUSTED, "no resource", true); + } + return true; + } + + bool NextReply(bool ok) { + auto logCb = [this, ok](int left) { + GRPC_LOG_DEBUG(Logger_, "[%p] ready for next reply Name# %s ok# %s peer# %s left# %d", this, Name_, + ok ? "true" : "false", this->Context.peer().c_str(), left); + }; + + if (!ok) { + logCb(-1); DecRequest(); - Counters_->FinishProcessing(RequestSize, ResponseSize, ok, ResponseStatus, - TDuration::Seconds(RequestTimer.Passed())); - return false; - } - - Ref(); // To prevent destroy during this call in case of execution Finish - size_t left = StreamAdaptor_->ProcessNext(); - logCb(left); - if (NextReplyCb_) { - NextReplyCb_(left); - } - // Now it is safe to destroy even if Finish was called - UnRef(); - return true; - } - - bool SetFinishDone(bool ok) { + Counters_->FinishProcessing(RequestSize, ResponseSize, ok, ResponseStatus, + TDuration::Seconds(RequestTimer.Passed())); + return false; + } + + Ref(); // To prevent destroy during this call in case of execution Finish + size_t left = StreamAdaptor_->ProcessNext(); + logCb(left); + if (NextReplyCb_) { + NextReplyCb_(left); + } + // Now it is safe to destroy even if Finish was called + UnRef(); + return true; + } + + bool SetFinishDone(bool ok) { GRPC_LOG_DEBUG(Logger_, "[%p] finished request Name# %s ok# %s peer# %s", this, Name_, - ok ? "true" : "false", this->Context.peer().c_str()); - //PrintBackTrace(); + ok ? "true" : "false", this->Context.peer().c_str()); + //PrintBackTrace(); DecRequest(); Counters_->FinishProcessing(RequestSize, ResponseSize, ok, ResponseStatus, TDuration::Seconds(RequestTimer.Passed())); - return false; - } - - bool SetFinishError(bool ok) { + return false; + } + + bool SetFinishError(bool ok) { GRPC_LOG_DEBUG(Logger_, "[%p] finished request with error Name# %s ok# %s peer# %s", this, Name_, - ok ? "true" : "false", this->Context.peer().c_str()); - if (!SkipUpdateCountersOnError) { + ok ? "true" : "false", this->Context.peer().c_str()); + if (!SkipUpdateCountersOnError) { DecRequest(); Counters_->FinishProcessing(RequestSize, ResponseSize, ok, ResponseStatus, TDuration::Seconds(RequestTimer.Passed())); - } - return false; - } - - // Returns pointer to IQueueEvent to pass into grpc c runtime - // Implicit C style cast from this to void* is wrong due to multiple inheritance - void* GetGRpcTag() { - return static_cast<IQueueEvent*>(this); - } - - void OnFinish(EQueueEventStatus evStatus) { - if (this->Context.IsCancelled()) { - FinishPromise_.SetValue(EFinishStatus::CANCEL); - } else { - FinishPromise_.SetValue(evStatus == EQueueEventStatus::OK ? EFinishStatus::OK : EFinishStatus::ERROR); - } - } - + } + return false; + } + + // Returns pointer to IQueueEvent to pass into grpc c runtime + // Implicit C style cast from this to void* is wrong due to multiple inheritance + void* GetGRpcTag() { + return static_cast<IQueueEvent*>(this); + } + + void OnFinish(EQueueEventStatus evStatus) { + if (this->Context.IsCancelled()) { + FinishPromise_.SetValue(EFinishStatus::CANCEL); + } else { + FinishPromise_.SetValue(evStatus == EQueueEventStatus::OK ? EFinishStatus::OK : EFinishStatus::ERROR); + } + } + bool IncRequest() { if (!Server_->IncRequest()) return false; @@ -480,36 +480,36 @@ private: } using TStateFunc = bool (TThis::*)(bool); - TService* Server_; - TOnRequest Cb_; - TRequestCallback RequestCallback_; - TStreamRequestCallback StreamRequestCallback_; + TService* Server_; + TOnRequest Cb_; + TRequestCallback RequestCallback_; + TStreamRequestCallback StreamRequestCallback_; const char* const Name_; TLoggerPtr Logger_; ICounterBlockPtr Counters_; IGRpcRequestLimiterPtr RequestLimiter_; - + THolder<grpc::ServerAsyncResponseWriter<TUniversalResponseRef<TOut>>> Writer_; - THolder<grpc::ServerAsyncWriterInterface<TUniversalResponse<TOut>>> StreamWriter_; - TStateFunc StateFunc_; - TIn* Request_; - - google::protobuf::Arena Arena_; - TOnNextReply NextReplyCb_; - ui32 RequestSize = 0; - ui32 ResponseSize = 0; + THolder<grpc::ServerAsyncWriterInterface<TUniversalResponse<TOut>>> StreamWriter_; + TStateFunc StateFunc_; + TIn* Request_; + + google::protobuf::Arena Arena_; + TOnNextReply NextReplyCb_; + ui32 RequestSize = 0; + ui32 ResponseSize = 0; ui32 ResponseStatus = 0; THPTimer RequestTimer; - TAuthState AuthState_ = 0; + TAuthState AuthState_ = 0; bool RequestRegistered_ = false; - + using TFixedEvent = TQueueFixedEvent<TGRpcRequestImpl>; TFixedEvent OnFinishTag = { this, &TGRpcRequestImpl::OnFinish }; - NThreading::TPromise<EFinishStatus> FinishPromise_; - bool SkipUpdateCountersOnError = false; - IStreamAdaptor::TPtr StreamAdaptor_; -}; - + NThreading::TPromise<EFinishStatus> FinishPromise_; + bool SkipUpdateCountersOnError = false; + IStreamAdaptor::TPtr StreamAdaptor_; +}; + template<typename TIn, typename TOut, typename TService, typename TInProtoPrinter=google::protobuf::TextFormat::Printer, typename TOutProtoPrinter=google::protobuf::TextFormat::Printer> class TGRpcRequest: public TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter> { using TBase = TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter>; diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h index 506221dd98..fcfce1c181 100644 --- a/library/cpp/grpc/server/grpc_request_base.h +++ b/library/cpp/grpc/server/grpc_request_base.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <google/protobuf/message.h> #include <library/cpp/threading/future/future.h> - + #include <grpc++/server_context.h> -namespace grpc { -class ByteBuffer; -} - +namespace grpc { +class ByteBuffer; +} + namespace NGrpc { - -extern const char* GRPC_USER_AGENT_HEADER; - -struct TAuthState { - enum EAuthState { - AS_NOT_PERFORMED, - AS_OK, - AS_FAIL, - AS_UNAVAILABLE - }; - TAuthState(bool needAuth) - : NeedAuth(needAuth) - , State(AS_NOT_PERFORMED) - {} - bool NeedAuth; - EAuthState State; -}; - + +extern const char* GRPC_USER_AGENT_HEADER; + +struct TAuthState { + enum EAuthState { + AS_NOT_PERFORMED, + AS_OK, + AS_FAIL, + AS_UNAVAILABLE + }; + TAuthState(bool needAuth) + : NeedAuth(needAuth) + , State(AS_NOT_PERFORMED) + {} + bool NeedAuth; + EAuthState State; +}; + //! An interface that may be used to limit concurrency of requests class IGRpcRequestLimiter: public TThrRefBase { @@ -38,79 +38,79 @@ public: using IGRpcRequestLimiterPtr = TIntrusivePtr<IGRpcRequestLimiter>; -//! State of current request +//! State of current request class IRequestContextBase: public TThrRefBase { -public: - enum class EFinishStatus { - OK, - ERROR, - CANCEL - }; - using TAsyncFinishResult = NThreading::TFuture<EFinishStatus>; - - using TOnNextReply = std::function<void (size_t left)>; - - //! Get pointer to the request's message. - virtual const NProtoBuf::Message* GetRequest() const = 0; - - //! Get current auth state - virtual TAuthState& GetAuthState() = 0; - - //! Send common response (The request shoult be created for protobuf response type) - //! Implementation can swap protobuf message +public: + enum class EFinishStatus { + OK, + ERROR, + CANCEL + }; + using TAsyncFinishResult = NThreading::TFuture<EFinishStatus>; + + using TOnNextReply = std::function<void (size_t left)>; + + //! Get pointer to the request's message. + virtual const NProtoBuf::Message* GetRequest() const = 0; + + //! Get current auth state + virtual TAuthState& GetAuthState() = 0; + + //! Send common response (The request shoult be created for protobuf response type) + //! Implementation can swap protobuf message virtual void Reply(NProtoBuf::Message* resp, ui32 status = 0) = 0; - + //! Send serialised response (The request shoult be created for bytes response type) - //! Implementation can swap ByteBuffer + //! Implementation can swap ByteBuffer virtual void Reply(grpc::ByteBuffer* resp, ui32 status = 0) = 0; - - //! Send grpc UNAUTHENTICATED status - virtual void ReplyUnauthenticated(const TString& in) = 0; - + + //! Send grpc UNAUTHENTICATED status + virtual void ReplyUnauthenticated(const TString& in) = 0; + //! Send grpc error virtual void ReplyError(grpc::StatusCode code, const TString& msg) = 0; - //! Returns deadline (server epoch related) if peer set it on its side, or Instanse::Max() otherwise - virtual TInstant Deadline() const = 0; + //! Returns deadline (server epoch related) if peer set it on its side, or Instanse::Max() otherwise + virtual TInstant Deadline() const = 0; //! Returns available peer metadata keys virtual TSet<TStringBuf> GetPeerMetaKeys() const = 0; - //! Returns peer optional metavalue + //! Returns peer optional metavalue virtual TVector<TStringBuf> GetPeerMetaValues(TStringBuf key) const = 0; - + //! Returns request compression level virtual grpc_compression_level GetCompressionLevel() const = 0; - //! Returns protobuf arena allocator associated with current request - //! Lifetime of the arena is lifetime of the context - virtual google::protobuf::Arena* GetArena() = 0; - - //! Add trailing metadata in to grpc context - //! The metadata will be send at the time of rpc finish - virtual void AddTrailingMetadata(const TString& key, const TString& value) = 0; - + //! Returns protobuf arena allocator associated with current request + //! Lifetime of the arena is lifetime of the context + virtual google::protobuf::Arena* GetArena() = 0; + + //! Add trailing metadata in to grpc context + //! The metadata will be send at the time of rpc finish + virtual void AddTrailingMetadata(const TString& key, const TString& value) = 0; + //! Use validated database name for counters virtual void UseDatabase(const TString& database) = 0; - // Streaming part - - //! Set callback. The callback will be called when response deliverid to the client - //! after that we can call Reply again in streaming mode. Yes, GRpc says there is only one - //! reply in flight - virtual void SetNextReplyCallback(TOnNextReply&& cb) = 0; - - //! Finish streaming reply - virtual void FinishStreamingOk() = 0; - - //! Returns future to get cancel of finish notification - virtual TAsyncFinishResult GetFinishFuture() = 0; + // Streaming part + + //! Set callback. The callback will be called when response deliverid to the client + //! after that we can call Reply again in streaming mode. Yes, GRpc says there is only one + //! reply in flight + virtual void SetNextReplyCallback(TOnNextReply&& cb) = 0; + + //! Finish streaming reply + virtual void FinishStreamingOk() = 0; + + //! Returns future to get cancel of finish notification + virtual TAsyncFinishResult GetFinishFuture() = 0; //! Returns peer address virtual TString GetPeer() const = 0; //! Returns true if server is using ssl virtual bool SslServer() const = 0; -}; - +}; + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_response.h b/library/cpp/grpc/server/grpc_response.h index 53a3195982..8e9afe44d5 100644 --- a/library/cpp/grpc/server/grpc_response.h +++ b/library/cpp/grpc/server/grpc_response.h @@ -11,7 +11,7 @@ namespace NGrpc { * Universal response that owns underlying message or buffer. */ template <typename TMsg> -class TUniversalResponse: public TAtomicRefCount<TUniversalResponse<TMsg>>, public TMoveOnly { +class TUniversalResponse: public TAtomicRefCount<TUniversalResponse<TMsg>>, public TMoveOnly { friend class grpc::SerializationTraits<TUniversalResponse<TMsg>>; public: diff --git a/library/cpp/grpc/server/grpc_server.cpp b/library/cpp/grpc/server/grpc_server.cpp index 3e68b26e1c..7437b7a8f5 100644 --- a/library/cpp/grpc/server/grpc_server.cpp +++ b/library/cpp/grpc/server/grpc_server.cpp @@ -1,12 +1,12 @@ -#include "grpc_server.h" - -#include <util/string/join.h> -#include <util/generic/yexception.h> -#include <util/system/thread.h> - -#include <grpc++/resource_quota.h> +#include "grpc_server.h" + +#include <util/string/join.h> +#include <util/generic/yexception.h> +#include <util/system/thread.h> + +#include <grpc++/resource_quota.h> #include <contrib/libs/grpc/src/core/lib/iomgr/socket_mutator.h> - + #if !defined(_WIN32) && !defined(_WIN64) #include <sys/socket.h> @@ -16,9 +16,9 @@ #endif namespace NGrpc { - -using NThreading::TFuture; - + +using NThreading::TFuture; + static void PullEvents(grpc::ServerCompletionQueue* cq) { TThread::SetCurrentThreadName("grpc_server"); while (true) { @@ -37,33 +37,33 @@ static void PullEvents(grpc::ServerCompletionQueue* cq) { } } -TGRpcServer::TGRpcServer(const TServerOptions& opts) - : Options_(opts) - , Limiter_(Options_.MaxGlobalRequestInFlight) - {} - -TGRpcServer::~TGRpcServer() { - Y_VERIFY(Ts.empty()); - Services_.clear(); -} - -void TGRpcServer::AddService(IGRpcServicePtr service) { - Services_.push_back(service); -} - -void TGRpcServer::Start() { +TGRpcServer::TGRpcServer(const TServerOptions& opts) + : Options_(opts) + , Limiter_(Options_.MaxGlobalRequestInFlight) + {} + +TGRpcServer::~TGRpcServer() { + Y_VERIFY(Ts.empty()); + Services_.clear(); +} + +void TGRpcServer::AddService(IGRpcServicePtr service) { + Services_.push_back(service); +} + +void TGRpcServer::Start() { TString server_address(Join(":", Options_.Host, Options_.Port)); // https://st.yandex-team.ru/DTCC-695 - using grpc::ServerBuilder; - using grpc::ResourceQuota; - ServerBuilder builder; + using grpc::ServerBuilder; + using grpc::ResourceQuota; + ServerBuilder builder; auto credentials = grpc::InsecureServerCredentials(); if (Options_.SslData) { - grpc::SslServerCredentialsOptions::PemKeyCertPair keycert; - keycert.cert_chain = std::move(Options_.SslData->Cert); - keycert.private_key = std::move(Options_.SslData->Key); - grpc::SslServerCredentialsOptions sslOps; - sslOps.pem_root_certs = std::move(Options_.SslData->Root); - sslOps.pem_key_cert_pairs.push_back(keycert); + grpc::SslServerCredentialsOptions::PemKeyCertPair keycert; + keycert.cert_chain = std::move(Options_.SslData->Cert); + keycert.private_key = std::move(Options_.SslData->Key); + grpc::SslServerCredentialsOptions sslOps; + sslOps.pem_root_certs = std::move(Options_.SslData->Root); + sslOps.pem_key_cert_pairs.push_back(keycert); credentials = grpc::SslServerCredentials(sslOps); } if (Options_.ExternalListener) { @@ -72,58 +72,58 @@ void TGRpcServer::Start() { credentials )); } else { - builder.AddListeningPort(server_address, credentials); - } - builder.SetMaxReceiveMessageSize(Options_.MaxMessageSize); - builder.SetMaxSendMessageSize(Options_.MaxMessageSize); - for (IGRpcServicePtr service : Services_) { + builder.AddListeningPort(server_address, credentials); + } + builder.SetMaxReceiveMessageSize(Options_.MaxMessageSize); + builder.SetMaxSendMessageSize(Options_.MaxMessageSize); + for (IGRpcServicePtr service : Services_) { service->SetServerOptions(Options_); - builder.RegisterService(service->GetService()); - service->SetGlobalLimiterHandle(&Limiter_); - } - + builder.RegisterService(service->GetService()); + service->SetGlobalLimiterHandle(&Limiter_); + } + class TKeepAliveOption: public grpc::ServerBuilderOption { - public: - TKeepAliveOption(int idle, int interval) - : Idle(idle) - , Interval(interval) - , KeepAliveEnabled(true) - {} - - TKeepAliveOption() - : Idle(0) - , Interval(0) - , KeepAliveEnabled(false) - {} - - void UpdateArguments(grpc::ChannelArguments *args) override { - args->SetInt(GRPC_ARG_HTTP2_MAX_PING_STRIKES, 0); - args->SetInt(GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 1000); - if (KeepAliveEnabled) { - args->SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); - args->SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); - args->SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, Idle * 1000); - args->SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, Idle * 1000); - args->SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, Interval * 1000); - } - } - - void UpdatePlugins(std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/) override - {} - private: - const int Idle; - const int Interval; - const bool KeepAliveEnabled; - }; - - if (Options_.KeepAliveEnable) { - builder.SetOption(std::make_unique<TKeepAliveOption>( - Options_.KeepAliveIdleTimeoutTriggerSec, - Options_.KeepAliveProbeIntervalSec)); - } else { - builder.SetOption(std::make_unique<TKeepAliveOption>()); - } - + public: + TKeepAliveOption(int idle, int interval) + : Idle(idle) + , Interval(interval) + , KeepAliveEnabled(true) + {} + + TKeepAliveOption() + : Idle(0) + , Interval(0) + , KeepAliveEnabled(false) + {} + + void UpdateArguments(grpc::ChannelArguments *args) override { + args->SetInt(GRPC_ARG_HTTP2_MAX_PING_STRIKES, 0); + args->SetInt(GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 1000); + if (KeepAliveEnabled) { + args->SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); + args->SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + args->SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, Idle * 1000); + args->SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, Idle * 1000); + args->SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, Interval * 1000); + } + } + + void UpdatePlugins(std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/) override + {} + private: + const int Idle; + const int Interval; + const bool KeepAliveEnabled; + }; + + if (Options_.KeepAliveEnable) { + builder.SetOption(std::make_unique<TKeepAliveOption>( + Options_.KeepAliveIdleTimeoutTriggerSec, + Options_.KeepAliveProbeIntervalSec)); + } else { + builder.SetOption(std::make_unique<TKeepAliveOption>()); + } + if (Options_.UseCompletionQueuePerThread) { for (size_t i = 0; i < Options_.WorkerThreads; ++i) { CQS_.push_back(builder.AddCompletionQueue()); @@ -132,30 +132,30 @@ void TGRpcServer::Start() { CQS_.push_back(builder.AddCompletionQueue()); } - if (Options_.GRpcMemoryQuotaBytes) { + if (Options_.GRpcMemoryQuotaBytes) { // See details KIKIMR-6932 - /* - grpc::ResourceQuota quota("memory_bound"); - quota.Resize(Options_.GRpcMemoryQuotaBytes); - - builder.SetResourceQuota(quota); - */ + /* + grpc::ResourceQuota quota("memory_bound"); + quota.Resize(Options_.GRpcMemoryQuotaBytes); + + builder.SetResourceQuota(quota); + */ Cerr << "GRpc memory quota temporarily disabled due to issues with grpc quoter" << Endl; - } + } Options_.ServerBuilderMutator(builder); builder.SetDefaultCompressionLevel(Options_.DefaultCompressionLevel); - - Server_ = builder.BuildAndStart(); - if (!Server_) { + + Server_ = builder.BuildAndStart(); + if (!Server_) { ythrow yexception() << "can't start grpc server on " << server_address; - } + } size_t index = 0; - for (IGRpcServicePtr service : Services_) { + for (IGRpcServicePtr service : Services_) { // TODO: provide something else for services instead of ServerCompletionQueue service->InitService(CQS_[index++ % CQS_.size()].get(), Options_.Logger); - } - + } + if (Options_.UseCompletionQueuePerThread) { for (size_t i = 0; i < Options_.WorkerThreads; ++i) { auto* cq = &CQS_[i]; @@ -170,71 +170,71 @@ void TGRpcServer::Start() { PullEvents(cq->get()); })); } - } + } if (Options_.ExternalListener) { Options_.ExternalListener->Start(); } -} - -void TGRpcServer::Stop() { - for (auto& service : Services_) { - service->StopService(); - } - - auto now = TInstant::Now(); - - if (Server_) { - i64 sec = Options_.GRpcShutdownDeadline.Seconds(); - Y_VERIFY(Options_.GRpcShutdownDeadline.NanoSecondsOfSecond() <= Max<i32>()); - i32 nanosecOfSec = Options_.GRpcShutdownDeadline.NanoSecondsOfSecond(); - Server_->Shutdown(gpr_timespec{sec, nanosecOfSec, GPR_TIMESPAN}); - } - +} + +void TGRpcServer::Stop() { + for (auto& service : Services_) { + service->StopService(); + } + + auto now = TInstant::Now(); + + if (Server_) { + i64 sec = Options_.GRpcShutdownDeadline.Seconds(); + Y_VERIFY(Options_.GRpcShutdownDeadline.NanoSecondsOfSecond() <= Max<i32>()); + i32 nanosecOfSec = Options_.GRpcShutdownDeadline.NanoSecondsOfSecond(); + Server_->Shutdown(gpr_timespec{sec, nanosecOfSec, GPR_TIMESPAN}); + } + for (ui64 attempt = 0; ; ++attempt) { bool unsafe = false; - size_t infly = 0; - for (auto& service : Services_) { + size_t infly = 0; + for (auto& service : Services_) { unsafe |= service->IsUnsafeToShutdown(); infly += service->RequestsInProgress(); - } - + } + if (!unsafe && !infly) - break; + break; - auto spent = (TInstant::Now() - now).SecondsFloat(); + auto spent = (TInstant::Now() - now).SecondsFloat(); if (attempt % 300 == 0) { // don't log too much Cerr << "GRpc shutdown warning: left infly: " << infly << ", spent: " << spent << " sec" << Endl; } if (!unsafe && spent > Options_.GRpcShutdownDeadline.SecondsFloat()) - break; - Sleep(TDuration::MilliSeconds(10)); - } - - // Always shutdown the completion queue after the server. + break; + Sleep(TDuration::MilliSeconds(10)); + } + + // Always shutdown the completion queue after the server. for (auto& cq : CQS_) { cq->Shutdown(); - } - - for (auto ti = Ts.begin(); ti != Ts.end(); ++ti) { - (*ti)->Join(); - } - - Ts.clear(); + } + + for (auto ti = Ts.begin(); ti != Ts.end(); ++ti) { + (*ti)->Join(); + } + + Ts.clear(); if (Options_.ExternalListener) { Options_.ExternalListener->Stop(); } -} - -ui16 TGRpcServer::GetPort() const { - return Options_.Port; -} - -TString TGRpcServer::GetHost() const { - return Options_.Host; -} - +} + +ui16 TGRpcServer::GetPort() const { + return Options_.Port; +} + +TString TGRpcServer::GetHost() const { + return Options_.Host; +} + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_server.h b/library/cpp/grpc/server/grpc_server.h index 0a4123d84e..d6814a90a0 100644 --- a/library/cpp/grpc/server/grpc_server.h +++ b/library/cpp/grpc/server/grpc_server.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include "grpc_request_base.h" #include "logger.h" #include <library/cpp/threading/future/future.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> #include <util/generic/maybe.h> -#include <util/generic/queue.h> -#include <util/generic/hash_set.h> -#include <util/system/types.h> -#include <util/system/mutex.h> +#include <util/generic/queue.h> +#include <util/generic/hash_set.h> +#include <util/system/types.h> +#include <util/system/mutex.h> #include <util/thread/factory.h> - -#include <grpc++/grpc++.h> - + +#include <grpc++/grpc++.h> + namespace NGrpc { - -constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; - -struct TSslData { - TString Cert; - TString Key; - TString Root; -}; - + +constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; + +struct TSslData { + TString Cert; + TString Key; + TString Root; +}; + struct IExternalListener : public TThrRefBase { @@ -36,55 +36,55 @@ struct IExternalListener virtual void Stop() = 0; }; -//! Server's options. -struct TServerOptions { -#define DECLARE_FIELD(name, type, default) \ - type name{default}; \ - inline TServerOptions& Set##name(const type& value) { \ - name = value; \ - return *this; \ - } - - //! Hostname of server to bind to. - DECLARE_FIELD(Host, TString, "[::]"); - //! Service port. - DECLARE_FIELD(Port, ui16, 0); - - //! Number of worker threads. - DECLARE_FIELD(WorkerThreads, size_t, 2); - +//! Server's options. +struct TServerOptions { +#define DECLARE_FIELD(name, type, default) \ + type name{default}; \ + inline TServerOptions& Set##name(const type& value) { \ + name = value; \ + return *this; \ + } + + //! Hostname of server to bind to. + DECLARE_FIELD(Host, TString, "[::]"); + //! Service port. + DECLARE_FIELD(Port, ui16, 0); + + //! Number of worker threads. + DECLARE_FIELD(WorkerThreads, size_t, 2); + //! Create one completion queue per thread DECLARE_FIELD(UseCompletionQueuePerThread, bool, false); - //! Memory quota size for grpc server in bytes. Zero means unlimited. - DECLARE_FIELD(GRpcMemoryQuotaBytes, size_t, 0); - - //! How long to wait until pending rpcs are forcefully terminated. - DECLARE_FIELD(GRpcShutdownDeadline, TDuration, TDuration::Seconds(30)); - - //! In/Out message size limit - DECLARE_FIELD(MaxMessageSize, size_t, DEFAULT_GRPC_MESSAGE_SIZE_LIMIT); - - //! Use GRpc keepalive - DECLARE_FIELD(KeepAliveEnable, TMaybe<bool>, TMaybe<bool>()); - - //! GRPC_ARG_KEEPALIVE_TIME_MS setting + //! Memory quota size for grpc server in bytes. Zero means unlimited. + DECLARE_FIELD(GRpcMemoryQuotaBytes, size_t, 0); + + //! How long to wait until pending rpcs are forcefully terminated. + DECLARE_FIELD(GRpcShutdownDeadline, TDuration, TDuration::Seconds(30)); + + //! In/Out message size limit + DECLARE_FIELD(MaxMessageSize, size_t, DEFAULT_GRPC_MESSAGE_SIZE_LIMIT); + + //! Use GRpc keepalive + DECLARE_FIELD(KeepAliveEnable, TMaybe<bool>, TMaybe<bool>()); + + //! GRPC_ARG_KEEPALIVE_TIME_MS setting DECLARE_FIELD(KeepAliveIdleTimeoutTriggerSec, int, 0); - //! Deprecated, ths option ignored. Will be removed soon. + //! Deprecated, ths option ignored. Will be removed soon. DECLARE_FIELD(KeepAliveMaxProbeCount, int, 0); - //! GRPC_ARG_KEEPALIVE_TIMEOUT_MS setting + //! GRPC_ARG_KEEPALIVE_TIMEOUT_MS setting DECLARE_FIELD(KeepAliveProbeIntervalSec, int, 0); - //! Max number of requests processing by services (global limit for grpc server) - DECLARE_FIELD(MaxGlobalRequestInFlight, size_t, 100000); - - //! SSL server data - DECLARE_FIELD(SslData, TMaybe<TSslData>, TMaybe<TSslData>()); - - //! GRPC auth - DECLARE_FIELD(UseAuth, bool, false); + //! Max number of requests processing by services (global limit for grpc server) + DECLARE_FIELD(MaxGlobalRequestInFlight, size_t, 100000); + + //! SSL server data + DECLARE_FIELD(SslData, TMaybe<TSslData>, TMaybe<TSslData>()); + + //! GRPC auth + DECLARE_FIELD(UseAuth, bool, false); //! Default compression level. Used when no compression options provided by client. // Mapping to particular compression algorithm depends on client. @@ -98,75 +98,75 @@ struct TServerOptions { //! Logger which will be used to write logs about requests handling (iff appropriate log level is enabled). DECLARE_FIELD(Logger, TLoggerPtr, nullptr); -#undef DECLARE_FIELD -}; - -class IQueueEvent { -public: - virtual ~IQueueEvent() = default; - +#undef DECLARE_FIELD +}; + +class IQueueEvent { +public: + virtual ~IQueueEvent() = default; + //! Execute an action defined by implementation. - virtual bool Execute(bool ok) = 0; - - //! It is time to perform action requested by AcquireToken server method. It will be called under lock which is also - // used in ReturnToken/AcquireToken methods. Default implementation does nothing assuming that request processor does - // not implement in flight management. - virtual void Process() {} - - //! Finish and destroy request. - virtual void DestroyRequest() = 0; -}; - -class ICancelableContext { -public: - virtual void Shutdown() = 0; - virtual ~ICancelableContext() = default; -}; - + virtual bool Execute(bool ok) = 0; + + //! It is time to perform action requested by AcquireToken server method. It will be called under lock which is also + // used in ReturnToken/AcquireToken methods. Default implementation does nothing assuming that request processor does + // not implement in flight management. + virtual void Process() {} + + //! Finish and destroy request. + virtual void DestroyRequest() = 0; +}; + +class ICancelableContext { +public: + virtual void Shutdown() = 0; + virtual ~ICancelableContext() = default; +}; + template <class TLimit> class TInFlightLimiterImpl { -public: +public: explicit TInFlightLimiterImpl(const TLimit& limit) - : Limit_(limit) - {} - - bool Inc() { - i64 newVal; - i64 prev; - do { - prev = AtomicGet(CurInFlightReqs_); - Y_VERIFY(prev >= 0); - if (Limit_ && prev > Limit_) { - return false; - } - newVal = prev + 1; - } while (!AtomicCas(&CurInFlightReqs_, newVal, prev)); - return true; - } - - void Dec() { - i64 newVal = AtomicDecrement(CurInFlightReqs_); - Y_VERIFY(newVal >= 0); - } - - i64 GetCurrentInFlight() const { - return AtomicGet(CurInFlightReqs_); - } - -private: + : Limit_(limit) + {} + + bool Inc() { + i64 newVal; + i64 prev; + do { + prev = AtomicGet(CurInFlightReqs_); + Y_VERIFY(prev >= 0); + if (Limit_ && prev > Limit_) { + return false; + } + newVal = prev + 1; + } while (!AtomicCas(&CurInFlightReqs_, newVal, prev)); + return true; + } + + void Dec() { + i64 newVal = AtomicDecrement(CurInFlightReqs_); + Y_VERIFY(newVal >= 0); + } + + i64 GetCurrentInFlight() const { + return AtomicGet(CurInFlightReqs_); + } + +private: const TLimit Limit_; - TAtomic CurInFlightReqs_ = 0; -}; - + TAtomic CurInFlightReqs_ = 0; +}; + using TGlobalLimiter = TInFlightLimiterImpl<i64>; class IGRpcService: public TThrRefBase { -public: - virtual grpc::Service* GetService() = 0; - virtual void StopService() noexcept = 0; +public: + virtual grpc::Service* GetService() = 0; + virtual void StopService() noexcept = 0; virtual void InitService(grpc::ServerCompletionQueue* cq, TLoggerPtr logger) = 0; - virtual void SetGlobalLimiterHandle(TGlobalLimiter* limiter) = 0; + virtual void SetGlobalLimiterHandle(TGlobalLimiter* limiter) = 0; virtual bool IsUnsafeToShutdown() const = 0; virtual size_t RequestsInProgress() const = 0; @@ -175,11 +175,11 @@ public: * service to inspect server options and initialize accordingly. */ virtual void SetServerOptions(const TServerOptions& options) = 0; -}; - -template<typename T> +}; + +template<typename T> class TGrpcServiceBase: public IGRpcService { -public: +public: class TShutdownGuard { using TOwner = TGrpcServiceBase<T>; friend class TGrpcServiceBase<T>; @@ -232,20 +232,20 @@ public: }; public: - using TCurrentGRpcService = T; - - void StopService() noexcept override { - with_lock(Lock_) { + using TCurrentGRpcService = T; + + void StopService() noexcept override { + with_lock(Lock_) { AtomicSet(ShuttingDown_, 1); - // Send TryCansel to event (can be send after finishing). - // Actual dtors will be called from grpc thread, so deadlock impossible - for (auto* request : Requests_) { - request->Shutdown(); - } - } - } - + // Send TryCansel to event (can be send after finishing). + // Actual dtors will be called from grpc thread, so deadlock impossible + for (auto* request : Requests_) { + request->Shutdown(); + } + } + } + TShutdownGuard ProtectShutdown() noexcept { AtomicIncrement(GuardCount_); if (IsShuttingDown()) { @@ -261,35 +261,35 @@ public: } size_t RequestsInProgress() const override { - size_t c = 0; - with_lock(Lock_) { - c = Requests_.size(); - } - return c; - } - + size_t c = 0; + with_lock(Lock_) { + c = Requests_.size(); + } + return c; + } + void SetServerOptions(const TServerOptions& options) override { SslServer_ = bool(options.SslData); NeedAuth_ = options.UseAuth; - } - - void SetGlobalLimiterHandle(TGlobalLimiter* /*limiter*/) override {} - - //! Check if the server is going to shut down. - bool IsShuttingDown() const { - return AtomicGet(ShuttingDown_); - } - + } + + void SetGlobalLimiterHandle(TGlobalLimiter* /*limiter*/) override {} + + //! Check if the server is going to shut down. + bool IsShuttingDown() const { + return AtomicGet(ShuttingDown_); + } + bool SslServer() const { return SslServer_; } - bool NeedAuth() const { - return NeedAuth_; - } - + bool NeedAuth() const { + return NeedAuth_; + } + bool RegisterRequestCtx(ICancelableContext* req) { - with_lock(Lock_) { + with_lock(Lock_) { auto r = Requests_.emplace(req); Y_VERIFY(r.second, "Ctx already registered"); @@ -298,59 +298,59 @@ public: Requests_.erase(r.first); return false; } - } + } return true; - } - - void DeregisterRequestCtx(ICancelableContext* req) { - with_lock(Lock_) { + } + + void DeregisterRequestCtx(ICancelableContext* req) { + with_lock(Lock_) { Y_VERIFY(Requests_.erase(req), "Ctx is not registered"); - } - } - -protected: - using TGrpcAsyncService = typename TCurrentGRpcService::AsyncService; - TGrpcAsyncService Service_; - + } + } + +protected: + using TGrpcAsyncService = typename TCurrentGRpcService::AsyncService; + TGrpcAsyncService Service_; + TGrpcAsyncService* GetService() override { - return &Service_; - } - -private: - TAtomic ShuttingDown_ = 0; + return &Service_; + } + +private: + TAtomic ShuttingDown_ = 0; TAtomic GuardCount_ = 0; - + bool SslServer_ = false; - bool NeedAuth_ = false; - - THashSet<ICancelableContext*> Requests_; - TAdaptiveLock Lock_; -}; - + bool NeedAuth_ = false; + + THashSet<ICancelableContext*> Requests_; + TAdaptiveLock Lock_; +}; + class TGRpcServer { -public: - using IGRpcServicePtr = TIntrusivePtr<IGRpcService>; - TGRpcServer(const TServerOptions& opts); - ~TGRpcServer(); - void AddService(IGRpcServicePtr service); - void Start(); - // Send stop to registred services and call Shutdown on grpc server - // This method MUST be called before destroying TGRpcServer - void Stop(); - ui16 GetPort() const; - TString GetHost() const; - -private: +public: + using IGRpcServicePtr = TIntrusivePtr<IGRpcService>; + TGRpcServer(const TServerOptions& opts); + ~TGRpcServer(); + void AddService(IGRpcServicePtr service); + void Start(); + // Send stop to registred services and call Shutdown on grpc server + // This method MUST be called before destroying TGRpcServer + void Stop(); + ui16 GetPort() const; + TString GetHost() const; + +private: using IThreadRef = TAutoPtr<IThreadFactory::IThread>; - - const TServerOptions Options_; - std::unique_ptr<grpc::Server> Server_; + + const TServerOptions Options_; + std::unique_ptr<grpc::Server> Server_; std::vector<std::unique_ptr<grpc::ServerCompletionQueue>> CQS_; TVector<IThreadRef> Ts; - + TVector<IGRpcServicePtr> Services_; - TGlobalLimiter Limiter_; -}; - + TGlobalLimiter Limiter_; +}; + } // namespace NGrpc diff --git a/library/cpp/grpc/server/ut/stream_adaptor_ut.cpp b/library/cpp/grpc/server/ut/stream_adaptor_ut.cpp index 0840c77176..c34d3b8c2b 100644 --- a/library/cpp/grpc/server/ut/stream_adaptor_ut.cpp +++ b/library/cpp/grpc/server/ut/stream_adaptor_ut.cpp @@ -1,121 +1,121 @@ #include <library/cpp/grpc/server/grpc_request.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/system/thread.h> -#include <util/thread/pool.h> - + +#include <util/system/thread.h> +#include <util/thread/pool.h> + using namespace NGrpc; - -// Here we emulate stream data producer + +// Here we emulate stream data producer class TOrderedProducer: public TThread { -public: - TOrderedProducer(IStreamAdaptor* adaptor, ui64 max, bool withSleep, std::function<void(ui64)>&& consumerOp) - : TThread(&ThreadProc, this) - , Adaptor_(adaptor) - , Max_(max) - , WithSleep_(withSleep) - , ConsumerOp_(std::move(consumerOp)) - {} - - static void* ThreadProc(void* _this) { - SetCurrentThreadName("OrderedProducerThread"); - static_cast<TOrderedProducer*>(_this)->Exec(); - return nullptr; - } - - void Exec() { - for (ui64 i = 0; i < Max_; i++) { - auto cb = [i, this]() mutable { - ConsumerOp_(i); - }; - Adaptor_->Enqueue(std::move(cb), false); - if (WithSleep_ && (i % 256 == 0)) { - Sleep(TDuration::MilliSeconds(10)); - } - } - } - -private: - IStreamAdaptor* Adaptor_; - const ui64 Max_; - const bool WithSleep_; - std::function<void(ui64)> ConsumerOp_; -}; - -Y_UNIT_TEST_SUITE(StreamAdaptor) { - static void OrderingTest(size_t threads, bool withSleep) { - - auto adaptor = CreateStreamAdaptor(); - - const i64 max = 10000; - - // Here we will emulate grpc stream (NextReply call after writing) +public: + TOrderedProducer(IStreamAdaptor* adaptor, ui64 max, bool withSleep, std::function<void(ui64)>&& consumerOp) + : TThread(&ThreadProc, this) + , Adaptor_(adaptor) + , Max_(max) + , WithSleep_(withSleep) + , ConsumerOp_(std::move(consumerOp)) + {} + + static void* ThreadProc(void* _this) { + SetCurrentThreadName("OrderedProducerThread"); + static_cast<TOrderedProducer*>(_this)->Exec(); + return nullptr; + } + + void Exec() { + for (ui64 i = 0; i < Max_; i++) { + auto cb = [i, this]() mutable { + ConsumerOp_(i); + }; + Adaptor_->Enqueue(std::move(cb), false); + if (WithSleep_ && (i % 256 == 0)) { + Sleep(TDuration::MilliSeconds(10)); + } + } + } + +private: + IStreamAdaptor* Adaptor_; + const ui64 Max_; + const bool WithSleep_; + std::function<void(ui64)> ConsumerOp_; +}; + +Y_UNIT_TEST_SUITE(StreamAdaptor) { + static void OrderingTest(size_t threads, bool withSleep) { + + auto adaptor = CreateStreamAdaptor(); + + const i64 max = 10000; + + // Here we will emulate grpc stream (NextReply call after writing) std::unique_ptr<IThreadPool> consumerQueue(new TThreadPool(TThreadPool::TParams().SetBlocking(false).SetCatching(false))); - // And make sure only one request inflight (see UNIT_ASSERT on adding to the queue) - consumerQueue->Start(threads, 1); - - // Non atomic!!! Stream adaptor must protect us - ui64 curVal = 0; - - // Used just to wait in the main thread - TAtomic finished = false; - auto consumerOp = [&finished, &curVal, ptr{adaptor.get()}, queue{consumerQueue.get()}](ui64 i) { - // Check no reordering inside stream adaptor - // and no simultanious consumer Op call - UNIT_ASSERT_VALUES_EQUAL(curVal, i); - curVal++; - // We must set finished flag after last ProcessNext, but we can`t compare curVal and max after ProcessNext - // so compare here and set after - bool tmp = curVal == max; - bool res = queue->AddFunc([ptr, &finished, tmp, &curVal, i]() { - // Additional check the value still same - // run under tsan makes sure no consumer Op call before we call ProcessNext - UNIT_ASSERT_VALUES_EQUAL(curVal, i + 1); - ptr->ProcessNext(); - // Reordering after ProcessNext is possible, so check tmp and set finished to true - if (tmp) - AtomicSet(finished, true); - }); - UNIT_ASSERT(res); - }; - - TOrderedProducer producer(adaptor.get(), max, withSleep, std::move(consumerOp)); - - producer.Start(); - producer.Join(); - - while (!AtomicGet(finished)) - { - Sleep(TDuration::MilliSeconds(100)); - } - - consumerQueue->Stop(); - - UNIT_ASSERT_VALUES_EQUAL(curVal, max); - } - - Y_UNIT_TEST(OrderingOneThread) { - OrderingTest(1, false); - } - - Y_UNIT_TEST(OrderingTwoThreads) { - OrderingTest(2, false); - } - - Y_UNIT_TEST(OrderingManyThreads) { - OrderingTest(10, false); - } - - Y_UNIT_TEST(OrderingOneThreadWithSleep) { - OrderingTest(1, true); - } - - Y_UNIT_TEST(OrderingTwoThreadsWithSleep) { - OrderingTest(2, true); - } - - Y_UNIT_TEST(OrderingManyThreadsWithSleep) { - OrderingTest(10, true); - } -} + // And make sure only one request inflight (see UNIT_ASSERT on adding to the queue) + consumerQueue->Start(threads, 1); + + // Non atomic!!! Stream adaptor must protect us + ui64 curVal = 0; + + // Used just to wait in the main thread + TAtomic finished = false; + auto consumerOp = [&finished, &curVal, ptr{adaptor.get()}, queue{consumerQueue.get()}](ui64 i) { + // Check no reordering inside stream adaptor + // and no simultanious consumer Op call + UNIT_ASSERT_VALUES_EQUAL(curVal, i); + curVal++; + // We must set finished flag after last ProcessNext, but we can`t compare curVal and max after ProcessNext + // so compare here and set after + bool tmp = curVal == max; + bool res = queue->AddFunc([ptr, &finished, tmp, &curVal, i]() { + // Additional check the value still same + // run under tsan makes sure no consumer Op call before we call ProcessNext + UNIT_ASSERT_VALUES_EQUAL(curVal, i + 1); + ptr->ProcessNext(); + // Reordering after ProcessNext is possible, so check tmp and set finished to true + if (tmp) + AtomicSet(finished, true); + }); + UNIT_ASSERT(res); + }; + + TOrderedProducer producer(adaptor.get(), max, withSleep, std::move(consumerOp)); + + producer.Start(); + producer.Join(); + + while (!AtomicGet(finished)) + { + Sleep(TDuration::MilliSeconds(100)); + } + + consumerQueue->Stop(); + + UNIT_ASSERT_VALUES_EQUAL(curVal, max); + } + + Y_UNIT_TEST(OrderingOneThread) { + OrderingTest(1, false); + } + + Y_UNIT_TEST(OrderingTwoThreads) { + OrderingTest(2, false); + } + + Y_UNIT_TEST(OrderingManyThreads) { + OrderingTest(10, false); + } + + Y_UNIT_TEST(OrderingOneThreadWithSleep) { + OrderingTest(1, true); + } + + Y_UNIT_TEST(OrderingTwoThreadsWithSleep) { + OrderingTest(2, true); + } + + Y_UNIT_TEST(OrderingManyThreadsWithSleep) { + OrderingTest(10, true); + } +} diff --git a/library/cpp/grpc/server/ut/ya.make b/library/cpp/grpc/server/ut/ya.make index 2f7fa1afc2..feb3291af9 100644 --- a/library/cpp/grpc/server/ut/ya.make +++ b/library/cpp/grpc/server/ut/ya.make @@ -1,21 +1,21 @@ UNITTEST_FOR(library/cpp/grpc/server) - -OWNER( - dcherednik - g:kikimr -) - + +OWNER( + dcherednik + g:kikimr +) + TIMEOUT(600) SIZE(MEDIUM) - -PEERDIR( + +PEERDIR( library/cpp/grpc/server -) - -SRCS( +) + +SRCS( grpc_response_ut.cpp - stream_adaptor_ut.cpp -) - -END() - + stream_adaptor_ut.cpp +) + +END() + diff --git a/library/cpp/grpc/server/ya.make b/library/cpp/grpc/server/ya.make index 75499b926a..356a1b6793 100644 --- a/library/cpp/grpc/server/ya.make +++ b/library/cpp/grpc/server/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - event_callback.cpp - grpc_request.cpp - grpc_server.cpp - grpc_counters.cpp -) - -GENERATE_ENUM_SERIALIZATION(grpc_request_base.h) - -PEERDIR( - contrib/libs/grpc +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + event_callback.cpp + grpc_request.cpp + grpc_server.cpp + grpc_counters.cpp +) + +GENERATE_ENUM_SERIALIZATION(grpc_request_base.h) + +PEERDIR( + contrib/libs/grpc library/cpp/monlib/dynamic_counters/percentile -) - -END() +) + +END() RECURSE_FOR_TESTS(ut) diff --git a/util/system/hp_timer.cpp b/util/system/hp_timer.cpp index 8121e8bce0..e4c3f21e6b 100644 --- a/util/system/hp_timer.cpp +++ b/util/system/hp_timer.cpp @@ -112,7 +112,7 @@ void NHPTimer::GetTime(STime* pTime) noexcept { double NHPTimer::GetTimePassed(STime* pTime) noexcept { STime old(*pTime); - *pTime = GetCycleCount(); + *pTime = GetCycleCount(); return GetSeconds(*pTime - old); } diff --git a/util/system/thread.cpp b/util/system/thread.cpp index 1733fb2942..6236746c2d 100644 --- a/util/system/thread.cpp +++ b/util/system/thread.cpp @@ -24,19 +24,19 @@ #include <util/generic/scope.h> #else #error "FIXME" -#endif - -bool SetHighestThreadPriority() { -#ifdef _win_ - return SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); -#else - struct sched_param sch; - memset(&sch, 0, sizeof(sch)); - sch.sched_priority = 31; - return pthread_setschedparam(pthread_self(), SCHED_RR, &sch) == 0; -#endif -} - +#endif + +bool SetHighestThreadPriority() { +#ifdef _win_ + return SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); +#else + struct sched_param sch; + memset(&sch, 0, sizeof(sch)); + sch.sched_priority = 31; + return pthread_setschedparam(pthread_self(), SCHED_RR, &sch) == 0; +#endif +} + namespace { using TParams = TThread::TParams; using TId = TThread::TId; diff --git a/util/system/thread.h b/util/system/thread.h index 8dfd202895..a6e8abdb5b 100644 --- a/util/system/thread.h +++ b/util/system/thread.h @@ -10,8 +10,8 @@ #include "defaults.h" #include "progname.h" -bool SetHighestThreadPriority(); - +bool SetHighestThreadPriority(); + class TThread { template <typename Callable> struct TCallableParams; diff --git a/ydb/core/actorlib_impl/actor_tracker.cpp b/ydb/core/actorlib_impl/actor_tracker.cpp index d15c2e6aa5..99787dfce1 100644 --- a/ydb/core/actorlib_impl/actor_tracker.cpp +++ b/ydb/core/actorlib_impl/actor_tracker.cpp @@ -137,7 +137,7 @@ namespace NActors { subactor->BindToTracker(this); // we create new actor and register it in local pool - TActorId subactorId = ctx.RegisterWithSameMailbox(subactor.Release()); + TActorId subactorId = ctx.RegisterWithSameMailbox(subactor.Release()); // send TEvTrackActor message to tracker actor Y_VERIFY(ActorId); diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index 63fc3f0e1e..c666f7468c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -161,7 +161,7 @@ struct TAppData { // Used to allow column families for testing bool AllowColumnFamiliesForTest = false; - bool AllowPrivateTableDescribeForTest = false; + bool AllowPrivateTableDescribeForTest = false; // Used to allow immediate ReadTable in tests bool AllowReadTableImmediate = false; diff --git a/ydb/core/base/board_publish.cpp b/ydb/core/base/board_publish.cpp index 9fa7a969a9..cdd4bf6e48 100644 --- a/ydb/core/base/board_publish.cpp +++ b/ydb/core/base/board_publish.cpp @@ -112,7 +112,7 @@ class TBoardPublishActor : public TActorBootstrapped<TBoardPublishActor> { if (known && *known) updated[replicaId] = *known; else - updated[replicaId] = RegisterWithSameMailbox(new TBoardReplicaPublishActor(Path, Payload, replicaId, SelfId())); + updated[replicaId] = RegisterWithSameMailbox(new TBoardReplicaPublishActor(Path, Payload, replicaId, SelfId())); } ReplicaPublishActors = std::move(updated); diff --git a/ydb/core/base/cputime.h b/ydb/core/base/cputime.h index 2b4e2f6146..6661a728fa 100644 --- a/ydb/core/base/cputime.h +++ b/ydb/core/base/cputime.h @@ -1,51 +1,51 @@ -#pragma once -#include "appdata.h" - -#include <util/system/datetime.h> -#include <library/cpp/actors/core/actor.h> - -namespace NKikimr { -namespace NCpuTime { - -class TCpuTimer { - ui64 GetCpuTime() const { - if (UseClockGettime) { - return ThreadCPUTime(); - } else { - return GetCycleCountFast(); - } - } -public: - TCpuTimer() +#pragma once +#include "appdata.h" + +#include <util/system/datetime.h> +#include <library/cpp/actors/core/actor.h> + +namespace NKikimr { +namespace NCpuTime { + +class TCpuTimer { + ui64 GetCpuTime() const { + if (UseClockGettime) { + return ThreadCPUTime(); + } else { + return GetCycleCountFast(); + } + } +public: + TCpuTimer() : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) - , StartTime(GetCpuTime()) - , Counter(nullptr) - {} - - TCpuTimer(TDuration& counter) + , StartTime(GetCpuTime()) + , Counter(nullptr) + {} + + TCpuTimer(TDuration& counter) : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) - , StartTime(GetCpuTime()) - , Counter(&counter) - {} - - ~TCpuTimer() { - if (Counter) - *Counter += GetTime(); - } - - TDuration GetTime() const { - if (UseClockGettime) { - return TDuration::MicroSeconds(GetCpuTime() - StartTime); - } else { - double time = NHPTimer::GetSeconds(GetCpuTime() - StartTime); - return TDuration::MicroSeconds(time * 1000000.0); - } - } -private: - const bool UseClockGettime; - const ui64 StartTime; - TDuration* Counter; -}; - -} // namespace NCpuTime -} // namespace NKikimr + , StartTime(GetCpuTime()) + , Counter(&counter) + {} + + ~TCpuTimer() { + if (Counter) + *Counter += GetTime(); + } + + TDuration GetTime() const { + if (UseClockGettime) { + return TDuration::MicroSeconds(GetCpuTime() - StartTime); + } else { + double time = NHPTimer::GetSeconds(GetCpuTime() - StartTime); + return TDuration::MicroSeconds(time * 1000000.0); + } + } +private: + const bool UseClockGettime; + const ui64 StartTime; + TDuration* Counter; +}; + +} // namespace NCpuTime +} // namespace NKikimr diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 33280ad78a..f5fedfe19b 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -81,7 +81,7 @@ struct TKikimrEvents : TEvents { ES_RESOURCE_BROKER, ES_VIEWER, ES_SUB_DOMAIN, - ES_GRPC_PROXY_STATUS, //OLD + ES_GRPC_PROXY_STATUS, //OLD ES_SQS, ES_BLOCKSTORE, //4162 ES_RTMR_ICBUS, @@ -90,7 +90,7 @@ struct TKikimrEvents : TEvents { ES_TVM_SETTINGS_UPDATER, ES_PQ_CLUSTERS_UPDATER, ES_TENANT_SLOT_BROKER, - ES_GRPC_CALLS, + ES_GRPC_CALLS, ES_CONSOLE, ES_KESUS_PROXY, ES_KESUS, diff --git a/ydb/core/base/kikimr_issue.h b/ydb/core/base/kikimr_issue.h index 9ff826d249..fd7e232d2c 100644 --- a/ydb/core/base/kikimr_issue.h +++ b/ydb/core/base/kikimr_issue.h @@ -44,21 +44,21 @@ inline NYql::TIssue MakeIssue(NKikimrIssues::TIssuesIds::EIssueCode id) { return MakeIssue(id, IssueCodeToString(id)); } -inline TString SerializeIssues(const NYql::TIssues& in) { - NYql::TIssue rootIssue; - for(const auto& i : in) { - rootIssue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); - } - return NYql::IssueToBinaryMessage(rootIssue); +inline TString SerializeIssues(const NYql::TIssues& in) { + NYql::TIssue rootIssue; + for(const auto& i : in) { + rootIssue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); + } + return NYql::IssueToBinaryMessage(rootIssue); +} + +inline NYql::TIssues DeserializeIssues(const TString& in) { + NYql::TIssues result; + NYql::TIssue issue = NYql::IssueFromBinaryMessage(in); + for (const auto& i : issue.GetSubIssues()) { + result.AddIssue(*i); + } + return result; +} + } - -inline NYql::TIssues DeserializeIssues(const TString& in) { - NYql::TIssues result; - NYql::TIssue issue = NYql::IssueFromBinaryMessage(in); - for (const auto& i : issue.GetSubIssues()) { - result.AddIssue(*i); - } - return result; -} - -} diff --git a/ydb/core/base/kikimr_issue.txt b/ydb/core/base/kikimr_issue.txt index 5b82a49871..56c375080b 100644 --- a/ydb/core/base/kikimr_issue.txt +++ b/ydb/core/base/kikimr_issue.txt @@ -28,9 +28,9 @@ ids { code: SCOPE_TXPROXY_INTERPRET severity: S_INFO} ids { code: SCOPE_TXPROXY_RESOLVE severity: S_INFO} ids { code: SCOPE_TXPROXY_PREPARE severity: S_INFO} ids { code: SCOPE_TXPROXY_EXECUTE severity: S_INFO} - -ids { code: YDB_API_VALIDATION_ERROR severity: S_ERROR } -ids { code: YDB_AUTH_UNAVAILABLE severity: S_ERROR } -ids { code: YDB_DB_NOT_READY severity: S_ERROR } -ids { code: YDB_RESOURCE_USAGE_LIMITED severity: S_ERROR } - + +ids { code: YDB_API_VALIDATION_ERROR severity: S_ERROR } +ids { code: YDB_AUTH_UNAVAILABLE severity: S_ERROR } +ids { code: YDB_DB_NOT_READY severity: S_ERROR } +ids { code: YDB_RESOURCE_USAGE_LIMITED severity: S_ERROR } + diff --git a/ydb/core/base/statestorage_guardian.cpp b/ydb/core/base/statestorage_guardian.cpp index 22811da7d1..f1df97896b 100644 --- a/ydb/core/base/statestorage_guardian.cpp +++ b/ydb/core/base/statestorage_guardian.cpp @@ -381,7 +381,7 @@ class TTabletGuardian : public TActorBootstrapped<TTabletGuardian> { ReplicaGuardians[idx].second = TActorId(); } else { if (Info) - updatedReplicaGuardians.emplace_back(replica, RegisterWithSameMailbox(new TReplicaGuardian(Info.Get(), replica, SelfId()))); + updatedReplicaGuardians.emplace_back(replica, RegisterWithSameMailbox(new TReplicaGuardian(Info.Get(), replica, SelfId()))); else updatedReplicaGuardians.emplace_back(replica, RegisterWithSameMailbox(new TFollowerGuardian(FollowerInfo.Get(), replica, SelfId()))); } diff --git a/ydb/core/base/statestorage_proxy.cpp b/ydb/core/base/statestorage_proxy.cpp index 5d0c055d3f..13d396cb0f 100644 --- a/ydb/core/base/statestorage_proxy.cpp +++ b/ydb/core/base/statestorage_proxy.cpp @@ -976,7 +976,7 @@ public: hFunc(TEvStateStorage::TEvReplicaProbeSubscribe, Handle); hFunc(TEvStateStorage::TEvReplicaProbeUnsubscribe, Handle); default: - TActivationContext::Send(ev->Forward(RegisterWithSameMailbox(new TStateStorageProxyRequest(Info, FlowControlledInfo)))); + TActivationContext::Send(ev->Forward(RegisterWithSameMailbox(new TStateStorageProxyRequest(Info, FlowControlledInfo)))); break; } } diff --git a/ydb/core/base/table_index.cpp b/ydb/core/base/table_index.cpp index 56c0eaae68..ac89504ef1 100644 --- a/ydb/core/base/table_index.cpp +++ b/ydb/core/base/table_index.cpp @@ -16,7 +16,7 @@ TVector<TString>::const_iterator IsUniq(const TVector<TString>& names) { namespace NKikimr { namespace NTableIndex { -TTableColumns CalcTableImplDescription(const TTableColumns& table, const TIndexColumns& index) { +TTableColumns CalcTableImplDescription(const TTableColumns& table, const TIndexColumns& index) { { TString explain; Y_VERIFY(IsCompatibleIndex(table, index, explain), "explain is %s", explain.c_str()); @@ -24,26 +24,26 @@ TTableColumns CalcTableImplDescription(const TTableColumns& table, const TIndexC TTableColumns result; - for (const auto& ik: index.KeyColumns) { + for (const auto& ik: index.KeyColumns) { result.Keys.push_back(ik); result.Columns.insert(ik); } - for (const auto& tk: table.Keys) { + for (const auto& tk: table.Keys) { if (!result.Columns.contains(tk)) { result.Keys.push_back(tk); result.Columns.insert(tk); } } - for (const auto& dk: index.DataColumns) { - result.Columns.insert(dk); - } - + for (const auto& dk: index.DataColumns) { + result.Columns.insert(dk); + } + return result; } -bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, TString& explain) { +bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, TString& explain) { { auto brokenAt = IsUniq(table.Keys); if (brokenAt != table.Keys.end()) { @@ -54,27 +54,27 @@ bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, T } { - auto brokenAt = IsUniq(index.KeyColumns); - if (brokenAt != index.KeyColumns.end()) { + auto brokenAt = IsUniq(index.KeyColumns); + if (brokenAt != index.KeyColumns.end()) { explain = TStringBuilder() << "all index keys should be uniq, for example " << *brokenAt; return false; } } - { - auto brokenAt = IsUniq(index.DataColumns); - if (brokenAt != index.DataColumns.end()) { - explain = TStringBuilder() - << "all data columns should be uniq, for example " << *brokenAt; - return false; - } - } - - THashSet<TString> indexKeys; - - for (const auto& tableKeyName: table.Keys) { - indexKeys.insert(tableKeyName); + { + auto brokenAt = IsUniq(index.DataColumns); + if (brokenAt != index.DataColumns.end()) { + explain = TStringBuilder() + << "all data columns should be uniq, for example " << *brokenAt; + return false; + } + } + + THashSet<TString> indexKeys; + + for (const auto& tableKeyName: table.Keys) { + indexKeys.insert(tableKeyName); if (!table.Columns.contains(tableKeyName)) { explain = TStringBuilder() << "all table keys should be in table columns too" @@ -83,8 +83,8 @@ bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, T } } - for (const auto& indexKeyName: index.KeyColumns) { - indexKeys.insert(indexKeyName); + for (const auto& indexKeyName: index.KeyColumns) { + indexKeys.insert(indexKeyName); if (!table.Columns.contains(indexKeyName)) { explain = TStringBuilder() << "all index keys should be in table columns" @@ -93,26 +93,26 @@ bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, T } } - if (index.KeyColumns == table.Keys) { + if (index.KeyColumns == table.Keys) { explain = TStringBuilder() << "table and index keys are the same"; return false; } - for (const auto& dataName: index.DataColumns) { - if (indexKeys.contains(dataName)) { - explain = TStringBuilder() - << "The same column can't be used as key column and data column for one index"; - return false; - } - if (!table.Columns.contains(dataName)) { - explain = TStringBuilder() - << "all index data columns should be in table columns" - << ", data columns " << dataName << " is missed"; - return false; - } - } - + for (const auto& dataName: index.DataColumns) { + if (indexKeys.contains(dataName)) { + explain = TStringBuilder() + << "The same column can't be used as key column and data column for one index"; + return false; + } + if (!table.Columns.contains(dataName)) { + explain = TStringBuilder() + << "all index data columns should be in table columns" + << ", data columns " << dataName << " is missed"; + return false; + } + } + return true; } diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h index 8dc8000f2f..ea1328cdb6 100644 --- a/ydb/core/base/table_index.h +++ b/ydb/core/base/table_index.h @@ -13,13 +13,13 @@ struct TTableColumns { TVector<TString> Keys; }; -struct TIndexColumns { - TVector<TString> KeyColumns; - TVector<TString> DataColumns; -}; +struct TIndexColumns { + TVector<TString> KeyColumns; + TVector<TString> DataColumns; +}; -bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, TString& explain); -TTableColumns CalcTableImplDescription(const TTableColumns& table, const TIndexColumns &index); +bool IsCompatibleIndex(const TTableColumns& table, const TIndexColumns& index, TString& explain); +TTableColumns CalcTableImplDescription(const TTableColumns& table, const TIndexColumns &index); } } diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp index 7a2e806abf..b73da3df8d 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp @@ -936,7 +936,7 @@ class TWaitForCompaction : public TActorBootstrapped<TWaitForCompaction> { ui32 total = Conf->GroupInfo->GetTotalVDisksNum(); for (ui32 i = 0; i < total; i++) { TAllVDisks::TVDiskInstance &instance = Conf->VDisks->Get(i); - ctx.RegisterWithSameMailbox(CreateWaitForCompaction(ctx.SelfID, instance)); + ctx.RegisterWithSameMailbox(CreateWaitForCompaction(ctx.SelfID, instance)); Counter++; } } diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_simplebs.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_simplebs.cpp index bda0b9197b..583b2beb1b 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/test_simplebs.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/test_simplebs.cpp @@ -42,7 +42,7 @@ private: if (finished) { if (WaitForCompaction) { Become(&TThis::StateFuncWait); - ctx.RegisterWithSameMailbox(CreateWaitForCompaction(ctx.SelfID, VDiskInfo)); + ctx.RegisterWithSameMailbox(CreateWaitForCompaction(ctx.SelfID, VDiskInfo)); } else { HandleWaitDone(ctx); } diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp index 7d00519df1..33a8e1153b 100644 --- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp +++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp @@ -115,26 +115,26 @@ namespace NKikimr { TActiveActors activeActors; // actor for LogCutter Notifier - auto logNotifierAid = ctx.RegisterWithSameMailbox(CreateHullLogCutterNotifier(hullLogCtx->VCtx, logCutterId, HullDs)); + auto logNotifierAid = ctx.RegisterWithSameMailbox(CreateHullLogCutterNotifier(hullLogCtx->VCtx, logCutterId, HullDs)); activeActors.Insert(logNotifierAid); Fields->SetLogNotifierActorId(logNotifierAid); // actor for LogoBlobs DB - HullDs->LogoBlobs->LIActor = ctx.RegisterWithSameMailbox(CreateLogoBlobsActor(config, HullDs, hullLogCtx, loggerId, + HullDs->LogoBlobs->LIActor = ctx.RegisterWithSameMailbox(CreateLogoBlobsActor(config, HullDs, hullLogCtx, loggerId, Fields->LogoBlobsRunTimeCtx, syncLogFirstLsnToKeep)); activeActors.Insert(HullDs->LogoBlobs->LIActor); // actor for Blocks DB - HullDs->Blocks->LIActor = ctx.RegisterWithSameMailbox(CreateBlocksActor(config, HullDs, hullLogCtx, loggerId, + HullDs->Blocks->LIActor = ctx.RegisterWithSameMailbox(CreateBlocksActor(config, HullDs, hullLogCtx, loggerId, Fields->BlocksRunTimeCtx, syncLogFirstLsnToKeep)); activeActors.Insert(HullDs->Blocks->LIActor); // actor for Barriers DB - HullDs->Barriers->LIActor = ctx.RegisterWithSameMailbox(CreateBarriersActor(config, HullDs, hullLogCtx, loggerId, + HullDs->Barriers->LIActor = ctx.RegisterWithSameMailbox(CreateBarriersActor(config, HullDs, hullLogCtx, loggerId, Fields->BarriersRunTimeCtx, syncLogFirstLsnToKeep)); activeActors.Insert(HullDs->Barriers->LIActor); // create delayed huge blob deleter actor only for LogoBlobs level index as huge blobs are only possible // for that data auto& deleterInfo = HullDs->LogoBlobs->DelayedHugeBlobDeleterInfo; - auto hugeBlobDeleterAid = ctx.RegisterWithSameMailbox(CreateDelayedHugeBlobDeleterActor(hullLogCtx->HugeKeeperId, + auto hugeBlobDeleterAid = ctx.RegisterWithSameMailbox(CreateDelayedHugeBlobDeleterActor(hullLogCtx->HugeKeeperId, deleterInfo)); deleterInfo->SetActorId(hugeBlobDeleterAid); activeActors.Insert(hugeBlobDeleterAid); diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp index d99976bd83..e0118b3acf 100644 --- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp +++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp @@ -566,7 +566,7 @@ namespace NKikimr { << " entryPoint# "<< entryPoint << " freeUpToLsn# " << freeUpToLsn << "}"; - auto aid = ctx.RegisterWithSameMailbox(new TAsyncAdvanceLsnCommitter(HullLogCtx, HullDbCommitterCtx, + auto aid = ctx.RegisterWithSameMailbox(new TAsyncAdvanceLsnCommitter(HullLogCtx, HullDbCommitterCtx, RTCtx->LevelIndex, ctx.SelfID, dbg.Str())); ActiveActors.Insert(aid); AdvanceCommitInProgress = true; diff --git a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp index 8932b5ae44..b9bdafdbb1 100644 --- a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp +++ b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp @@ -204,7 +204,7 @@ namespace NKikimr { } void BeginApplyingLog(const TActorContext& ctx) { - auto replayerId = ctx.RegisterWithSameMailbox(CreateRecoveryLogReplayer(ctx.SelfID, LocRecCtx)); + auto replayerId = ctx.RegisterWithSameMailbox(CreateRecoveryLogReplayer(ctx.SelfID, LocRecCtx)); ActiveActors.Insert(replayerId); Become(&TThis::StateApplyRecoveryLog); VDiskMonGroup.VDiskLocalRecoveryState() = TDbMon::TDbLocalRecovery::ApplyLog; diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp index f4d4dfe12a..96b813d317 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp @@ -1679,7 +1679,7 @@ namespace NKikimr { // run LogCutter in the same mailbox TLogCutterCtx logCutterCtx = {VCtx, PDiskCtx, Db->LsnMngr, Config, (TActorId)(Db->LoggerID)}; - Db->LogCutterID.Set(ctx.RegisterWithSameMailbox(CreateRecoveryLogCutter(std::move(logCutterCtx)))); + Db->LogCutterID.Set(ctx.RegisterWithSameMailbox(CreateRecoveryLogCutter(std::move(logCutterCtx)))); ActiveActors.Insert(Db->LogCutterID); // keep forever // run HugeBlobKeeper diff --git a/ydb/core/client/flat_ut.cpp b/ydb/core/client/flat_ut.cpp index 1996ac17c3..362c052aa1 100644 --- a/ydb/core/client/flat_ut.cpp +++ b/ydb/core/client/flat_ut.cpp @@ -622,277 +622,277 @@ Y_UNIT_TEST_SUITE(TFlatTest) { UNIT_ASSERT_VALUES_EQUAL(s3, 0); } - Y_UNIT_TEST(ShardFreezeRejectBadProtobuf) { - TPortManager pm; - ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port)); - if (!true) { - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); - } - - TFlatMsgBusClient annoyingClient(port); - - const char * table = R"(Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Uint32" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2)"; - - annoyingClient.InitRoot(); - annoyingClient.CreateTable("/dc-1", table); - - { - // Alter and freeze - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" + Y_UNIT_TEST(ShardFreezeRejectBadProtobuf) { + TPortManager pm; + ui16 port = pm.GetPort(2134); + TServer cleverServer = TServer(TServerSettings(port)); + if (!true) { + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); + } + + TFlatMsgBusClient annoyingClient(port); + + const char * table = R"(Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uint32" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2)"; + + annoyingClient.InitRoot(); + annoyingClient.CreateTable("/dc-1", table); + + { + // Alter and freeze + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" PartitionConfig { FreezeState: Freeze FollowerCount: 1 } - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unspecified} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - DropColumns { Name: "value" } - PartitionConfig { FreezeState: Unspecified } - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - DropColumns { Name: "value" } - PartitionConfig { FreezeState: Unfreeze } - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); - } - } - - Y_UNIT_TEST(ShardUnfreezeNonFrozen) { - TPortManager pm; - ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port)); - if (!true) { - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); - } - - TFlatMsgBusClient annoyingClient(port); - - const char * table = R"(Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Uint32" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2)"; - - annoyingClient.InitRoot(); - annoyingClient.CreateTable("/dc-1", table); - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unfreeze} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - } - - Y_UNIT_TEST(ShardFreezeUnfreezeAlreadySet) { - TPortManager pm; - ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port)); - if (!true) { - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); - } - - TFlatMsgBusClient annoyingClient(port); - - const char * table = R"(Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Uint32" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2)"; - - annoyingClient.InitRoot(); - annoyingClient.CreateTable("/dc-1", table); - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Freeze} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Freeze} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unfreeze} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unfreeze} - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - } - - Y_UNIT_TEST(ShardFreezeUnfreezeRejectScheme) { - TPortManager pm; - ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port)); - if (!true) { - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); - } - - TFlatMsgBusClient annoyingClient(port); - - const char * table = R"(Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Uint32" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2)"; - - annoyingClient.InitRoot(); - annoyingClient.CreateTable("/dc-1", table); - - // Alter, freeze datashard - annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Freeze } - )"); - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - DropColumns { Name: "value" } - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); - } - - annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unfreeze } - )"); - { - auto res = annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - DropColumns { Name: "value" } - )"); - UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); - } - } - - Y_UNIT_TEST(ShardFreezeUnfreeze) { - TPortManager pm; - ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port)); - if (!true) { - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); - cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); - } - - TFlatMsgBusClient annoyingClient(port); - - const char * table = R"(Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Uint32" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2)"; - - annoyingClient.InitRoot(); - annoyingClient.CreateTable("/dc-1", table); - - // Alter, freeze datashard - annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Freeze } - )"); - { - // write should be rejected - TString insertRowQuery = "(" - "(let key '('('key (Uint32 '%u))))" - "(let value '('('value (Uint32 '%u))))" - "(let ret_ (AsList" - " (UpdateRow '/dc-1/Table key value)" - "))" - "(return ret_)" - ")"; - - TClient::TFlatQueryOptions opts; - NKikimrClient::TResponse response; - - ui32 status = annoyingClient.FlatQueryRaw(Sprintf(insertRowQuery.data(), 1, 6000000).data(), opts, response); - UNIT_ASSERT(status == NMsgBusProxy::MSTATUS_REJECTED); - - Cout << "SELECT value FROM Table WHERE key = 0" << Endl; - annoyingClient.FlatQuery(R"(( - (let row_ '('('key (Uint32 '0)))) - (let cols_ '('value)) - (let select_ (SelectRow '/dc-1/Table row_ cols_)) - (let ret_ (AsList (SetResult 'ret0 select_))) - (return ret_) - ))", NMsgBusProxy::MSTATUS_OK); - } - - annoyingClient.AlterTable("/dc-1", R"( - Name: "Table" - PartitionConfig { FreezeState: Unfreeze } - )"); - - { - - TString insertRowQuery = "(" - "(let key '('('key (Uint32 '%u))))" - "(let value '('('value (Uint32 '%u))))" - "(let ret_ (AsList" - " (UpdateRow '/dc-1/Table key value)" - "))" - "(return ret_)" - ")"; - - TClient::TFlatQueryOptions opts; - NKikimrClient::TResponse response; - - ui32 status = annoyingClient.FlatQueryRaw(Sprintf(insertRowQuery.data(), 1, 6000000).data(), opts, response); - - UNIT_ASSERT(status == NMsgBusProxy::MSTATUS_OK); - } - } - + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unspecified} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + DropColumns { Name: "value" } + PartitionConfig { FreezeState: Unspecified } + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + DropColumns { Name: "value" } + PartitionConfig { FreezeState: Unfreeze } + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); + } + } + + Y_UNIT_TEST(ShardUnfreezeNonFrozen) { + TPortManager pm; + ui16 port = pm.GetPort(2134); + TServer cleverServer = TServer(TServerSettings(port)); + if (!true) { + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); + } + + TFlatMsgBusClient annoyingClient(port); + + const char * table = R"(Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uint32" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2)"; + + annoyingClient.InitRoot(); + annoyingClient.CreateTable("/dc-1", table); + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unfreeze} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + } + + Y_UNIT_TEST(ShardFreezeUnfreezeAlreadySet) { + TPortManager pm; + ui16 port = pm.GetPort(2134); + TServer cleverServer = TServer(TServerSettings(port)); + if (!true) { + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); + } + + TFlatMsgBusClient annoyingClient(port); + + const char * table = R"(Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uint32" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2)"; + + annoyingClient.InitRoot(); + annoyingClient.CreateTable("/dc-1", table); + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Freeze} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Freeze} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unfreeze} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unfreeze} + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + } + + Y_UNIT_TEST(ShardFreezeUnfreezeRejectScheme) { + TPortManager pm; + ui16 port = pm.GetPort(2134); + TServer cleverServer = TServer(TServerSettings(port)); + if (!true) { + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); + } + + TFlatMsgBusClient annoyingClient(port); + + const char * table = R"(Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uint32" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2)"; + + annoyingClient.InitRoot(); + annoyingClient.CreateTable("/dc-1", table); + + // Alter, freeze datashard + annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Freeze } + )"); + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + DropColumns { Name: "value" } + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_ERROR); + } + + annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unfreeze } + )"); + { + auto res = annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + DropColumns { Name: "value" } + )"); + UNIT_ASSERT_VALUES_EQUAL(res, NMsgBusProxy::MSTATUS_OK); + } + } + + Y_UNIT_TEST(ShardFreezeUnfreeze) { + TPortManager pm; + ui16 port = pm.GetPort(2134); + TServer cleverServer = TServer(TServerSettings(port)); + if (!true) { + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); + cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_MAIN, NActors::NLog::PRI_DEBUG); + } + + TFlatMsgBusClient annoyingClient(port); + + const char * table = R"(Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Uint32" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2)"; + + annoyingClient.InitRoot(); + annoyingClient.CreateTable("/dc-1", table); + + // Alter, freeze datashard + annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Freeze } + )"); + { + // write should be rejected + TString insertRowQuery = "(" + "(let key '('('key (Uint32 '%u))))" + "(let value '('('value (Uint32 '%u))))" + "(let ret_ (AsList" + " (UpdateRow '/dc-1/Table key value)" + "))" + "(return ret_)" + ")"; + + TClient::TFlatQueryOptions opts; + NKikimrClient::TResponse response; + + ui32 status = annoyingClient.FlatQueryRaw(Sprintf(insertRowQuery.data(), 1, 6000000).data(), opts, response); + UNIT_ASSERT(status == NMsgBusProxy::MSTATUS_REJECTED); + + Cout << "SELECT value FROM Table WHERE key = 0" << Endl; + annoyingClient.FlatQuery(R"(( + (let row_ '('('key (Uint32 '0)))) + (let cols_ '('value)) + (let select_ (SelectRow '/dc-1/Table row_ cols_)) + (let ret_ (AsList (SetResult 'ret0 select_))) + (return ret_) + ))", NMsgBusProxy::MSTATUS_OK); + } + + annoyingClient.AlterTable("/dc-1", R"( + Name: "Table" + PartitionConfig { FreezeState: Unfreeze } + )"); + + { + + TString insertRowQuery = "(" + "(let key '('('key (Uint32 '%u))))" + "(let value '('('value (Uint32 '%u))))" + "(let ret_ (AsList" + " (UpdateRow '/dc-1/Table key value)" + "))" + "(return ret_)" + ")"; + + TClient::TFlatQueryOptions opts; + NKikimrClient::TResponse response; + + ui32 status = annoyingClient.FlatQueryRaw(Sprintf(insertRowQuery.data(), 1, 6000000).data(), opts, response); + + UNIT_ASSERT(status == NMsgBusProxy::MSTATUS_OK); + } + } + Y_UNIT_TEST(Mix_DML_DDL) { TPortManager pm; ui16 port = pm.GetPort(2134); @@ -1422,8 +1422,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) { TEvTxUserProxy::TResultStatus::AccessDenied); // as argonaut@ annoyingClient.SetSecurityToken("berkanavt@" BUILTIN_ACL_DOMAIN); // as berkanavt@ - NACLib::TDiffACL newAcl; - newAcl.ClearAccess(); + NACLib::TDiffACL newAcl; + newAcl.ClearAccess(); newAcl.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericWrite, "argonaut@" BUILTIN_ACL_DOMAIN); annoyingClient.ModifyACL("/dc-1", "Berkanavt", newAcl.SerializeAsString()); // as berkanavt@ annoyingClient.ResetSchemeCache(cleverServer, tabletId); @@ -3780,7 +3780,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { } } - static NKikimrMiniKQL::TResult CreateTableAndExecuteMkql(const TString& mkql) { + static NKikimrMiniKQL::TResult CreateTableAndExecuteMkql(const TString& mkql) { TPortManager pm; ui16 port = pm.GetPort(2134); TServer cleverServer = TServer(TServerSettings(port)); @@ -3823,11 +3823,11 @@ Y_UNIT_TEST_SUITE(TFlatTest) { fillValues(2, {}, "Four"); fillValues(2, TString("k1"), "Five"); - return annoyingClient.FlatQuery(mkql); - } - - Y_UNIT_TEST(SelectRangeSkipNullKeys) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( + return annoyingClient.FlatQuery(mkql); + } + + Y_UNIT_TEST(SelectRangeSkipNullKeys) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( ( (let range '('ExcFrom 'ExcTo '('Key1 (Null) (Void)) '('Key2 (Null) (Void)))) (let data (SelectRange @@ -3851,147 +3851,147 @@ Y_UNIT_TEST_SUITE(TFlatTest) { UNIT_ASSERT_VALUES_EQUAL((TString)list[1]["Value"], "Five"); } - Y_UNIT_TEST(SelectRangeForbidNullArgs1) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Null) (Void)) '('Key2 (Null) (Void)))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - '('"ForbidNullArgsFrom" '('Key2)) - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); - } - - Y_UNIT_TEST(SelectRangeForbidNullArgs2) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Null) (Void)) '('Key2 (Null) (Void)))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - '('"ForbidNullArgsFrom" '('Key1)) - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); - } - - Y_UNIT_TEST(SelectRangeNullArgs3) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Void)))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 2); - UNIT_ASSERT_VALUES_EQUAL((TString)list[0]["Value"], "Two"); - UNIT_ASSERT_VALUES_EQUAL((TString)list[1]["Value"], "Three"); - } - - Y_UNIT_TEST(SelectRangeForbidNullArgs3) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Void)))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - '('"ForbidNullArgsFrom" '('Key2)) - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); - } - - Y_UNIT_TEST(SelectRangeNullArgs4) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Null) ))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 1); - UNIT_ASSERT_VALUES_EQUAL((TString)list[0]["Value"], "Two"); - } - - Y_UNIT_TEST(SelectRangeForbidNullArgs4) { - auto res = CreateTableAndExecuteMkql(Sprintf(R"( - ( - (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Null) ))) - (let data (SelectRange - '"/dc-1/test/TestTable" - range - '('Value) - '( - '('"ForbidNullArgsTo" '('Key2)) - ) - )) - (return (AsList (SetResult 'Result data))) - ) - )" - )); - - TValue value = TValue::Create(res.GetValue(), res.GetType()); - TValue result = value["Result"]; - auto list = result["List"]; - UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); - } - + Y_UNIT_TEST(SelectRangeForbidNullArgs1) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Null) (Void)) '('Key2 (Null) (Void)))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + '('"ForbidNullArgsFrom" '('Key2)) + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); + } + + Y_UNIT_TEST(SelectRangeForbidNullArgs2) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Null) (Void)) '('Key2 (Null) (Void)))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + '('"ForbidNullArgsFrom" '('Key1)) + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); + } + + Y_UNIT_TEST(SelectRangeNullArgs3) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Void)))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 2); + UNIT_ASSERT_VALUES_EQUAL((TString)list[0]["Value"], "Two"); + UNIT_ASSERT_VALUES_EQUAL((TString)list[1]["Value"], "Three"); + } + + Y_UNIT_TEST(SelectRangeForbidNullArgs3) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Void)))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + '('"ForbidNullArgsFrom" '('Key2)) + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); + } + + Y_UNIT_TEST(SelectRangeNullArgs4) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Null) ))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 1); + UNIT_ASSERT_VALUES_EQUAL((TString)list[0]["Value"], "Two"); + } + + Y_UNIT_TEST(SelectRangeForbidNullArgs4) { + auto res = CreateTableAndExecuteMkql(Sprintf(R"( + ( + (let range '('IncFrom 'IncTo '('Key1 (Uint64 '1) (Uint64 '1)) '('Key2 (Null) (Null) ))) + (let data (SelectRange + '"/dc-1/test/TestTable" + range + '('Value) + '( + '('"ForbidNullArgsTo" '('Key2)) + ) + )) + (return (AsList (SetResult 'Result data))) + ) + )" + )); + + TValue value = TValue::Create(res.GetValue(), res.GetType()); + TValue result = value["Result"]; + auto list = result["List"]; + UNIT_ASSERT_VALUES_EQUAL(list.Size(), 0); + } + Y_UNIT_TEST(SelectRangeReverse) { const ui32 TABLE_ROWS = 10; diff --git a/ydb/core/client/metadata/functions_metadata.h b/ydb/core/client/metadata/functions_metadata.h index e10aabed98..a9a17dd2eb 100644 --- a/ydb/core/client/metadata/functions_metadata.h +++ b/ydb/core/client/metadata/functions_metadata.h @@ -4,10 +4,10 @@ namespace NKikimr { -namespace NMiniKQL { - class IBuiltinFunctionRegistry; -} - +namespace NMiniKQL { + class IBuiltinFunctionRegistry; +} + void SerializeMetadata(const NMiniKQL::IBuiltinFunctionRegistry& funcRegistry, TString* out); void DeserializeMetadata(TStringBuf buffer, NMiniKQL::IBuiltinFunctionRegistry& funcRegistry); diff --git a/ydb/core/client/minikql_compile/compile_context.cpp b/ydb/core/client/minikql_compile/compile_context.cpp index 2e7c008c3b..44526ee5aa 100644 --- a/ydb/core/client/minikql_compile/compile_context.cpp +++ b/ydb/core/client/minikql_compile/compile_context.cpp @@ -38,8 +38,8 @@ void TContext::AddTableLookup(const IDbSchemeResolver::TTable& request) { } } -template<typename TStringType> -IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TStringType& tableName) { +template<typename TStringType> +IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TStringType& tableName) { auto entry = Tables.FindPtr(tableName); if (!entry) { ythrow TNodeException(node) << "Table is not found: " << tableName; @@ -52,11 +52,11 @@ IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, return entry->Response.Get(); } -template -typename IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TString& tableName); -template -typename IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TStringBuf& tableName); - +template +typename IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TString& tableName); +template +typename IDbSchemeResolver::TTableResult* TContext::GetTableLookup(const TExprNode& node, const TStringBuf& tableName); + TContext::TTableMap& TContext::GetTablesToResolve() { return Tables; } diff --git a/ydb/core/client/minikql_compile/compile_context.h b/ydb/core/client/minikql_compile/compile_context.h index 5b736fab66..22a9ebdf37 100644 --- a/ydb/core/client/minikql_compile/compile_context.h +++ b/ydb/core/client/minikql_compile/compile_context.h @@ -40,8 +40,8 @@ struct TContext : public TAtomicRefCount<TContext> { TRuntimeNode NewParam(TStringBuf name, TType* type); void AddTableLookup(const IDbSchemeResolver::TTable& request); - template<typename TStringType> - IDbSchemeResolver::TTableResult* GetTableLookup(const TExprNode& node, const TStringType& tableName); + template<typename TStringType> + IDbSchemeResolver::TTableResult* GetTableLookup(const TExprNode& node, const TStringType& tableName); TTableMap& GetTablesToResolve(); TConvertResult Finish(TRuntimeNode convertedNode); @@ -52,8 +52,8 @@ public: NKikimr::NUdf::ITypeInfoHelper::TPtr TypeInfoHelper; TAutoPtr<TKikimrProgramBuilder> PgmBuilder; TMaybe<TParametersBuilder> ParamsBuilder; - -private: + +private: bool WasParams; TTableMap Tables; }; diff --git a/ydb/core/client/minikql_compile/compile_result.cpp b/ydb/core/client/minikql_compile/compile_result.cpp index caafb2567f..9891b7fc6f 100644 --- a/ydb/core/client/minikql_compile/compile_result.cpp +++ b/ydb/core/client/minikql_compile/compile_result.cpp @@ -3,11 +3,11 @@ namespace NYql { -TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssue& error) { +TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssue& error) { Errors.AddIssue(error); } -TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssues& errors) +TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssues& errors) : Errors(errors) { } diff --git a/ydb/core/client/minikql_compile/compile_result.h b/ydb/core/client/minikql_compile/compile_result.h index d1fa4bf789..d92fd02603 100644 --- a/ydb/core/client/minikql_compile/compile_result.h +++ b/ydb/core/client/minikql_compile/compile_result.h @@ -9,16 +9,16 @@ using NKikimr::NMiniKQL::TRuntimeNode; struct TConvertResult { TRuntimeNode Node; - TIssues Errors; + TIssues Errors; }; struct TMiniKQLCompileResult { TMiniKQLCompileResult() = default; - explicit TMiniKQLCompileResult(const TIssue& error); - explicit TMiniKQLCompileResult(const TIssues& errors); + explicit TMiniKQLCompileResult(const TIssue& error); + explicit TMiniKQLCompileResult(const TIssues& errors); explicit TMiniKQLCompileResult(const TString& compiledProgram); - TIssues Errors; + TIssues Errors; TString CompiledProgram; }; diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index 3c1a2eae39..76f38bf35c 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -92,33 +92,33 @@ EInplaceUpdateMode ParseUpdateMode(const TStringBuf& name) { return mode; } -bool CheckVersionedTable(const TExprNode* list) { - if (list->ChildrenSize() == 3) { - const auto name = list->Child(0); - const auto version = list->Child(1); - const auto pathId = list->Child(2); - // TODO: KIKIMR-8446 add check for empty version - if (name->IsAtom() && version->IsAtom() && name->Content() && pathId->IsAtom()) { - return true; - } - } - return false; -} - -TStringBuf GetTableName(const TExprNode* node) { - if (node->Child(0)->IsList()) { - const auto list = node->Child(0); - Y_ENSURE_EX(CheckVersionedTable(list), TNodeException(list) - << "Expected list of 3 atoms as versioned table."); - return list->Child(0)->Content(); - } else if (node->Child(0)->IsAtom()) { - return node->Child(0)->Content(); - } else { - Y_ENSURE_EX(false, TNodeException(node) << "unexpected table node"); - } - return {}; -} - +bool CheckVersionedTable(const TExprNode* list) { + if (list->ChildrenSize() == 3) { + const auto name = list->Child(0); + const auto version = list->Child(1); + const auto pathId = list->Child(2); + // TODO: KIKIMR-8446 add check for empty version + if (name->IsAtom() && version->IsAtom() && name->Content() && pathId->IsAtom()) { + return true; + } + } + return false; +} + +TStringBuf GetTableName(const TExprNode* node) { + if (node->Child(0)->IsList()) { + const auto list = node->Child(0); + Y_ENSURE_EX(CheckVersionedTable(list), TNodeException(list) + << "Expected list of 3 atoms as versioned table."); + return list->Child(0)->Content(); + } else if (node->Child(0)->IsAtom()) { + return node->Child(0)->Content(); + } else { + Y_ENSURE_EX(false, TNodeException(node) << "unexpected table node"); + } + return {}; +} + const TTypeAnnotationNode* GetMkqlDataTypeAnnotation(TDataType* dataType, TExprContext& ctx) { return ctx.MakeType<TDataExprType>(*dataType->GetDataSlot()); @@ -127,10 +127,10 @@ const TTypeAnnotationNode* GetMkqlDataTypeAnnotation(TDataType* dataType, TExprC void CollectEraseRowKey(const TExprNode* child, TContext::TPtr ctx) { IDbSchemeResolver::TTable request; - Y_ENSURE_EX(child->ChildrenSize() == 2, TNodeException(child) << child->Content() - << " takes 2 args."); - request.TableName = GetTableName(child); - + Y_ENSURE_EX(child->ChildrenSize() == 2, TNodeException(child) << child->Content() + << " takes 2 args."); + request.TableName = GetTableName(child); + auto rowTuple = child->Child(1); Y_ENSURE_EX(rowTuple->IsList() && rowTuple->ChildrenSize() > 0, TNodeException(rowTuple) << child->Content() << "Expected non-empty tuple"); @@ -140,8 +140,8 @@ void CollectEraseRowKey(const TExprNode* child, TContext::TPtr ctx) { TNodeException(*tupleItem) << child->Content() << "Expected pair"); Y_ENSURE_EX(tupleItem->Child(0)->IsAtom() && - !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) - << "Expected column name as non-empty atom."); + !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) + << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); } @@ -151,10 +151,10 @@ void CollectEraseRowKey(const TExprNode* child, TContext::TPtr ctx) { void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { IDbSchemeResolver::TTable request; - Y_ENSURE_EX(child->ChildrenSize() == 3, TNodeException(child) << child->Content() - << " takes 3 args."); - - request.TableName = GetTableName(child); + Y_ENSURE_EX(child->ChildrenSize() == 3, TNodeException(child) << child->Content() + << " takes 3 args."); + + request.TableName = GetTableName(child); auto rowTuple = child->Child(1); Y_ENSURE_EX(rowTuple->IsList() && rowTuple->ChildrenSize() > 0, TNodeException(rowTuple) << child->Content() << "Expected non-empty tuple"); @@ -164,8 +164,8 @@ void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { TNodeException(*tupleItem) << child->Content() << "Expected pair"); Y_ENSURE_EX(tupleItem->Child(0)->IsAtom() && - !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) - << "Expected column name as non-empty atom."); + !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) + << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); } @@ -176,8 +176,8 @@ void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { Y_ENSURE_EX(tupleItem->IsList() && tupleItem->ChildrenSize() >= 1 && tupleItem->ChildrenSize() <= 3, TNodeException(*tupleItem) << child->Content() << "Expected tuple of size 1..3"); Y_ENSURE_EX(tupleItem->Child(0)->IsAtom() && - !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) - << "Expected column name as non-empty atom."); + !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) + << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); } @@ -189,8 +189,8 @@ void CollectSelectRowKey(const TExprNode* child, TContext::TPtr ctx) { Y_ENSURE_EX(child->ChildrenSize() == 3 || child->ChildrenSize() == 4, TNodeException(child) << child->Content() << " takes 3 or 4 args."); - - request.TableName = GetTableName(child); + + request.TableName = GetTableName(child); auto rowTuple = child->Child(1); Y_ENSURE_EX(rowTuple->IsList() && rowTuple->ChildrenSize() > 0, TNodeException(rowTuple) << child->Content() << "Expected non-empty tuple"); @@ -200,8 +200,8 @@ void CollectSelectRowKey(const TExprNode* child, TContext::TPtr ctx) { TNodeException(*tupleItem) << child->Content() << "Expected pair"); Y_ENSURE_EX(tupleItem->Child(0)->IsAtom() && - !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) - << "Expected column name as non-empty atom."); + !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) + << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); } @@ -222,8 +222,8 @@ void CollectSelectRangeKey(const TExprNode* child, TContext::TPtr ctx) { Y_ENSURE_EX(child->ChildrenSize() == 4 || child->ChildrenSize() == 5, TNodeException(child) << child->Content() << " takes 4 or 5 args."); - - request.TableName = GetTableName(child); + + request.TableName = GetTableName(child); auto rangeTuple = child->Child(1); Y_ENSURE_EX(rangeTuple->IsList() && rangeTuple->ChildrenSize() > 0, TNodeException(rangeTuple) << child->Content() << "Expected non-empty tuple"); @@ -334,44 +334,44 @@ public: } private: - static bool CheckKeyColumn(const TStringBuf& columnName, ui32 keyIndex, IDbSchemeResolver::TTableResult* lookup, TExprNode& node, TExprContext& ctx) { + static bool CheckKeyColumn(const TStringBuf& columnName, ui32 keyIndex, IDbSchemeResolver::TTableResult* lookup, TExprNode& node, TExprContext& ctx) { auto column = lookup->Columns.FindPtr(columnName); - if (!column) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() - << "Unknown column '" << columnName - << "' for table [" << lookup->Table.TableName - << "] at key position #" << keyIndex)); - return false; - } - - if ((ui32)column->KeyPosition != keyIndex) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() - << "Mismatched key column '" << columnName - << "' for table [" << lookup->Table.TableName - << "] at key position #" << keyIndex)); - return false; - } - - return true; + if (!column) { + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() + << "Unknown column '" << columnName + << "' for table [" << lookup->Table.TableName + << "] at key position #" << keyIndex)); + return false; + } + + if ((ui32)column->KeyPosition != keyIndex) { + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() + << "Mismatched key column '" << columnName + << "' for table [" << lookup->Table.TableName + << "] at key position #" << keyIndex)); + return false; + } + + return true; } - static bool CheckRowTuple(IDbSchemeResolver::TTableResult* lookup, TExprNode& node, TExprNode& rowTuple, TExprContext& ctx) { - if (rowTuple.ChildrenSize() != lookup->KeyColumnCount) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() - << "Mismatch of key columns count for table [" << lookup->Table.TableName - << "], expected: " << lookup->KeyColumnCount - << ", but got " << rowTuple.ChildrenSize() << ".")); - return false; - } + static bool CheckRowTuple(IDbSchemeResolver::TTableResult* lookup, TExprNode& node, TExprNode& rowTuple, TExprContext& ctx) { + if (rowTuple.ChildrenSize() != lookup->KeyColumnCount) { + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() + << "Mismatch of key columns count for table [" << lookup->Table.TableName + << "], expected: " << lookup->KeyColumnCount + << ", but got " << rowTuple.ChildrenSize() << ".")); + return false; + } for (ui32 i = 0; i < rowTuple.ChildrenSize(); ++i) { auto columnName = rowTuple.Child(i)->Child(0)->Content(); - if (!CheckKeyColumn(columnName, i, lookup, node, ctx)) { - return false; - } + if (!CheckKeyColumn(columnName, i, lookup, node, ctx)) { + return false; + } } - - return true; + + return true; } const TTypeAnnotationNode* GetSelectType(IDbSchemeResolver::TTableResult* lookup, TExprNode& selectTuple, @@ -413,12 +413,12 @@ private: private: IGraphTransformer::TStatus SelectRowWrapper(TExprNode& node, TExprContext& ctx) { - const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); + const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rowTuple = node.Child(1); - if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { - return TStatus::Error; - } + if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { + return TStatus::Error; + } auto selectTuple = node.Child(2); auto selectType = GetSelectType(lookup, *selectTuple, ctx); @@ -429,7 +429,7 @@ private: } IGraphTransformer::TStatus SelectRangeWrapper(TExprNode& node, TExprContext& ctx) { - const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); + const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rangeTuple = node.Child(1); ui32 keyCount = 0; @@ -491,9 +491,9 @@ private: } auto columnName = rangeItem->Child(0)->Content(); - if (!CheckKeyColumn(columnName, keyIndex, lookup, node, ctx)) { - return TStatus::Error; - } + if (!CheckKeyColumn(columnName, keyIndex, lookup, node, ctx)) { + return TStatus::Error; + } ++keyIndex; } @@ -524,9 +524,9 @@ private: if (optionName != "ItemsLimit" && optionName != "BytesLimit" && optionName != "SkipNullKeys" && - optionName != "Reverse" && - optionName != "ForbidNullArgsFrom" && - optionName != "ForbidNullArgsTo") + optionName != "Reverse" && + optionName != "ForbidNullArgsFrom" && + optionName != "ForbidNullArgsTo") { ythrow TNodeException(node) << "Unsupported option: " << optionName; } @@ -547,12 +547,12 @@ private: } IGraphTransformer::TStatus UpdateRowWrapper(TExprNode& node, TExprContext& ctx) { - const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); + const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rowTuple = node.Child(1); - if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { - return TStatus::Error; - } + if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { + return TStatus::Error; + } auto updateTuple = node.Child(2); for (ui32 i = 0; i < updateTuple->ChildrenSize(); ++i) { @@ -584,12 +584,12 @@ private: } IGraphTransformer::TStatus EraseRowWrapper(TExprNode& node, TExprContext& ctx) { - const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); + const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rowTuple = node.Child(1); - if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { - return TStatus::Error; - } + if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { + return TStatus::Error; + } node.SetTypeAnn(ctx.MakeType<TVoidExprType>()); return TStatus::Ok; @@ -843,52 +843,52 @@ void FillColumnsToRead(IDbSchemeResolver::TTableResult* lookup, TExprNode* selec } } -void ValidateCompiledTable(const TExprNode& node, const TTableId& tableId) { - auto currentVersion = ToString(tableId.SchemaVersion); - auto programVersion = node.Child(0)->Child(1)->Content(); - - if (programVersion && programVersion != "0" && programVersion != currentVersion) { +void ValidateCompiledTable(const TExprNode& node, const TTableId& tableId) { + auto currentVersion = ToString(tableId.SchemaVersion); + auto programVersion = node.Child(0)->Child(1)->Content(); + + if (programVersion && programVersion != "0" && programVersion != currentVersion) { throw TErrorException(TIssuesIds::KIKIMR_SCHEME_MISMATCH) - << " Schema version missmatch, compiled over: " << programVersion - << " current: " << currentVersion - << " for table: " << TString(node.Child(0)->Child(0)->Content()); - } - auto currentPathId = TKikimrPathId(tableId.PathId.OwnerId, tableId.PathId.LocalPathId).ToString(); - const auto& programPathId = node.Child(0)->Child(2)->Content(); - // TODO: Remove this checks. - // Check for programPathId just to be able to disable this check - // by '"" record in ut - // - // Check for programPathId != "0:0" to allow legacy yql requests where we - // can use DDL and DML in one program but no additional metadata - // update after DDL executed. - // KIKIMR-9451 for details. - if (programPathId && programPathId != "0:0" && programPathId != currentPathId) { - throw TErrorException(TIssuesIds::KIKIMR_SCHEME_MISMATCH) - << " Path id missmatch, compiled over: " << programPathId - << " current: " << currentPathId - << " for table: " << TString(node.Child(0)->Child(0)->Content()); - } -} - -bool HasUnversionedTable(const TExprNode& node) { - return node.Child(0)->IsAtom(); -} - -IDbSchemeResolver::TTableResult* GetTableLookup(TContext::TPtr ctx, const TExprNode& node) { - return ctx->GetTableLookup(node, GetTableName(&node)); -} - -static void FillKeyPosition(TVector<bool>& arr, const TExprNode::TPtr& listNode, const IDbSchemeResolver::TTableResult* lookup) { - for (auto& valueNode : listNode->Children()) { - YQL_ENSURE(valueNode->IsAtom()); - auto column = lookup->Columns.FindPtr(valueNode->Content()); - YQL_ENSURE(column); - YQL_ENSURE(column->KeyPosition >= 0 && (ui32)column->KeyPosition < arr.size()); - arr[column->KeyPosition] = true; - } -} - + << " Schema version missmatch, compiled over: " << programVersion + << " current: " << currentVersion + << " for table: " << TString(node.Child(0)->Child(0)->Content()); + } + auto currentPathId = TKikimrPathId(tableId.PathId.OwnerId, tableId.PathId.LocalPathId).ToString(); + const auto& programPathId = node.Child(0)->Child(2)->Content(); + // TODO: Remove this checks. + // Check for programPathId just to be able to disable this check + // by '"" record in ut + // + // Check for programPathId != "0:0" to allow legacy yql requests where we + // can use DDL and DML in one program but no additional metadata + // update after DDL executed. + // KIKIMR-9451 for details. + if (programPathId && programPathId != "0:0" && programPathId != currentPathId) { + throw TErrorException(TIssuesIds::KIKIMR_SCHEME_MISMATCH) + << " Path id missmatch, compiled over: " << programPathId + << " current: " << currentPathId + << " for table: " << TString(node.Child(0)->Child(0)->Content()); + } +} + +bool HasUnversionedTable(const TExprNode& node) { + return node.Child(0)->IsAtom(); +} + +IDbSchemeResolver::TTableResult* GetTableLookup(TContext::TPtr ctx, const TExprNode& node) { + return ctx->GetTableLookup(node, GetTableName(&node)); +} + +static void FillKeyPosition(TVector<bool>& arr, const TExprNode::TPtr& listNode, const IDbSchemeResolver::TTableResult* lookup) { + for (auto& valueNode : listNode->Children()) { + YQL_ENSURE(valueNode->IsAtom()); + auto column = lookup->Columns.FindPtr(valueNode->Content()); + YQL_ENSURE(column); + YQL_ENSURE(column->KeyPosition >= 0 && (ui32)column->KeyPosition < arr.size()); + arr[column->KeyPosition] = true; + } +} + TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr mkqlContext) { auto compiler = MakeIntrusive<NCommon::TMkqlCommonCallableCompiler>(); @@ -903,16 +903,16 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr compiler->AddCallable("SelectRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { - YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); - const auto lookup = GetTableLookup(mkqlContext, node); - YQL_ENSURE(lookup->TableId); - const bool legacy = HasUnversionedTable(node); - if (!legacy && lookup->TableId->SchemaVersion) { - ValidateCompiledTable(node, *lookup->TableId); - } - if (legacy) { - lookup->TableId->SchemaVersion = 0; - } + YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); + const auto lookup = GetTableLookup(mkqlContext, node); + YQL_ENSURE(lookup->TableId); + const bool legacy = HasUnversionedTable(node); + if (!legacy && lookup->TableId->SchemaVersion) { + ValidateCompiledTable(node, *lookup->TableId); + } + if (legacy) { + lookup->TableId->SchemaVersion = 0; + } auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); @@ -938,17 +938,17 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr compiler->AddCallable("SelectRange", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { - YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); - const auto lookup = GetTableLookup(mkqlContext, node); - YQL_ENSURE(lookup->TableId); - const bool legacy = HasUnversionedTable(node); - if (!legacy && lookup->TableId->SchemaVersion) { - ValidateCompiledTable(node, *lookup->TableId); - } - if (legacy) { - lookup->TableId->SchemaVersion = 0; - } - + YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); + const auto lookup = GetTableLookup(mkqlContext, node); + YQL_ENSURE(lookup->TableId); + const bool legacy = HasUnversionedTable(node); + if (!legacy && lookup->TableId->SchemaVersion) { + ValidateCompiledTable(node, *lookup->TableId); + } + if (legacy) { + lookup->TableId->SchemaVersion = 0; + } + bool includeFrom = true; bool includeTo = true; auto rangeTuple = node.Child(1); @@ -1026,10 +1026,10 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr TVector<bool> skipNullKeys(lookup->KeyColumnCount, false); options.SkipNullKeys = skipNullKeys; - TVector<bool> forbidNullArgsFrom(lookup->KeyColumnCount, false); - options.ForbidNullArgsFrom = forbidNullArgsFrom; - TVector<bool> forbidNullArgsTo(lookup->KeyColumnCount, false); - options.ForbidNullArgsTo = forbidNullArgsTo; + TVector<bool> forbidNullArgsFrom(lookup->KeyColumnCount, false); + options.ForbidNullArgsFrom = forbidNullArgsFrom; + TVector<bool> forbidNullArgsTo(lookup->KeyColumnCount, false); + options.ForbidNullArgsTo = forbidNullArgsTo; TVector<TSelectColumn> columnsToRead; FillColumnsToRead(lookup, node.Child(2), columnsToRead); @@ -1041,11 +1041,11 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr } else if (optionsItem->Child(0)->Content() == "BytesLimit") { options.BytesLimit = MkqlBuildExpr(*optionsItem->Child(1), ctx); } else if (optionsItem->Child(0)->Content() == "SkipNullKeys") { - FillKeyPosition(skipNullKeys, optionsItem->Child(1), lookup); - } else if (optionsItem->Child(0)->Content() == "ForbidNullArgsFrom") { - FillKeyPosition(forbidNullArgsFrom, optionsItem->Child(1), lookup); - } else if (optionsItem->Child(0)->Content() == "ForbidNullArgsTo") { - FillKeyPosition(forbidNullArgsTo, optionsItem->Child(1), lookup); + FillKeyPosition(skipNullKeys, optionsItem->Child(1), lookup); + } else if (optionsItem->Child(0)->Content() == "ForbidNullArgsFrom") { + FillKeyPosition(forbidNullArgsFrom, optionsItem->Child(1), lookup); + } else if (optionsItem->Child(0)->Content() == "ForbidNullArgsTo") { + FillKeyPosition(forbidNullArgsTo, optionsItem->Child(1), lookup); } else if (optionsItem->Child(0)->Content() == "Reverse") { options.Reverse = MkqlBuildExpr(*optionsItem->Child(1), ctx); } else { @@ -1062,16 +1062,16 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr compiler->AddCallable("UpdateRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { - YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); - const auto lookup = GetTableLookup(mkqlContext, node); - YQL_ENSURE(lookup->TableId); - const bool legacy = HasUnversionedTable(node); - if (!legacy && lookup->TableId->SchemaVersion) { - ValidateCompiledTable(node, *lookup->TableId); - } - if (legacy) { - lookup->TableId->SchemaVersion = 0; - } + YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); + const auto lookup = GetTableLookup(mkqlContext, node); + YQL_ENSURE(lookup->TableId); + const bool legacy = HasUnversionedTable(node); + if (!legacy && lookup->TableId->SchemaVersion) { + ValidateCompiledTable(node, *lookup->TableId); + } + if (legacy) { + lookup->TableId->SchemaVersion = 0; + } auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); @@ -1113,16 +1113,16 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr compiler->AddCallable("EraseRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { - YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); - const auto lookup = GetTableLookup(mkqlContext, node); - YQL_ENSURE(lookup->TableId); - const bool legacy = HasUnversionedTable(node); - if (!legacy && lookup->TableId->SchemaVersion) { - ValidateCompiledTable(node, *lookup->TableId); - } - if (legacy) { - lookup->TableId->SchemaVersion = 0; - } + YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); + const auto lookup = GetTableLookup(mkqlContext, node); + YQL_ENSURE(lookup->TableId); + const bool legacy = HasUnversionedTable(node); + if (!legacy && lookup->TableId->SchemaVersion) { + ValidateCompiledTable(node, *lookup->TableId); + } + if (legacy) { + lookup->TableId->SchemaVersion = 0; + } auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); @@ -1272,12 +1272,12 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (typeId == NUdf::TDataType<NUdf::TDecimal>::Id) { auto precision = node.Child(1)->Content(); auto scale = node.Child(2)->Content(); - auto type = ctx.ProgramBuilder.NewDecimalType(FromString<ui8>(precision), FromString<ui8>(scale)); + auto type = ctx.ProgramBuilder.NewDecimalType(FromString<ui8>(precision), FromString<ui8>(scale)); return TRuntimeNode(type, true); - } else { - auto type = ctx.ProgramBuilder.NewDataType(typeId); + } else { + auto type = ctx.ProgramBuilder.NewDataType(typeId); return TRuntimeNode(type, true); - } + } }); return compiler; @@ -1338,11 +1338,11 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, TContext::TPtr ctx(new TContext(functionRegistry, typeEnv)); auto compiler = CreateMkqlCompiler(ctx); CollectKeys(expr->Root.Get(), ctx); - const auto& tablesToResolve = ctx->GetTablesToResolve(); + const auto& tablesToResolve = ctx->GetTablesToResolve(); if (!tablesToResolve.empty()) { TVector<IDbSchemeResolver::TTable> requests; requests.reserve(tablesToResolve.size()); - for (const auto& x : tablesToResolve) { + for (const auto& x : tablesToResolve) { requests.push_back(x.second.Request); } Y_VERIFY_DEBUG(dbSchemeResolver); @@ -1354,7 +1354,7 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, Y_VERIFY_DEBUG(tablesToResolve.size() == results.size(), "tablesToResolve.size() != results.size()"); ui32 i = 0; for (auto& x : tablesToResolve) { - const auto& response = results[i]; + const auto& response = results[i]; Y_ENSURE_EX(response.Status == IDbSchemeResolver::TTableResult::Ok, TNodeException() << "Failed to resolve table " << x.second.Request.TableName << ", error: " << response.Reason); @@ -1382,7 +1382,7 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace TConvertResult convRes; - convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); + convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); promise.SetValue(convRes); } }); @@ -1407,7 +1407,7 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace TConvertResult convRes; - convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); + convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); promise.SetValue(convRes); } @@ -1453,7 +1453,7 @@ public: CollectKeys(Expr->Root.Get(), CompileCtx); Compiler = CreateMkqlCompiler(CompileCtx); - const auto& tablesToResolve = CompileCtx->GetTablesToResolve(); + const auto& tablesToResolve = CompileCtx->GetTablesToResolve(); if (!tablesToResolve.empty()) { TVector<IDbSchemeResolver::TTable> requests; requests.reserve(tablesToResolve.size()); @@ -1478,7 +1478,7 @@ public: return SendResponseAndDie(res, {}, ctx); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace - TMiniKQLCompileResult res(NYql::ExceptionToIssue(ex)); + TMiniKQLCompileResult res(NYql::ExceptionToIssue(ex)); return SendResponseAndDie(res, {}, ctx); } } @@ -1554,12 +1554,12 @@ private: } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace - TMiniKQLCompileResult result(NYql::ExceptionToIssue(ex)); + TMiniKQLCompileResult result(NYql::ExceptionToIssue(ex)); return SendResponseAndDie(result, std::move(compileResolveCookies), ctx); } } - bool ParseProgram(TIssues& errors) { + bool ParseProgram(TIssues& errors) { Expr.Reset(new NYql::TExprContainer()); NYql::TAstParseResult astRes = NYql::ParseAst(Program); auto root = astRes.Root; @@ -1570,7 +1570,7 @@ private: } Expr->Context.IssueManager.AddIssues(std::move(astRes.Issues)); if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr)) { - errors = Expr->Context.IssueManager.GetIssues(); + errors = Expr->Context.IssueManager.GetIssues(); return false; } IGraphTransformer::TStatus status(IGraphTransformer::TStatus::Ok); @@ -1580,7 +1580,7 @@ private: Y_VERIFY_DEBUG(status.Level == IGraphTransformer::TStatus::Ok || status.Level == IGraphTransformer::TStatus::Error); if (status.Level != IGraphTransformer::TStatus::Ok) { - errors = Expr->Context.IssueManager.GetIssues(); + errors = Expr->Context.IssueManager.GetIssues(); return false; } return true; diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp index a0a15ac75b..4fcdf735ba 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp @@ -101,7 +101,7 @@ Y_UNIT_TEST_SUITE(TTestYqlToMiniKQLCompile) { UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); UNIT_ASSERT_EQUAL(res[0].Status, IDbSchemeResolver::TTableResult::Ok); UNIT_ASSERT(!!res[0].TableId); - UNIT_ASSERT(res[0].TableId->HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(res[0].TableId->HasSamePath(TTableId(1, 2))); UNIT_ASSERT_EQUAL(res[0].KeyColumnCount, 1); UNIT_ASSERT_VALUES_EQUAL(res[0].Columns.size(), 1); UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].Column, 56); diff --git a/ydb/core/client/minikql_result_lib/data_funcs.inl b/ydb/core/client/minikql_result_lib/data_funcs.inl index 7b7bf99094..a152c054a1 100644 --- a/ydb/core/client/minikql_result_lib/data_funcs.inl +++ b/ydb/core/client/minikql_result_lib/data_funcs.inl @@ -8,49 +8,49 @@ template <> -inline bool HasData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Bool); return value.HasBool(); } template <> -inline bool HasData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Int32); return value.HasInt32(); } template <> -inline bool HasData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Uint32); return value.HasUint32(); } template <> -inline bool HasData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Int64); return value.HasInt64(); } template <> -inline bool HasData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Uint64); return value.HasUint64(); } template <> -inline bool HasData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Float); return value.HasFloat(); } template <> -inline bool HasData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_SCHEME_TYPE(schemeType, Double); return value.HasDouble(); } template <> -inline bool HasData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { if (schemeType == NScheme::NTypeIds::Utf8) { return value.HasText(); } else { @@ -66,49 +66,49 @@ inline bool HasData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schem } while (0); template <> -inline bool GetData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool GetData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(bool, value, schemeType); return value.GetBool(); } template <> -inline i32 GetData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline i32 GetData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(i32, value, schemeType); return value.GetInt32(); } template <> -inline ui32 GetData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline ui32 GetData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(ui32, value, schemeType); return value.GetUint32(); } template <> -inline i64 GetData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline i64 GetData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(i64, value, schemeType); return value.GetInt64(); } template <> -inline ui64 GetData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline ui64 GetData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(ui64, value, schemeType); return value.GetUint64(); } template <> -inline float GetData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline float GetData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(float, value, schemeType); return value.GetFloat(); } template <> -inline double GetData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline double GetData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(double, value, schemeType); return value.GetDouble(); } template <> -inline TStringBuf GetData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline TStringBuf GetData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { ENSURE_HAS_DATA(TStringBuf, value, schemeType); if (schemeType == NScheme::NTypeIds::Utf8) { return value.GetText(); diff --git a/ydb/core/client/minikql_result_lib/list_funcs.inl b/ydb/core/client/minikql_result_lib/list_funcs.inl index 7b9e6a5a92..6fb2d46ab8 100644 --- a/ydb/core/client/minikql_result_lib/list_funcs.inl +++ b/ydb/core/client/minikql_result_lib/list_funcs.inl @@ -46,7 +46,7 @@ inline T TListType::iterator<T>::operator*() const { template <typename T> inline T TListType::iterator<T>::Get() const { ENSURE_KIND(ItemType, Data); - auto schemeType = ItemType.GetData().GetScheme(); + auto schemeType = ItemType.GetData().GetScheme(); return NPrivate::GetData<T>(*Item, schemeType); } diff --git a/ydb/core/client/minikql_result_lib/objects.h b/ydb/core/client/minikql_result_lib/objects.h index cc5749168a..4b5ae0da45 100644 --- a/ydb/core/client/minikql_result_lib/objects.h +++ b/ydb/core/client/minikql_result_lib/objects.h @@ -26,14 +26,14 @@ class TDict; namespace NPrivate { template <typename T> -inline bool HasData(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline bool HasData(const TProtoValue& value, NScheme::TTypeId schemeType) { Y_UNUSED(value); Y_UNUSED(schemeType); Y_FAIL("Not scpecified type."); } template <typename T> -inline T GetData(const TProtoValue& value, NScheme::TTypeId schemeType) { +inline T GetData(const TProtoValue& value, NScheme::TTypeId schemeType) { Y_UNUSED(value); Y_UNUSED(schemeType); Y_FAIL("Not scpecified type."); @@ -252,7 +252,7 @@ T TOptional::GetItem() const { Y_ENSURE(HasItem(), "Optional is empty!"); const auto& itemType = RootType.GetOptional().GetItem(); ENSURE_KIND(itemType, Data); - auto schemeType = itemType.GetData().GetScheme(); + auto schemeType = itemType.GetData().GetScheme(); return NPrivate::GetData<T>(RootValue.GetOptional(), schemeType); } @@ -270,7 +270,7 @@ T TTuple::GetElement(size_t index) const { const auto& elementType = RootType.GetTuple().GetElement(index); const auto& element = RootValue.GetTuple(index); ENSURE_KIND(elementType, Data); - auto schemeType = elementType.GetData().GetScheme(); + auto schemeType = elementType.GetData().GetScheme(); return NPrivate::GetData<T>(element, schemeType); } @@ -290,7 +290,7 @@ T TStruct::GetMember(size_t memberIndex) { const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); const auto& member = RootValue.GetStruct(memberIndex); ENSURE_KIND(memberType, Data); - auto schemeType = memberType.GetData().GetScheme(); + auto schemeType = memberType.GetData().GetScheme(); return NPrivate::GetData<T>(member, schemeType); } diff --git a/ydb/core/client/server/grpc_server.cpp b/ydb/core/client/server/grpc_server.cpp index 9f3d69ec4d..aa122da36a 100644 --- a/ydb/core/client/server/grpc_server.cpp +++ b/ydb/core/client/server/grpc_server.cpp @@ -32,16 +32,16 @@ using NKikimrClient::TResponse; using NKikimrClient::TPersQueueRequest; using NGrpc::IQueueEvent; - + using namespace NActors; -using namespace NThreading; +using namespace NThreading; namespace NKikimr { namespace NGRpcProxy { namespace { using TGrpcBaseAsyncContext = NGrpc::TBaseAsyncContext<NGRpcProxy::TGRpcService>; - + template <typename TIn, typename TOut = TResponse> class TSimpleRequest : public IQueueEvent @@ -54,8 +54,8 @@ class TSimpleRequest ServerAsyncResponseWriter<TOut>*, CompletionQueue*, ServerCompletionQueue*, void*); public: - - TSimpleRequest(TGRpcService* server, + + TSimpleRequest(TGRpcService* server, NKikimrClient::TGRpcServer::AsyncService* service, ServerCompletionQueue* cq, TOnRequest cb, @@ -75,26 +75,26 @@ public: , RequestSize(0) , ResponseSize(0) , ResponseStatus(0) - , InProgress_(false) + , InProgress_(false) { LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] created request Name# %s", this, Name); } ~TSimpleRequest() { - if (InProgress_) { - //If we are ShuttingDown probably ActorSystem unable to recieve new events - if (!Server->IsShuttingDown()) { - LOG_ERROR(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] request destroyed with InProgress state Name# %s", this, Name); - } + if (InProgress_) { + //If we are ShuttingDown probably ActorSystem unable to recieve new events + if (!Server->IsShuttingDown()) { + LOG_ERROR(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] request destroyed with InProgress state Name# %s", this, Name); + } Counters->FinishProcessing(RequestSize, ResponseSize, false, ResponseStatus, TDuration::Seconds(RequestTimer.Passed())); - Server->DecRequest(); + Server->DecRequest(); } } void Start() { if (auto guard = Server->ProtectShutdown()) { - (Service->*RequestCallback)(&Context, &Request, Writer.Get(), CQ, CQ, GetGRpcTag()); + (Service->*RequestCallback)(&Context, &Request, Writer.Get(), CQ, CQ, GetGRpcTag()); } else { // Server is shutting down, new requests cannot be started delete this; @@ -245,10 +245,10 @@ public: } private: - void* GetGRpcTag() { - return static_cast<IQueueEvent*>(this); - } - + void* GetGRpcTag() { + return static_cast<IQueueEvent*>(this); + } + void Finish(const TOut& resp, ui32 status) { auto makeResponseString = [&] { TString x; @@ -257,26 +257,26 @@ private: printer.PrintToString(resp, &x); return x; }; - LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] issuing response Name# %s data# %s peer# %s", this, + LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] issuing response Name# %s data# %s peer# %s", this, Name, makeResponseString().data(), Context.peer().c_str()); ResponseSize = resp.ByteSize(); ResponseStatus = status; - StateFunc = &TSimpleRequest::FinishDone; - Writer->Finish(resp, Status::OK, GetGRpcTag()); + StateFunc = &TSimpleRequest::FinishDone; + Writer->Finish(resp, Status::OK, GetGRpcTag()); + } + + void FinishNoResource() { + TOut resp; + TString msg = "no resource"; + LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] issuing response Name# %s nodata (no resources) peer# %s", this, + Name, Context.peer().c_str()); + + StateFunc = &TSimpleRequest::FinishDoneWithoutProcessing; + Writer->Finish(resp, + grpc::Status(grpc::StatusCode::RESOURCE_EXHAUSTED, msg), + GetGRpcTag()); } - void FinishNoResource() { - TOut resp; - TString msg = "no resource"; - LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] issuing response Name# %s nodata (no resources) peer# %s", this, - Name, Context.peer().c_str()); - - StateFunc = &TSimpleRequest::FinishDoneWithoutProcessing; - Writer->Finish(resp, - grpc::Status(grpc::StatusCode::RESOURCE_EXHAUSTED, msg), - GetGRpcTag()); - } - bool RequestDone(bool ok) { auto makeRequestString = [&] { TString resp; @@ -289,7 +289,7 @@ private: } return resp; }; - LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] received request Name# %s ok# %s data# %s peer# %s current inflight# %li", this, + LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] received request Name# %s ok# %s data# %s peer# %s current inflight# %li", this, Name, ok ? "true" : "false", makeRequestString().data(), Context.peer().c_str(), Server->GetCurrentInFlight()); if (Context.c_call() == nullptr) { @@ -303,50 +303,50 @@ private: return false; } - Clone(); - + Clone(); + if (!ok) { Counters->CountNotOkRequest(); return false; } - if (Server->IncRequest()) { + if (Server->IncRequest()) { - RequestSize = Request.ByteSize(); - Counters->StartProcessing(RequestSize); + RequestSize = Request.ByteSize(); + Counters->StartProcessing(RequestSize); RequestTimer.Reset(); - InProgress_ = true; + InProgress_ = true; - Cb(this); - } else { - FinishNoResource(); - } + Cb(this); + } else { + FinishNoResource(); + } - return true; + return true; } bool FinishDone(bool ok) { - LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] finished request Name# %s ok# %s peer# %s", this, - Name, ok ? "true" : "false", Context.peer().c_str()); + LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] finished request Name# %s ok# %s peer# %s", this, + Name, ok ? "true" : "false", Context.peer().c_str()); Counters->FinishProcessing(RequestSize, ResponseSize, ok, ResponseStatus, TDuration::Seconds(RequestTimer.Passed())); - Server->DecRequest(); - InProgress_ = false; + Server->DecRequest(); + InProgress_ = false; - return false; - } + return false; + } + + bool FinishDoneWithoutProcessing(bool ok) { + LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] finished request without processing Name# %s ok# %s peer# %s", this, + Name, ok ? "true" : "false", Context.peer().c_str()); - bool FinishDoneWithoutProcessing(bool ok) { - LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] finished request without processing Name# %s ok# %s peer# %s", this, - Name, ok ? "true" : "false", Context.peer().c_str()); - return false; } private: using TStateFunc = bool (TSimpleRequest::*)(bool); - TGRpcService* const Server; + TGRpcService* const Server; TOnRequest Cb; TRequestCallback RequestCallback; TActorSystem& ActorSystem; @@ -363,62 +363,62 @@ private: THPTimer RequestTimer; TMaybe<NMsgBusProxy::TBusMessageContext> BusContext; - bool InProgress_; + bool InProgress_; bool RequestRegistered_ = false; }; } // namespace -TGRpcService::TGRpcService() - : ActorSystem(nullptr) -{} +TGRpcService::TGRpcService() + : ActorSystem(nullptr) +{} void TGRpcService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr) { - CQ = cq; - Y_ASSERT(InitCb_); - InitCb_(); + CQ = cq; + Y_ASSERT(InitCb_); + InitCb_(); } TFuture<void> TGRpcService::Prepare(TActorSystem* system, const TActorId& pqMeta, const TActorId& msgBusProxy, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { - auto promise = NewPromise<void>(); - InitCb_ = [=]() mutable { - try { - ActorSystem = system; - PQMeta = pqMeta; - MsgBusProxy = msgBusProxy; - Counters = counters; - - promise.SetValue(); - } catch (...) { - promise.SetException(std::current_exception()); - } - }; - return promise.GetFuture(); + TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { + auto promise = NewPromise<void>(); + InitCb_ = [=]() mutable { + try { + ActorSystem = system; + PQMeta = pqMeta; + MsgBusProxy = msgBusProxy; + Counters = counters; + + promise.SetValue(); + } catch (...) { + promise.SetException(std::current_exception()); + } + }; + return promise.GetFuture(); } void TGRpcService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter *limiter) { - Limiter_ = limiter; -} - -bool TGRpcService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - -i64 TGRpcService::GetCurrentInFlight() const { - return Limiter_->GetCurrentInFlight(); -} - -void TGRpcService::Start() { - Y_VERIFY(ActorSystem); + Limiter_ = limiter; +} + +bool TGRpcService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + +i64 TGRpcService::GetCurrentInFlight() const { + return Limiter_->GetCurrentInFlight(); +} + +void TGRpcService::Start() { + Y_VERIFY(ActorSystem); ui32 nodeId = ActorSystem->NodeId; - ActorSystem->Send(MakeGRpcProxyStatusID(nodeId), new TEvGRpcProxyStatus::TEvSetup(true, PersQueueWriteSessionsMaxCount, - PersQueueReadSessionsMaxCount)); + ActorSystem->Send(MakeGRpcProxyStatusID(nodeId), new TEvGRpcProxyStatus::TEvSetup(true, PersQueueWriteSessionsMaxCount, + PersQueueReadSessionsMaxCount)); SetupIncomingRequests(); } @@ -426,7 +426,7 @@ void TGRpcService::RegisterRequestActor(NActors::IActor* req) { ActorSystem->Register(req, TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); } -void TGRpcService::SetupIncomingRequests() { +void TGRpcService::SetupIncomingRequests() { auto getCounterBlock = NGRpcService::CreateCounterCb(Counters, ActorSystem); diff --git a/ydb/core/client/server/grpc_server.h b/ydb/core/client/server/grpc_server.h index 63a598d661..2f2ae1da6b 100644 --- a/ydb/core/client/server/grpc_server.h +++ b/ydb/core/client/server/grpc_server.h @@ -51,11 +51,11 @@ public: }; //! Implements interaction Kikimr via gRPC protocol. -class TGRpcService +class TGRpcService : public NGrpc::TGrpcServiceBase<NKikimrClient::TGRpcServer> { public: - TGRpcService(); + TGRpcService(); void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; @@ -63,15 +63,15 @@ public: NThreading::TFuture<void> Prepare(NActors::TActorSystem* system, const NActors::TActorId& pqMeta, const NActors::TActorId& msgBusProxy, TIntrusivePtr<NMonitoring::TDynamicCounters> counters); void Start(); - bool IncRequest(); - void DecRequest(); - i64 GetCurrentInFlight() const; + bool IncRequest(); + void DecRequest(); + i64 GetCurrentInFlight() const; private: void RegisterRequestActor(NActors::IActor* req); - //! Setup handlers for incoming requests. - void SetupIncomingRequests(); + //! Setup handlers for incoming requests. + void SetupIncomingRequests(); private: using IThreadRef = TAutoPtr<IThreadFactory::IThread>; @@ -81,14 +81,14 @@ private: NActors::TActorId PQMeta; NActors::TActorId MsgBusProxy; - grpc::ServerCompletionQueue* CQ = nullptr; + grpc::ServerCompletionQueue* CQ = nullptr; - size_t PersQueueWriteSessionsMaxCount = 1000000; - size_t PersQueueReadSessionsMaxCount = 100000; + size_t PersQueueWriteSessionsMaxCount = 1000000; + size_t PersQueueReadSessionsMaxCount = 100000; TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - std::function<void()> InitCb_; + std::function<void()> InitCb_; // In flight request management. NGrpc::TGlobalLimiter* Limiter_ = nullptr; diff --git a/ydb/core/client/server/msgbus_securereq.h b/ydb/core/client/server/msgbus_securereq.h index 3d52546cfc..5e68ca0eeb 100644 --- a/ydb/core/client/server/msgbus_securereq.h +++ b/ydb/core/client/server/msgbus_securereq.h @@ -15,11 +15,11 @@ template <typename TDerived> class TMessageBusSecureRequest<TMessageBusServerRequestBase<TDerived>> : public TSecureRequestActor<TMessageBusServerRequestBase<TMessageBusSecureRequest<TMessageBusServerRequestBase<TDerived>>>, TDerived> { public: - void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { + void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { TMessageBusServerRequestBase<TMessageBusSecureRequest<TMessageBusServerRequestBase<TDerived>>>::HandleError( MSTATUS_ERROR, TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::AccessDenied, - error.Message, + error.Message, ctx); } @@ -51,11 +51,11 @@ public: TMessageBusSimpleTabletRequest<TDerived, TTabletReplyEvent, Activity>::Die(ctx); } - void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { + void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { HandleError( MSTATUS_ERROR, TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::AccessDenied, - error.Message, + error.Message, ctx); } @@ -90,11 +90,11 @@ public: TMessageBusTabletRequest<TDerived, TTabletReplyEvent>::Die(ctx); } - void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { + void OnAccessDenied(const TEvTicketParser::TError& error, const TActorContext& ctx) { HandleError( MSTATUS_ERROR, TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::AccessDenied, - error.Message, + error.Message, ctx); } diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp index a4e1d0bc22..8a4d8a3bbd 100644 --- a/ydb/core/client/server/msgbus_server.cpp +++ b/ydb/core/client/server/msgbus_server.cpp @@ -145,7 +145,7 @@ public: void ForgetRequest() { if (RequestContext) { - RequestContext->ReplyError("request wasn't processed properly"); + RequestContext->ReplyError("request wasn't processed properly"); RequestContext = nullptr; } } @@ -181,7 +181,7 @@ public: default: Y_FAIL("unexpected response type %" PRIu32, type); } - RequestContext = nullptr; + RequestContext = nullptr; } void SendReplyMove(NBus::TBusMessageAutoPtr response) override { @@ -311,14 +311,14 @@ public: Y_VERIFY(Context); Context->SendReply(resp); - auto context = std::move(Context); + auto context = std::move(Context); } void SendReplyMove(NBus::TBusMessageAutoPtr resp) override { Y_VERIFY(Context); Context->SendReplyMove(resp); - auto context = std::move(Context); + auto context = std::move(Context); } ui64 GetTotalTimeout() const override { diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h index 6e0b3276e0..3d40fc9001 100644 --- a/ydb/core/client/server/msgbus_server.h +++ b/ydb/core/client/server/msgbus_server.h @@ -103,8 +103,8 @@ struct TEvBusProxy { EvInitRoot, EvChooseProxy, - EvStreamIsReadyNotUsed = EvRequest + 512, - EvStreamIsDeadNotUsed, + EvStreamIsReadyNotUsed = EvRequest + 512, + EvStreamIsDeadNotUsed, EvEnd }; diff --git a/ydb/core/client/server/msgbus_server_cms.cpp b/ydb/core/client/server/msgbus_server_cms.cpp index 63be46e430..53ada2a3fa 100644 --- a/ydb/core/client/server/msgbus_server_cms.cpp +++ b/ydb/core/client/server/msgbus_server_cms.cpp @@ -64,7 +64,7 @@ public: NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 10}; auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeCmsID(StateStorageGroup), pipeConfig); - CmsPipe = ctx.RegisterWithSameMailbox(pipe); + CmsPipe = ctx.RegisterWithSameMailbox(pipe); LOG_DEBUG(ctx, NKikimrServices::CMS, "Forwarding CMS request: %s", Request.ShortDebugString().data()); diff --git a/ydb/core/client/server/msgbus_server_console.cpp b/ydb/core/client/server/msgbus_server_console.cpp index 96f5cabf66..66bc0b02eb 100644 --- a/ydb/core/client/server/msgbus_server_console.cpp +++ b/ydb/core/client/server/msgbus_server_console.cpp @@ -66,7 +66,7 @@ public: NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 10}; auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeConsoleID(StateStorageGroup), pipeConfig); - ConsolePipe = ctx.RegisterWithSameMailbox(pipe); + ConsolePipe = ctx.RegisterWithSameMailbox(pipe); // Don't print security token. Request.ClearSecurityToken(); diff --git a/ydb/core/client/server/msgbus_server_db.cpp b/ydb/core/client/server/msgbus_server_db.cpp index fc2aefa1f0..08f3d71786 100644 --- a/ydb/core/client/server/msgbus_server_db.cpp +++ b/ydb/core/client/server/msgbus_server_db.cpp @@ -368,7 +368,7 @@ public: return builder.NewDataLiteral<NUdf::EDataSlot::String>(jsonValue.GetString()); case NScheme::NTypeIds::Yson: return pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Yson>(jsonValue.GetString()); - case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::Json: return pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Json>(jsonValue.GetString()); case NScheme::NTypeIds::JsonDocument: return pgmBuilder.NewDataLiteral<NUdf::EDataSlot::JsonDocument>(jsonValue.GetString()); @@ -662,7 +662,7 @@ protected: auto itPipe = Pipes.find(schemeShardId); if (itPipe == Pipes.end()) { static NTabletPipe::TClientConfig clientConfig; - pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, schemeShardId, clientConfig)); + pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, schemeShardId, clientConfig)); Pipes.emplace(schemeShardId, pipe); } else { pipe = itPipe->second; diff --git a/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp b/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp index 4a1c332733..0c41e53f09 100644 --- a/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp +++ b/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp @@ -274,7 +274,7 @@ public: ui64 hiveTabletId = domainsInfo.GetHive(hiveUid); if (Status == NKikimrProto::OK) { - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, hiveTabletId, clientConfig)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, hiveTabletId, clientConfig)); for (ui64 i = 0; i < Requests.size(); ++i) { const TRequest &cmd = Requests[i]; diff --git a/ydb/core/client/server/msgbus_server_node_registration.cpp b/ydb/core/client/server/msgbus_server_node_registration.cpp index 569c165e19..a3dda6092d 100644 --- a/ydb/core/client/server/msgbus_server_node_registration.cpp +++ b/ydb/core/client/server/msgbus_server_node_registration.cpp @@ -58,7 +58,7 @@ public: NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 10}; auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeNodeBrokerID(group), pipeConfig); - NodeBrokerPipe = ctx.RegisterWithSameMailbox(pipe); + NodeBrokerPipe = ctx.RegisterWithSameMailbox(pipe); TAutoPtr<TEvNodeBroker::TEvRegistrationRequest> request = new TEvNodeBroker::TEvRegistrationRequest; diff --git a/ydb/core/client/server/msgbus_server_persqueue.cpp b/ydb/core/client/server/msgbus_server_persqueue.cpp index ac2703cd1d..78b83ef3a3 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.cpp +++ b/ydb/core/client/server/msgbus_server_persqueue.cpp @@ -987,7 +987,7 @@ public: } NTabletPipe::TClientConfig clientConfig; - PQClient.push_back(ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); + PQClient.push_back(ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); ActorIdToProto(PQClient.back(), RequestProto.MutablePartitionRequest()->MutablePipeClient()); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); @@ -1032,7 +1032,7 @@ public: tabletInfo.Topic = topic; NTabletPipe::TClientConfig clientConfig; - TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, it->second.BalancerTabletId, clientConfig)); + TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, it->second.BalancerTabletId, clientConfig)); tabletInfo.PipeClient = pipeClient; PQClient.push_back(pipeClient); @@ -1056,7 +1056,7 @@ public: // Tablet node resolution relies on opening a pipe NTabletPipe::TClientConfig clientConfig; - TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); + TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); tabletInfo.PipeClient = pipeClient; PQClient.push_back(pipeClient); if (needAskOffset) { diff --git a/ydb/core/client/server/msgbus_server_persqueue.h b/ydb/core/client/server/msgbus_server_persqueue.h index 575c4414fe..52c67832e4 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.h +++ b/ydb/core/client/server/msgbus_server_persqueue.h @@ -224,7 +224,7 @@ protected: TActorId CreatePipe(ui64 tabletId, const TActorContext& ctx) { NTabletPipe::TClientConfig clientConfig; - const TActorId pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); + const TActorId pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); Y_VERIFY(Pipes.emplace(tabletId, pipe).second); return pipe; } diff --git a/ydb/core/client/server/msgbus_server_pq_read_session_info.cpp b/ydb/core/client/server/msgbus_server_pq_read_session_info.cpp index 684cdc62e7..05c67d8f06 100644 --- a/ydb/core/client/server/msgbus_server_pq_read_session_info.cpp +++ b/ydb/core/client/server/msgbus_server_pq_read_session_info.cpp @@ -1,113 +1,113 @@ -#include "msgbus_server_pq_read_session_info.h" - -namespace NKikimr { -namespace NMsgBusProxy { - -IPersQueueGetReadSessionsInfoWorker::IPersQueueGetReadSessionsInfoWorker( - const TActorId& parentId, - const THashMap<TString, TActorId>& readSessions, - std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo -) - : ParentId(parentId) - , ReadSessions(std::move(readSessions)) - , NodesInfo(nodesInfo) -{ } - -TString IPersQueueGetReadSessionsInfoWorker::GetHostName(ui32 hostId) const { - const auto host = NodesInfo->HostNames.find(hostId); - return host != NodesInfo->HostNames.end() ? host->second : TString(); -} - -void IPersQueueGetReadSessionsInfoWorker::Bootstrap(const TActorContext& ctx) { - Become(&IPersQueueGetReadSessionsInfoWorker::StateFunc); - - for (auto & r : ReadSessions) { - SendStatusRequest(r.first, r.second, ctx); - ReadSessionNames.insert(r.first); - } -} - -void IPersQueueGetReadSessionsInfoWorker::ProcessStatus(const NKikimrPQ::TReadSessionStatusResponse& response, const TActorContext& ctx) { - auto it = ReadSessions.find(response.GetSession()); - if (it == ReadSessions.end()) - return; - ReadSessions.erase(it); - - ReadSessionStatus[response.GetSession()] = response; - - ReadSessionNames.erase(response.GetSession()); - - if (ReadyToAnswer()) { - Answer(ctx); - } -} - -void IPersQueueGetReadSessionsInfoWorker::Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { - for (auto &s: ReadSessions) { - if (s.second == ev->Sender) { - ReadSessions.erase(s.first); - return; - } - } - - if (ReadyToAnswer()) { - Answer(ctx); - } -} - -void IPersQueueGetReadSessionsInfoWorker::Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { - ui32 nodeId = ev->Get()->NodeId; - for (auto it = ReadSessions.begin(); it != ReadSessions.end();) { - auto jt = it++; - if (jt->second.NodeId() == nodeId) { - ReadSessions.erase(jt); - } - } - if (ReadyToAnswer()) { - Answer(ctx); - } - -} - -bool IPersQueueGetReadSessionsInfoWorker::ReadyToAnswer() const { - return ReadSessions.empty(); -} - -void IPersQueueGetReadSessionsInfoWorker::Answer(const TActorContext& ctx) { - NKikimrClient::TResponse response; - response.SetStatus(MSTATUS_OK); - response.SetErrorCode(NPersQueue::NErrorCode::OK); - auto stat = response.MutableMetaResponse()->MutableCmdGetReadSessionsInfoResult(); - - for (const auto& s : ReadSessionStatus) { - auto res = stat->AddSessionResult(); - res->SetSession(s.second.GetSession()); - res->SetClientNode(s.second.GetClientNode()); - res->SetProxyNode(GetHostName(s.second.GetProxyNodeId())); - res->SetTimestamp(s.second.GetTimestamp()); - - for (const auto& p : s.second.GetPartition()) { - auto partRes = res->AddPartitionResult(); - partRes->SetTopic(p.GetTopic()); - partRes->SetPartition(p.GetPartition()); - - for (const auto& c : p.GetNextCommits()) { - partRes->AddNextCommits(c); - } - partRes->SetLastReadId(p.GetLastReadId()); - partRes->SetReadIdCommitted(p.GetReadIdCommitted()); - - partRes->SetTimestamp(p.GetTimestampMs()); - } - } - - THolder<TEvPersQueue::TEvResponse> result(new TEvPersQueue::TEvResponse()); - result->Record.Swap(&response); - - ctx.Send(ParentId, result.Release()); - - Die(ctx); -} - -} // namespace NMsgBusProxy -} // namespace NKikimr +#include "msgbus_server_pq_read_session_info.h" + +namespace NKikimr { +namespace NMsgBusProxy { + +IPersQueueGetReadSessionsInfoWorker::IPersQueueGetReadSessionsInfoWorker( + const TActorId& parentId, + const THashMap<TString, TActorId>& readSessions, + std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo +) + : ParentId(parentId) + , ReadSessions(std::move(readSessions)) + , NodesInfo(nodesInfo) +{ } + +TString IPersQueueGetReadSessionsInfoWorker::GetHostName(ui32 hostId) const { + const auto host = NodesInfo->HostNames.find(hostId); + return host != NodesInfo->HostNames.end() ? host->second : TString(); +} + +void IPersQueueGetReadSessionsInfoWorker::Bootstrap(const TActorContext& ctx) { + Become(&IPersQueueGetReadSessionsInfoWorker::StateFunc); + + for (auto & r : ReadSessions) { + SendStatusRequest(r.first, r.second, ctx); + ReadSessionNames.insert(r.first); + } +} + +void IPersQueueGetReadSessionsInfoWorker::ProcessStatus(const NKikimrPQ::TReadSessionStatusResponse& response, const TActorContext& ctx) { + auto it = ReadSessions.find(response.GetSession()); + if (it == ReadSessions.end()) + return; + ReadSessions.erase(it); + + ReadSessionStatus[response.GetSession()] = response; + + ReadSessionNames.erase(response.GetSession()); + + if (ReadyToAnswer()) { + Answer(ctx); + } +} + +void IPersQueueGetReadSessionsInfoWorker::Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { + for (auto &s: ReadSessions) { + if (s.second == ev->Sender) { + ReadSessions.erase(s.first); + return; + } + } + + if (ReadyToAnswer()) { + Answer(ctx); + } +} + +void IPersQueueGetReadSessionsInfoWorker::Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { + ui32 nodeId = ev->Get()->NodeId; + for (auto it = ReadSessions.begin(); it != ReadSessions.end();) { + auto jt = it++; + if (jt->second.NodeId() == nodeId) { + ReadSessions.erase(jt); + } + } + if (ReadyToAnswer()) { + Answer(ctx); + } + +} + +bool IPersQueueGetReadSessionsInfoWorker::ReadyToAnswer() const { + return ReadSessions.empty(); +} + +void IPersQueueGetReadSessionsInfoWorker::Answer(const TActorContext& ctx) { + NKikimrClient::TResponse response; + response.SetStatus(MSTATUS_OK); + response.SetErrorCode(NPersQueue::NErrorCode::OK); + auto stat = response.MutableMetaResponse()->MutableCmdGetReadSessionsInfoResult(); + + for (const auto& s : ReadSessionStatus) { + auto res = stat->AddSessionResult(); + res->SetSession(s.second.GetSession()); + res->SetClientNode(s.second.GetClientNode()); + res->SetProxyNode(GetHostName(s.second.GetProxyNodeId())); + res->SetTimestamp(s.second.GetTimestamp()); + + for (const auto& p : s.second.GetPartition()) { + auto partRes = res->AddPartitionResult(); + partRes->SetTopic(p.GetTopic()); + partRes->SetPartition(p.GetPartition()); + + for (const auto& c : p.GetNextCommits()) { + partRes->AddNextCommits(c); + } + partRes->SetLastReadId(p.GetLastReadId()); + partRes->SetReadIdCommitted(p.GetReadIdCommitted()); + + partRes->SetTimestamp(p.GetTimestampMs()); + } + } + + THolder<TEvPersQueue::TEvResponse> result(new TEvPersQueue::TEvResponse()); + result->Record.Swap(&response); + + ctx.Send(ParentId, result.Release()); + + Die(ctx); +} + +} // namespace NMsgBusProxy +} // namespace NKikimr diff --git a/ydb/core/client/server/msgbus_server_request.cpp b/ydb/core/client/server/msgbus_server_request.cpp index fc4ae3d3d2..9223bea9f1 100644 --- a/ydb/core/client/server/msgbus_server_request.cpp +++ b/ydb/core/client/server/msgbus_server_request.cpp @@ -28,7 +28,7 @@ class TMessageBusServerRequest : public TMessageBusSecureRequest<TMessageBusServ bool CompilationRetried; void ReplyWithResult(EResponseStatus status, NKikimrTxUserProxy::TEvProposeTransactionStatus &result, - const TActorContext &ctx); + const TActorContext &ctx); bool RetryResolve(const TActorContext &ctx); void FinishReply(const TActorContext &ctx); void TryToAllocateQuota(const TActorContext &ctx); @@ -88,7 +88,7 @@ public: record.SetExecTimeoutPeriod(msgBusTimeout); } - record.SetStreamResponse(false); + record.SetStreamResponse(false); auto* transaction = record.MutableTransaction(); if (transaction->HasMiniKQLTransaction()) { @@ -134,8 +134,8 @@ public: } return; } else if (transaction->HasReadTableTransaction()) { - NKikimrTxUserProxy::TEvProposeTransactionStatus status; - return ReplyWithResult(EResponseStatus::MSTATUS_ERROR, status, ctx); + NKikimrTxUserProxy::TEvProposeTransactionStatus status; + return ReplyWithResult(EResponseStatus::MSTATUS_ERROR, status, ctx); } ctx.Send(MakeTxProxyID(), Proposal.Release()); } @@ -157,7 +157,7 @@ bool TMessageBusServerRequest::RetryResolve(const TActorContext &ctx) { void TMessageBusServerRequest::ReplyWithResult(EResponseStatus status, NKikimrTxUserProxy::TEvProposeTransactionStatus &result, - const TActorContext &ctx) + const TActorContext &ctx) { TAutoPtr<TBusResponse> response(ProposeTransactionStatusToResponse(status, result)); @@ -175,9 +175,9 @@ void TMessageBusServerRequest::ReplyWithResult(EResponseStatus status, response->Record.MutableTxStats()->Swap(result.MutableTxStats()); } - SendReplyAutoPtr(response); + SendReplyAutoPtr(response); - FinishReply(ctx); + FinishReply(ctx); } void TMessageBusServerRequest::FinishReply(const TActorContext &ctx) @@ -207,7 +207,7 @@ void TMessageBusServerRequest::Handle(TMiniKQLCompileServiceEvents::TEvCompileSt pgm->SetBin(result.CompiledProgram); compileResults->SetCompiledProgram(result.CompiledProgram); } else { - NYql::IssuesToMessage(result.Errors, compileResults->MutableProgramCompileErrors()); + NYql::IssuesToMessage(result.Errors, compileResults->MutableProgramCompileErrors()); } --InFlyRequests; CompileResolveCookies = std::move(ev->Get()->CompileResolveCookies); @@ -221,7 +221,7 @@ void TMessageBusServerRequest::Handle(TMiniKQLCompileServiceEvents::TEvCompileSt params->SetBin(result.CompiledProgram); compileResults->SetCompiledParams(result.CompiledProgram); } else { - NYql::IssuesToMessage(result.Errors, compileResults->MutableParamsCompileErrors()); + NYql::IssuesToMessage(result.Errors, compileResults->MutableParamsCompileErrors()); } --InFlyRequests; } @@ -289,7 +289,7 @@ void TMessageBusServerRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatu case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAlready: // completion - return ReplyWithResult(MSTATUS_OK, msg->Record, ctx); + return ReplyWithResult(MSTATUS_OK, msg->Record, ctx); case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecTimeout: return ReplyWithResult(MSTATUS_INPROGRESS, msg->Record, ctx); case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyNotReady: @@ -328,10 +328,10 @@ void TMessageBusServerRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatu void TMessageBusServerRequest::Handle(TEvDataShard::TEvGetReadTableStreamStateRequest::TPtr &ev, const TActorContext &ctx) { auto *response = new TEvDataShard::TEvGetReadTableStreamStateResponse; - response->Record.MutableStatus()->SetCode(Ydb::StatusIds::GENERIC_ERROR); - auto *issue = response->Record.MutableStatus()->AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message("request proxy is not streaming"); + response->Record.MutableStatus()->SetCode(Ydb::StatusIds::GENERIC_ERROR); + auto *issue = response->Record.MutableStatus()->AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message("request proxy is not streaming"); ctx.Send(ev->Sender, response); } diff --git a/ydb/core/client/server/msgbus_server_s3_listing.cpp b/ydb/core/client/server/msgbus_server_s3_listing.cpp index 9646b436b3..3626866322 100644 --- a/ydb/core/client/server/msgbus_server_s3_listing.cpp +++ b/ydb/core/client/server/msgbus_server_s3_listing.cpp @@ -744,7 +744,7 @@ protected: if (!description.empty()) { GrpcRequest->RaiseIssue(NYql::TIssue(description)); } - GrpcRequest->ReplyWithYdbStatus(grpcStatus); + GrpcRequest->ReplyWithYdbStatus(grpcStatus); } else { Ydb::S3Internal::S3ListingResult grpcResult = TMessageConverter::ConvertResult(msgbusResponse); GrpcRequest->SendResult(grpcResult, Ydb::StatusIds::SUCCESS); diff --git a/ydb/core/client/server/msgbus_server_scheme_initroot.cpp b/ydb/core/client/server/msgbus_server_scheme_initroot.cpp index dfd0126cf7..a8538eea1c 100644 --- a/ydb/core/client/server/msgbus_server_scheme_initroot.cpp +++ b/ydb/core/client/server/msgbus_server_scheme_initroot.cpp @@ -82,7 +82,7 @@ public: if (WithRetry) { clientConfig.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries(); } - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, domain->SchemeRoot, clientConfig)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, domain->SchemeRoot, clientConfig)); NTabletPipe::SendData(ctx, PipeClient, x.Release()); } else { ReplyWithResult(MSTATUS_ERROR, TEvSchemeShard::TEvInitRootShardResult::StatusBadArgument, ctx); diff --git a/ydb/core/client/server/msgbus_tabletreq.h b/ydb/core/client/server/msgbus_tabletreq.h index fbc7b6e34e..f173ae9097 100644 --- a/ydb/core/client/server/msgbus_tabletreq.h +++ b/ydb/core/client/server/msgbus_tabletreq.h @@ -99,7 +99,7 @@ public: TabletId = reqPair.first; if (reqPair.first) { - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, reqPair.first, clientConfig)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, reqPair.first, clientConfig)); NTabletPipe::SendData(ctx, PipeClient, reqPair.second.Release()); this->Become(&TDerived::StateFunc, ctx, Timeout, new TEvents::TEvWakeup()); diff --git a/ydb/core/client/ya.make b/ydb/core/client/ya.make index f9fe0e2c96..cd39222ad1 100644 --- a/ydb/core/client/ya.make +++ b/ydb/core/client/ya.make @@ -1,9 +1,9 @@ LIBRARY() -OWNER( - ddoarn - g:kikimr -) +OWNER( + ddoarn + g:kikimr +) PEERDIR( library/cpp/grpc/server @@ -13,7 +13,7 @@ PEERDIR( ydb/core/engine ydb/public/lib/deprecated/kicli ) - + END() RECURSE( diff --git a/ydb/core/cms/cms.cpp b/ydb/core/cms/cms.cpp index e5ebf61af0..0a205024e3 100644 --- a/ydb/core/cms/cms.cpp +++ b/ydb/core/cms/cms.cpp @@ -1369,7 +1369,7 @@ void TCms::Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx } if (State->Config.SentinelConfig.Enable && !State->Sentinel) - State->Sentinel = RegisterWithSameMailbox(CreateSentinel(State)); + State->Sentinel = RegisterWithSameMailbox(CreateSentinel(State)); info->DebugDump(ctx); @@ -1777,25 +1777,25 @@ void TCms::Handle(TEvCms::TEvManageNotificationRequest::TPtr &ev, const TActorCo void TCms::Handle(TEvCms::TEvWalleCreateTaskRequest::TPtr &ev, const TActorContext &ctx) { auto adapter = CreateWalleAdapter(ev, SelfId()); - ctx.RegisterWithSameMailbox(adapter); + ctx.RegisterWithSameMailbox(adapter); } void TCms::Handle(TEvCms::TEvWalleListTasksRequest::TPtr &ev, const TActorContext &ctx) { auto adapter = CreateWalleAdapter(ev, State); - ctx.RegisterWithSameMailbox(adapter); + ctx.RegisterWithSameMailbox(adapter); } void TCms::Handle(TEvCms::TEvWalleCheckTaskRequest::TPtr &ev, const TActorContext &ctx) { auto adapter = CreateWalleAdapter(ev, State, SelfId()); - ctx.RegisterWithSameMailbox(adapter); + ctx.RegisterWithSameMailbox(adapter); } void TCms::Handle(TEvCms::TEvWalleRemoveTaskRequest::TPtr &ev, const TActorContext &ctx) { auto adapter = CreateWalleAdapter(ev, State, SelfId()); - ctx.RegisterWithSameMailbox(adapter); + ctx.RegisterWithSameMailbox(adapter); } void TCms::Handle(TEvCms::TEvStoreWalleTask::TPtr &ev, const TActorContext &ctx) diff --git a/ydb/core/cms/cms_tx_update_config.cpp b/ydb/core/cms/cms_tx_update_config.cpp index 9afe2eb474..59f962b3e2 100644 --- a/ydb/core/cms/cms_tx_update_config.cpp +++ b/ydb/core/cms/cms_tx_update_config.cpp @@ -55,7 +55,7 @@ public: if (Self->State->Config.SentinelConfig.Enable) { if (!Self->State->Sentinel) { - Self->State->Sentinel = Self->RegisterWithSameMailbox(CreateSentinel(Self->State)); + Self->State->Sentinel = Self->RegisterWithSameMailbox(CreateSentinel(Self->State)); } } else { if (Self->State->Sentinel) { diff --git a/ydb/core/cms/console/configs_cache.cpp b/ydb/core/cms/console/configs_cache.cpp index f6880aafb3..9f69ceacb1 100644 --- a/ydb/core/cms/console/configs_cache.cpp +++ b/ydb/core/cms/console/configs_cache.cpp @@ -97,7 +97,7 @@ void TConfigsCache::Bootstrap(const TActorContext &ctx) { } auto client = CreateConfigsSubscriber(SelfId(), kinds, CurrentConfig); - SubscriptionClient = ctx.RegisterWithSameMailbox(client); + SubscriptionClient = ctx.RegisterWithSameMailbox(client); Become(&TThis::StateWork); } diff --git a/ydb/core/cms/console/console.cpp b/ydb/core/cms/console/console.cpp index d649ef8cb5..fe477202af 100644 --- a/ydb/core/cms/console/console.cpp +++ b/ydb/core/cms/console/console.cpp @@ -26,12 +26,12 @@ void TConsole::OnActivateExecutor(const TActorContext &ctx) TValidatorsRegistry::Instance()->LockValidators(); ConfigsManager = new TConfigsManager(*this); - ctx.RegisterWithSameMailbox(ConfigsManager); + ctx.RegisterWithSameMailbox(ConfigsManager); TenantsManager = new TTenantsManager(*this, domains->Domains.at(domainId), Counters, AppData()->FeatureFlags); - ctx.RegisterWithSameMailbox(TenantsManager); + ctx.RegisterWithSameMailbox(TenantsManager); if (AppData(ctx)->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()) { NetClassifierUpdaterId = ctx.Register(NNetClassifierUpdater::MakeNetClassifierUpdaterActor(SelfId())); diff --git a/ydb/core/cms/console/console__alter_tenant.cpp b/ydb/core/cms/console/console__alter_tenant.cpp index a985883517..dc44c25082 100644 --- a/ydb/core/cms/console/console__alter_tenant.cpp +++ b/ydb/core/cms/console/console__alter_tenant.cpp @@ -23,8 +23,8 @@ public: operation.set_ready(true); operation.set_status(code); auto issue = operation.add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(error); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(error); Tenant = nullptr; @@ -197,27 +197,27 @@ public: return Error(Ydb::StatusIds::BAD_REQUEST, "Data size soft quota cannot be larger than hard quota", ctx); } } - - // Check attributes. - THashSet<TString> attrNames; - for (const auto& [key, value] : rec.alter_attributes()) { - if (!key) - return Error(Ydb::StatusIds::BAD_REQUEST, - "Attribute name shouldn't be empty", ctx); - if (attrNames.contains(key)) - return Error(Ydb::StatusIds::BAD_REQUEST, - Sprintf("Multiple attributes with name '%s'", key.data()), ctx); - attrNames.insert(key); - } - - THashMap<TString, ui64> attributeMap; - for (ui64 i = 0 ; i < Tenant->Attributes.UserAttributesSize(); i++) { - bool res = attributeMap.emplace(Tenant->Attributes.GetUserAttributes(i).GetKey(), i).second; - if (!res) - return Error(Ydb::StatusIds::INTERNAL_ERROR, - "Unexpected duplicate attribute found in CMS local db", ctx); - } - + + // Check attributes. + THashSet<TString> attrNames; + for (const auto& [key, value] : rec.alter_attributes()) { + if (!key) + return Error(Ydb::StatusIds::BAD_REQUEST, + "Attribute name shouldn't be empty", ctx); + if (attrNames.contains(key)) + return Error(Ydb::StatusIds::BAD_REQUEST, + Sprintf("Multiple attributes with name '%s'", key.data()), ctx); + attrNames.insert(key); + } + + THashMap<TString, ui64> attributeMap; + for (ui64 i = 0 ; i < Tenant->Attributes.UserAttributesSize(); i++) { + bool res = attributeMap.emplace(Tenant->Attributes.GetUserAttributes(i).GetKey(), i).second; + if (!res) + return Error(Ydb::StatusIds::INTERNAL_ERROR, + "Unexpected duplicate attribute found in CMS local db", ctx); + } + // Apply computational units changes. if (rec.computational_units_to_add_size() || rec.computational_units_to_remove_size()) { ComputationalUnitsModified = true; @@ -273,27 +273,27 @@ public: Self->DbUpdateTenantAlterIdempotencyKey(Tenant, Tenant->AlterIdempotencyKey, txc, ctx); } - if (!rec.alter_attributes().empty()) { - for (const auto& [key, value] : rec.alter_attributes()) { - const auto it = attributeMap.find(key); - if (it != attributeMap.end()) { - if (value) { - Tenant->Attributes.MutableUserAttributes(it->second)->SetValue(value); - } else { - Tenant->Attributes.MutableUserAttributes(it->second)->ClearValue(); - } - } else { - auto &r = *Tenant->Attributes.AddUserAttributes(); - r.SetKey(key); - if (value) { - r.SetValue(value); - } - } - } - Self->DbUpdateTenantUserAttributes(Tenant, Tenant->Attributes, txc, ctx); - updateSubdomainVersion = true; - } - + if (!rec.alter_attributes().empty()) { + for (const auto& [key, value] : rec.alter_attributes()) { + const auto it = attributeMap.find(key); + if (it != attributeMap.end()) { + if (value) { + Tenant->Attributes.MutableUserAttributes(it->second)->SetValue(value); + } else { + Tenant->Attributes.MutableUserAttributes(it->second)->ClearValue(); + } + } else { + auto &r = *Tenant->Attributes.AddUserAttributes(); + r.SetKey(key); + if (value) { + r.SetValue(value); + } + } + } + Self->DbUpdateTenantUserAttributes(Tenant, Tenant->Attributes, txc, ctx); + updateSubdomainVersion = true; + } + if (updateSubdomainVersion) { SubdomainVersion = Tenant->SubdomainVersion + 1; Self->DbUpdateSubdomainVersion(Tenant, *SubdomainVersion, txc, ctx); diff --git a/ydb/core/cms/console/console__create_tenant.cpp b/ydb/core/cms/console/console__create_tenant.cpp index ddb57d902f..77d8166d54 100644 --- a/ydb/core/cms/console/console__create_tenant.cpp +++ b/ydb/core/cms/console/console__create_tenant.cpp @@ -34,8 +34,8 @@ public: operation.set_ready(true); operation.set_status(code); auto issue = operation.add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(error); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(error); Tenant = nullptr; @@ -109,17 +109,17 @@ public: // Check attributes. THashSet<TString> attrNames; - for (const auto& [key, value] : rec.attributes()) { - if (!key) + for (const auto& [key, value] : rec.attributes()) { + if (!key) return Error(Ydb::StatusIds::BAD_REQUEST, "Attribute name shouldn't be empty", ctx); - if (!value) + if (!value) return Error(Ydb::StatusIds::BAD_REQUEST, "Attribute value shouldn't be empty", ctx); - if (attrNames.contains(key)) + if (attrNames.contains(key)) return Error(Ydb::StatusIds::BAD_REQUEST, - Sprintf("Multiple attributes with name '%s'", key.data()), ctx); - attrNames.insert(key); + Sprintf("Multiple attributes with name '%s'", key.data()), ctx); + attrNames.insert(key); } Tenant = new TTenant(path, TTenant::CREATING_POOLS, token); @@ -152,10 +152,10 @@ public: Tenant->StorageUnitsQuota = Self->Config.DefaultStorageUnitsQuota; Tenant->ComputationalUnitsQuota = Self->Config.DefaultComputationalUnitsQuota; - for (const auto& [key, value] : rec.attributes()) { + for (const auto& [key, value] : rec.attributes()) { auto &r = *Tenant->Attributes.AddUserAttributes(); - r.SetKey(key); - r.SetValue(value); + r.SetKey(key); + r.SetValue(value); } switch (rec.resources_kind_case()) { diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index f1cdb32791..4ba6f46033 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -296,7 +296,7 @@ private: TActorId OwnerId; ui64 NextOrder; - + public: TSubscriptionClientSender(TInMemorySubscription::TPtr subscription, const TActorId &ownerId) : Subscription(subscription) @@ -570,7 +570,7 @@ void TConfigsProvider::CheckSubscription(TSubscription::TPtr subscription, worker = new TServiceConfigSender(subscription, ctx.SelfID); else worker = new TTabletConfigSender(subscription, ctx.SelfID); - subscription->Worker = ctx.RegisterWithSameMailbox(worker); + subscription->Worker = ctx.RegisterWithSameMailbox(worker); } void TConfigsProvider::CheckSubscription(TInMemorySubscription::TPtr subscription, @@ -694,7 +694,7 @@ void TConfigsProvider::Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev "TConfigsProvider registered new subscription " << subscriber.ToString() << ":" << rec.GetGeneration()); - subscription->Worker = RegisterWithSameMailbox(new TSubscriptionClientSender(subscription, SelfId())); + subscription->Worker = RegisterWithSameMailbox(new TSubscriptionClientSender(subscription, SelfId())); CheckSubscription(subscription, ctx); } diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index afa6a820ee..0db3f5dced 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -375,7 +375,7 @@ public: enum EAction { CREATE, CONFIGURE, - CONFIGURE_ATTR, + CONFIGURE_ATTR, GET_KEY, REMOVE }; @@ -513,30 +513,30 @@ public: } } - void AlterUserAttribute(const TActorContext &ctx) { - BLOG_D("TSubDomainManip(" << Tenant->Path << ") alter user attribute "); - auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - - request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); - request->Record.SetExecTimeoutPeriod(Max<ui64>()); - - if (Tenant->UserToken.GetUserSID()) - request->Record.SetUserToken(Tenant->UserToken.SerializeAsString()); - - auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); - tx.SetWorkingDir(Subdomain.first); - + void AlterUserAttribute(const TActorContext &ctx) { + BLOG_D("TSubDomainManip(" << Tenant->Path << ") alter user attribute "); + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + + request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + + if (Tenant->UserToken.GetUserSID()) + request->Record.SetUserToken(Tenant->UserToken.SerializeAsString()); + + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + tx.SetWorkingDir(Subdomain.first); + tx.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes); - - tx.MutableAlterUserAttributes()->CopyFrom(Tenant->Attributes); - tx.MutableAlterUserAttributes()->SetPathName(Subdomain.second); - - BLOG_TRACE("TSubdomainManip(" << Tenant->Path << ") send alter user attribute cmd: " - << request->ToString()); - - ctx.Send(MakeTxProxyID(), request.Release()); - } - + + tx.MutableAlterUserAttributes()->CopyFrom(Tenant->Attributes); + tx.MutableAlterUserAttributes()->SetPathName(Subdomain.second); + + BLOG_TRACE("TSubdomainManip(" << Tenant->Path << ") send alter user attribute cmd: " + << request->ToString()); + + ctx.Send(MakeTxProxyID(), request.Release()); + } + void AlterSubdomain(const TActorContext &ctx) { BLOG_D("TSubDomainManip(" << Tenant->Path << ") alter subdomain version " << Version); @@ -544,15 +544,15 @@ public: auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); request->Record.SetExecTimeoutPeriod(Max<ui64>()); - + if (Tenant->UserToken.GetUserSID()) request->Record.SetUserToken(Tenant->UserToken.SerializeAsString()); - + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); - tx.SetWorkingDir(Subdomain.first); - - FillSubdomainAlterInfo(*tx.MutableSubDomain(), true); - + tx.SetWorkingDir(Subdomain.first); + + FillSubdomainAlterInfo(*tx.MutableSubDomain(), true); + if (Tenant->IsExternalSubdomain) { tx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterExtSubDomain); } else { @@ -649,7 +649,7 @@ public: BLOG_D("TSubdomainManip(" << Tenant->Path << ") done"); if (Action == CREATE) ReplyAndDie(new TTenantsManager::TEvPrivate::TEvSubdomainCreated(Tenant, SchemeshardId, PathId), ctx); - else if (Action == CONFIGURE || Action == CONFIGURE_ATTR) + else if (Action == CONFIGURE || Action == CONFIGURE_ATTR) ReplyAndDie(new TTenantsManager::TEvPrivate::TEvSubdomainReady(Tenant, Version), ctx); else if (Action == GET_KEY) ReplyAndDie(new TTenantsManager::TEvPrivate::TEvSubdomainKey(Tenant, SchemeshardId, PathId), ctx); @@ -737,12 +737,12 @@ public: BLOG_D("TSubdomainManip(" << Tenant->Path << ") got TEvNotifyTxCompletionResult: " << ev->Get()->Record.ShortDebugString()); - if (Action == CONFIGURE && Tenant->Attributes.UserAttributesSize()) { - AlterUserAttribute(ctx); - Action = CONFIGURE_ATTR; - } else { - ActionFinished(ctx); - } + if (Action == CONFIGURE && Tenant->Attributes.UserAttributesSize()) { + AlterUserAttribute(ctx); + Action = CONFIGURE_ATTR; + } else { + ActionFinished(ctx); + } } void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { @@ -764,12 +764,12 @@ public: switch (rec.GetStatus()) { case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: TabletId = rec.GetSchemeShardTabletId(); - if (Action == CONFIGURE && Tenant->Attributes.UserAttributesSize()) { - AlterUserAttribute(ctx); - Action = CONFIGURE_ATTR; - } else { - ActionFinished(ctx); - } + if (Action == CONFIGURE && Tenant->Attributes.UserAttributesSize()) { + AlterUserAttribute(ctx); + Action = CONFIGURE_ATTR; + } else { + ActionFinished(ctx); + } break; case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: TxId = rec.GetTxId(); @@ -1762,7 +1762,7 @@ void TTenantsManager::AllocateTenantPools(TTenant::TPtr tenant, const TActorCont for (auto &pr : tenant->StoragePools) { if (pr.second->State != TStoragePool::ALLOCATED && !pr.second->Worker) - pr.second->Worker = ctx.RegisterWithSameMailbox(new TPoolManip(SelfId(), Domain, tenant, pr.second, TPoolManip::ALLOCATE)); + pr.second->Worker = ctx.RegisterWithSameMailbox(new TPoolManip(SelfId(), Domain, tenant, pr.second, TPoolManip::ALLOCATE)); } } @@ -1775,7 +1775,7 @@ void TTenantsManager::DeleteTenantPools(TTenant::TPtr tenant, const TActorContex for (auto &pr : tenant->StoragePools) { if (pr.second->State != TStoragePool::DELETED) - pr.second->Worker = ctx.RegisterWithSameMailbox(new TPoolManip(SelfId(), Domain, tenant, pr.second, TPoolManip::DEALLOCATE)); + pr.second->Worker = ctx.RegisterWithSameMailbox(new TPoolManip(SelfId(), Domain, tenant, pr.second, TPoolManip::DEALLOCATE)); } } @@ -1843,21 +1843,21 @@ void TTenantsManager::RetryResourcesRequests(const TActorContext &ctx) } } -void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status) +void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status) { status.set_path(tenant->Path); if (tenant->IsRunning() && tenant->SubdomainVersion != tenant->ConfirmedSubdomain) status.set_state(Ydb::Cms::GetDatabaseStatusResult::CONFIGURING); else if (tenant->IsRunning()) - status.set_state(Ydb::Cms::GetDatabaseStatusResult::RUNNING); + status.set_state(Ydb::Cms::GetDatabaseStatusResult::RUNNING); else if (tenant->IsConfiguring()) status.set_state(Ydb::Cms::GetDatabaseStatusResult::PENDING_RESOURCES); else if (tenant->IsCreating()) - status.set_state(Ydb::Cms::GetDatabaseStatusResult::CREATING); + status.set_state(Ydb::Cms::GetDatabaseStatusResult::CREATING); else if (tenant->IsRemoving()) - status.set_state(Ydb::Cms::GetDatabaseStatusResult::REMOVING); + status.set_state(Ydb::Cms::GetDatabaseStatusResult::REMOVING); else - status.set_state(Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED); + status.set_state(Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED); auto resources = tenant->AreResourcesShared ? status.mutable_required_shared_resources() : @@ -1905,7 +1905,7 @@ void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDataba } } -void TTenantsManager::FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, +void TTenantsManager::FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, const NKikimrTenantSlotBroker::TTenantState &slots) { THashMap<TSlotDescription, ui64> allocated; @@ -1948,7 +1948,7 @@ void TTenantsManager::CheckSubDomainKey(TTenant::TPtr tenant, return; auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::GET_KEY); - tenant->Worker = ctx.RegisterWithSameMailbox(actor); + tenant->Worker = ctx.RegisterWithSameMailbox(actor); } void TTenantsManager::ConfigureTenantSubDomain(TTenant::TPtr tenant, const TActorContext &ctx) @@ -1958,7 +1958,7 @@ void TTenantsManager::ConfigureTenantSubDomain(TTenant::TPtr tenant, const TActo && !tenant->Worker) { auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CONFIGURE, GetTenant(tenant->SharedDomainId)); - tenant->Worker = ctx.RegisterWithSameMailbox(actor); + tenant->Worker = ctx.RegisterWithSameMailbox(actor); } } @@ -1967,7 +1967,7 @@ void TTenantsManager::CreateTenantSubDomain(TTenant::TPtr tenant, const TActorCo Y_VERIFY(tenant->State == TTenant::CREATING_SUBDOMAIN); Y_VERIFY(!tenant->Worker); auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CREATE, GetTenant(tenant->SharedDomainId)); - tenant->Worker = ctx.RegisterWithSameMailbox(actor); + tenant->Worker = ctx.RegisterWithSameMailbox(actor); } void TTenantsManager::DeleteTenantSubDomain(TTenant::TPtr tenant, const TActorContext &ctx) @@ -1975,7 +1975,7 @@ void TTenantsManager::DeleteTenantSubDomain(TTenant::TPtr tenant, const TActorCo Y_VERIFY(tenant->State == TTenant::REMOVING_SUBDOMAIN); if (!tenant->Worker) { auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::REMOVE); - tenant->Worker = ctx.RegisterWithSameMailbox(actor); + tenant->Worker = ctx.RegisterWithSameMailbox(actor); } } @@ -2010,7 +2010,7 @@ void TTenantsManager::ProcessTenantActions(TTenant::TPtr tenant, const TActorCon } TTenantsManager::TTenant::TPtr TTenantsManager::FillOperationStatus(const TString &id, - Ydb::Operations::Operation &operation) + Ydb::Operations::Operation &operation) { operation.set_id(id); @@ -2763,20 +2763,20 @@ void TTenantsManager::DbUpdateTenantAlterIdempotencyKey(TTenant::TPtr tenant, .Update<Schema::Tenants::AlterIdempotencyKey>(idempotencyKey); } -void TTenantsManager::DbUpdateTenantUserAttributes(TTenant::TPtr tenant, +void TTenantsManager::DbUpdateTenantUserAttributes(TTenant::TPtr tenant, const NKikimrSchemeOp::TAlterUserAttributes &attributes, - TTransactionContext &txc, - const TActorContext &ctx) -{ - LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, - "Update alter user attributes for " << tenant->Path - << " userAttributes=" << attributes.ShortDebugString()); - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Tenants>().Key(tenant->Path) - .Update<Schema::Tenants::Attributes>(attributes); -} - + TTransactionContext &txc, + const TActorContext &ctx) +{ + LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, + "Update alter user attributes for " << tenant->Path + << " userAttributes=" << attributes.ShortDebugString()); + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::Tenants>().Key(tenant->Path) + .Update<Schema::Tenants::Attributes>(attributes); +} + void TTenantsManager::DbUpdateTenantGeneration(TTenant::TPtr tenant, ui64 generation, TTransactionContext &txc, @@ -2997,7 +2997,7 @@ void TTenantsManager::Handle(TEvConsole::TEvGetTenantStatusRequest::TPtr &ev, co if (!tenant) { operation.set_status(Ydb::StatusIds::NOT_FOUND); auto issue = operation.add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_severity(NYql::TSeverityIds::S_ERROR); issue->set_message(Sprintf("Unknown tenant %s", path.data())); LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, @@ -3009,7 +3009,7 @@ void TTenantsManager::Handle(TEvConsole::TEvGetTenantStatusRequest::TPtr &ev, co ctx.Send(ev->Sender, resp.Release(), 0, ev->Cookie); } else if (!tenant->IsRunning() && !tenant->IsConfiguring()) { - Ydb::Cms::GetDatabaseStatusResult result; + Ydb::Cms::GetDatabaseStatusResult result; FillTenantStatus(tenant, result); operation.set_status(Ydb::StatusIds::SUCCESS); @@ -3034,7 +3034,7 @@ void TTenantsManager::Handle(TEvConsole::TEvListTenantsRequest::TPtr &ev, const { Counters.Inc(COUNTER_LIST_REQUESTS); - Ydb::Cms::ListDatabasesResult result; + Ydb::Cms::ListDatabasesResult result; for (auto &pr : Tenants) result.add_paths(pr.first); @@ -3054,7 +3054,7 @@ void TTenantsManager::Handle(TEvConsole::TEvListTenantsRequest::TPtr &ev, const void TTenantsManager::Handle(TEvConsole::TEvNotifyOperationCompletionRequest::TPtr &ev, const TActorContext &ctx) { auto &rec = ev->Get()->Record; - Ydb::Operations::Operation operation; + Ydb::Operations::Operation operation; auto tenant = FillOperationStatus(rec.GetRequest().id(), operation); if (operation.ready() && operation.status() != Ydb::StatusIds::NOT_FOUND) { @@ -3360,7 +3360,7 @@ void TTenantsManager::Handle(TEvTenantSlotBroker::TEvTenantState::TPtr &ev, cons } for (auto &req : tenant->StatusRequests) { - Ydb::Cms::GetDatabaseStatusResult result; + Ydb::Cms::GetDatabaseStatusResult result; FillTenantStatus(tenant, result); FillTenantAllocatedSlots(tenant, result, rec); diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index 55f599916a..a21615dd62 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -785,15 +785,15 @@ public: void RequestTenantSlotsStats(const TActorContext &ctx); void RetryResourcesRequests(const TActorContext &ctx); - void FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status); - void FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, + void FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status); + void FillTenantAllocatedSlots(TTenant::TPtr tenant, Ydb::Cms::GetDatabaseStatusResult &status, const NKikimrTenantSlotBroker::TTenantState &slots); void CheckSubDomainKey(TTenant::TPtr tenant, const TActorContext &ctx); void ConfigureTenantSubDomain(TTenant::TPtr tenant, const TActorContext &ctx); void CreateTenantSubDomain(TTenant::TPtr tenant, const TActorContext &ctx); void DeleteTenantSubDomain(TTenant::TPtr tenant, const TActorContext &ctx); void ProcessTenantActions(TTenant::TPtr tenant, const TActorContext &ctx); - TTenant::TPtr FillOperationStatus(const TString &id, Ydb::Operations::Operation &operation); + TTenant::TPtr FillOperationStatus(const TString &id, Ydb::Operations::Operation &operation); void SendTenantNotifications(TTenant::TPtr tenant, TTenant::EAction action, Ydb::StatusIds::StatusCode code, const TActorContext &ctx); void DumpStateHTML(IOutputStream &os); @@ -866,10 +866,10 @@ public: const TString &idempotencyKey, TTransactionContext &txc, const TActorContext &ctx); - void DbUpdateTenantUserAttributes(TTenant::TPtr tenant, + void DbUpdateTenantUserAttributes(TTenant::TPtr tenant, const NKikimrSchemeOp::TAlterUserAttributes &attributes, - TTransactionContext &txc, - const TActorContext &ctx); + TTransactionContext &txc, + const TActorContext &ctx); void DbUpdateTenantGeneration(TTenant::TPtr tenant, ui64 generation, TTransactionContext &txc, diff --git a/ydb/core/cms/console/console_ut_tenants.cpp b/ydb/core/cms/console/console_ut_tenants.cpp index e918ed3dd1..57e4820af7 100644 --- a/ydb/core/cms/console/console_ut_tenants.cpp +++ b/ydb/core/cms/console/console_ut_tenants.cpp @@ -121,8 +121,8 @@ void CheckAlterTenantSlots(TTenantTestRuntime &runtime, const TString &path, ui64 generation, Ydb::StatusIds::StatusCode code, TVector<TSlotRequest> add, TVector<TSlotRequest> remove, - const TString &idempotencyKey = TString(), - const TVector<std::pair<TString, TString>> attrs = {}) + const TString &idempotencyKey = TString(), + const TVector<std::pair<TString, TString>> attrs = {}) { auto *event = new TEvConsole::TEvAlterTenantRequest; event->Record.MutableRequest()->set_path(path); @@ -139,9 +139,9 @@ void CheckAlterTenantSlots(TTenantTestRuntime &runtime, const TString &path, unit.set_availability_zone(slot.Zone); unit.set_count(slot.Count); } - for (const auto& [key, value] : attrs) { - (*event->Record.MutableRequest()->mutable_alter_attributes())[key] = value; - } + for (const auto& [key, value] : attrs) { + (*event->Record.MutableRequest()->mutable_alter_attributes())[key] = value; + } if (idempotencyKey) { event->Record.MutableRequest()->set_idempotency_key(idempotencyKey); } @@ -235,7 +235,7 @@ void CheckListTenants(TTenantTestRuntime &runtime, TVector<TString> tenants) TAutoPtr<IEventHandle> handle; runtime.SendToConsole(event); auto reply = runtime.GrabEdgeEventRethrow<TEvConsole::TEvListTenantsResponse>(handle); - Ydb::Cms::ListDatabasesResult result; + Ydb::Cms::ListDatabasesResult result; reply->Record.GetResponse().operation().result().UnpackTo(&result); for (auto &tenant : result.paths()) { UNIT_ASSERT(paths.contains(tenant)); @@ -666,7 +666,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 10, 10, 10}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, shared, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, SLOT1_TYPE, ZONE1, 3, 3, SLOT2_TYPE, ZONE1, 2, 2, @@ -844,7 +844,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { void RunTestGetUnknownTenantStatus(TTenantTestRuntime& runtime) { CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); CheckCounter(runtime, {}, TTenantsManager::COUNTER_STATUS_REQUESTS, 1); CheckCounter(runtime, {{ {"status", "NOT_FOUND"} }}, TTenantsManager::COUNTER_STATUS_RESPONSES, 1); @@ -879,7 +879,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 10, 10, 10}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, SLOT2_TYPE, ZONE_ANY, 8, 5); RestartConsole(runtime); @@ -891,7 +891,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 16, 16, 16}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, SLOT2_TYPE, ZONE_ANY, 8, 8); CheckCounter(runtime, {}, TTenantsManager::COUNTER_TENANTS, 1); @@ -947,7 +947,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 45, 45, 45}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, SLOT1_TYPE, ZONE1, 5, 5, SLOT2_TYPE, ZONE1, 8, 8, SLOT3_TYPE, ZONE1, 10, 8); @@ -961,7 +961,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 27, 27, 27}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, SLOT1_TYPE, ZONE1, 10, 8, SLOT2_TYPE, ZONE1, 5, 5, SLOT3_TYPE, ZONE1, 3, 3); @@ -975,7 +975,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { runtime.WaitForHiveState({{{DOMAIN1_NAME, 8, 8, 8}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}); + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}); CheckAlterTenantSlots(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{ {SLOT1_TYPE, ZONE1, 1}, @@ -987,7 +987,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {TENANT1_1_NAME, 14, 14, 14}}}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}}, {}, SLOT1_TYPE, ZONE1, 1, 1, SLOT2_TYPE, ZONE1, 2, 2, SLOT3_TYPE, ZONE1, 3, 3); @@ -1545,7 +1545,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND); @@ -1863,7 +1863,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {}); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}, {"hdd-1", 1, 1}}, {{"host2", 2, "kind2"}}, SLOT1_TYPE, ZONE1, 1, 1); @@ -2029,7 +2029,7 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { } - bool CheckAttrsPresent(TTenantTestRuntime& runtime, const TString& tenantName, THashMap<TString, TString> attrs, bool skipAbsent = false) { + bool CheckAttrsPresent(TTenantTestRuntime& runtime, const TString& tenantName, THashMap<TString, TString> attrs, bool skipAbsent = false) { auto request = MakeHolder<TEvSchemeShard::TEvDescribeScheme>(tenantName); ForwardToTablet(runtime, SCHEME_SHARD1_ID, runtime.Sender, request.Release()); @@ -2037,33 +2037,33 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { auto reply = runtime.GrabEdgeEvent<TEvSchemeShard::TEvDescribeSchemeResult>(handle); Cerr << "Reply: " << reply->GetRecord().DebugString() << "\n"; for (auto &attr : reply->GetRecord().GetPathDescription().GetUserAttributes()) { - if (!skipAbsent) { - UNIT_ASSERT(attrs.contains(attr.GetKey())); - UNIT_ASSERT_VALUES_EQUAL(attrs.at(attr.GetKey()), attr.GetValue()); - } + if (!skipAbsent) { + UNIT_ASSERT(attrs.contains(attr.GetKey())); + UNIT_ASSERT_VALUES_EQUAL(attrs.at(attr.GetKey()), attr.GetValue()); + } attrs.erase(attr.GetKey()); } - return attrs.empty(); - } - - void RunTestAttributes(TTenantTestRuntime& runtime) { - // Create tenant with attrs and check subdomain has them attached. - runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::TX_COORDINATOR, NActors::NLog::PRI_DEBUG); - CheckCreateTenant(runtime, TENANT1_1_NAME, - Ydb::StatusIds::SUCCESS, - {{"hdd", 1}, {"hdd-1", 3}}, - TVector<std::pair<TString, TString>>({{"name1", "value1"}, {"name2", "value2"}}), - SLOT1_TYPE, ZONE1, 1); - - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}, {"hdd-1", 3, 3}}, {}, - SLOT1_TYPE, ZONE1, 1, 1); - - - UNIT_ASSERT(CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}})); - + return attrs.empty(); + } + + void RunTestAttributes(TTenantTestRuntime& runtime) { + // Create tenant with attrs and check subdomain has them attached. + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::TX_COORDINATOR, NActors::NLog::PRI_DEBUG); + CheckCreateTenant(runtime, TENANT1_1_NAME, + Ydb::StatusIds::SUCCESS, + {{"hdd", 1}, {"hdd-1", 3}}, + TVector<std::pair<TString, TString>>({{"name1", "value1"}, {"name2", "value2"}}), + SLOT1_TYPE, ZONE1, 1); + + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}, {"hdd-1", 3, 3}}, {}, + SLOT1_TYPE, ZONE1, 1, 1); + + + UNIT_ASSERT(CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}})); + // Requests with wrong attributes. CheckCreateTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::BAD_REQUEST, @@ -2073,57 +2073,57 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { Ydb::StatusIds::BAD_REQUEST, {{"hdd", 1}, {"hdd-1", 3}}, TVector<std::pair<TString, TString>>({{"name1", ""}})); - - CheckAlterTenantSlots(runtime, TENANT1_1_NAME, 0, - Ydb::StatusIds::SUCCESS, {}, {}, - TString(), - TVector<std::pair<TString, TString>>({{"name2", "value2_2"},{"name3", "value3"}})); - - int cnt = 1000; - while (--cnt) { - if (CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name3", "value3"}}, true)) - break; - TDispatchOptions options; - runtime.DispatchEvents(options, TDuration::MilliSeconds(100)); - } - - UNIT_ASSERT_C(cnt, "attribute was has not been found"); - - CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2_2"}, {"name3", "value3"}}); - } - - void RunTestRemoveAttributes(TTenantTestRuntime& runtime) { - // Create tenant with attrs and check subdomain has them attached. - runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::TX_COORDINATOR, NActors::NLog::PRI_DEBUG); - CheckCreateTenant(runtime, TENANT1_1_NAME, - Ydb::StatusIds::SUCCESS, + + CheckAlterTenantSlots(runtime, TENANT1_1_NAME, 0, + Ydb::StatusIds::SUCCESS, {}, {}, + TString(), + TVector<std::pair<TString, TString>>({{"name2", "value2_2"},{"name3", "value3"}})); + + int cnt = 1000; + while (--cnt) { + if (CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name3", "value3"}}, true)) + break; + TDispatchOptions options; + runtime.DispatchEvents(options, TDuration::MilliSeconds(100)); + } + + UNIT_ASSERT_C(cnt, "attribute was has not been found"); + + CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2_2"}, {"name3", "value3"}}); + } + + void RunTestRemoveAttributes(TTenantTestRuntime& runtime) { + // Create tenant with attrs and check subdomain has them attached. + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::TX_COORDINATOR, NActors::NLog::PRI_DEBUG); + CheckCreateTenant(runtime, TENANT1_1_NAME, + Ydb::StatusIds::SUCCESS, {{"hdd", 1}, {"hdd-1", 3}}, - TVector<std::pair<TString, TString>>({{"name1", "value1"}, {"name2", "value2"}}), - SLOT1_TYPE, ZONE1, 1); - - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}, {"hdd-1", 3, 3}}, {}, - SLOT1_TYPE, ZONE1, 1, 1); - - - UNIT_ASSERT(CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}})); - - CheckAlterTenantSlots(runtime, TENANT1_1_NAME, 0, - Ydb::StatusIds::SUCCESS, {}, {}, - TString(), - TVector<std::pair<TString, TString>>({{"name2", TString()}})); - - int cnt = 1000; - while (--cnt) { - if (!CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}}, true)) - break; - TDispatchOptions options; - runtime.DispatchEvents(options, TDuration::MilliSeconds(100)); - } - - CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}}); + TVector<std::pair<TString, TString>>({{"name1", "value1"}, {"name2", "value2"}}), + SLOT1_TYPE, ZONE1, 1); + + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}, {"hdd-1", 3, 3}}, {}, + SLOT1_TYPE, ZONE1, 1, 1); + + + UNIT_ASSERT(CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}})); + + CheckAlterTenantSlots(runtime, TENANT1_1_NAME, 0, + Ydb::StatusIds::SUCCESS, {}, {}, + TString(), + TVector<std::pair<TString, TString>>({{"name2", TString()}})); + + int cnt = 1000; + while (--cnt) { + if (!CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}, {"name2", "value2"}}, true)) + break; + TDispatchOptions options; + runtime.DispatchEvents(options, TDuration::MilliSeconds(100)); + } + + CheckAttrsPresent(runtime, TENANT1_1_NAME, THashMap<TString, TString> {{"name1", "value1"}}); } Y_UNIT_TEST(TestAttributes) { @@ -2136,16 +2136,16 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { RunTestAttributes(runtime); } - Y_UNIT_TEST(TestRemoveAttributes) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig()); - RunTestRemoveAttributes(runtime); - } - - Y_UNIT_TEST(TestRemoveAttributesExtSubdomain) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - RunTestRemoveAttributes(runtime); - } - + Y_UNIT_TEST(TestRemoveAttributes) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig()); + RunTestRemoveAttributes(runtime); + } + + Y_UNIT_TEST(TestRemoveAttributesExtSubdomain) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + RunTestRemoveAttributes(runtime); + } + Y_UNIT_TEST(TestTenantsQuota) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); diff --git a/ydb/core/cms/json_proxy.h b/ydb/core/cms/json_proxy.h index d480eb1099..185d0ce01b 100644 --- a/ydb/core/cms/json_proxy.h +++ b/ydb/core/cms/json_proxy.h @@ -72,7 +72,7 @@ public: NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 10}; - Pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tid, pipeConfig)); + Pipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tid, pipeConfig)); NTabletPipe::SendData(ctx, Pipe, request.Release()); TBase::Become(&TBase::TThis::StateWork, ctx, TDuration::Seconds(120), new TEvents::TEvWakeup()); diff --git a/ydb/core/cms/ui/datashard_info.js b/ydb/core/cms/ui/datashard_info.js index 4d7fc2ebb1..f1c8bad40f 100644 --- a/ydb/core/cms/ui/datashard_info.js +++ b/ydb/core/cms/ui/datashard_info.js @@ -37,10 +37,10 @@ function onDataShardInfoLoaded(data) { <td class="ds-info">LocalId</td> <td class="ds-info">${table.LocalId}</td> </tr> - <tr class="ds-info"> - <td class="ds-info">SchemaVersion</td> - <td class="ds-info">${table.SchemaVersion}</td> - </tr> + <tr class="ds-info"> + <td class="ds-info">SchemaVersion</td> + <td class="ds-info">${table.SchemaVersion}</td> + </tr> </tbody> </table> `; diff --git a/ydb/core/cms/walle_api_handler.cpp b/ydb/core/cms/walle_api_handler.cpp index 60eec46a7c..7b62f3275a 100644 --- a/ydb/core/cms/walle_api_handler.cpp +++ b/ydb/core/cms/walle_api_handler.cpp @@ -340,7 +340,7 @@ private: .MinRetryTime = TDuration::MilliSeconds(10), .MaxRetryTime = TDuration::Seconds(10), }; - CmsPipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, MakeCmsID(domain), pipeConfig)); + CmsPipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, MakeCmsID(domain), pipeConfig)); NTabletPipe::SendData(ctx, CmsPipe, ev); } diff --git a/ydb/core/cms/walle_create_task_adapter.cpp b/ydb/core/cms/walle_create_task_adapter.cpp index ea299c4b80..56989e2db0 100644 --- a/ydb/core/cms/walle_create_task_adapter.cpp +++ b/ydb/core/cms/walle_create_task_adapter.cpp @@ -51,7 +51,7 @@ public: } auto collector = CreateInfoCollector(SelfId(), TDuration::Seconds(15)); - ctx.RegisterWithSameMailbox(collector); + ctx.RegisterWithSameMailbox(collector); Become(&TThis::StateWork); } diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 02f9d006d3..450d4135c6 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -4,12 +4,12 @@ #include <ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h> #include <ydb/library/aclib/aclib.h> - + #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> - + #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> @@ -854,10 +854,10 @@ public: } NGrpc::TCallMeta meta; - if (config.SecurityToken) { - meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, config.SecurityToken}); - } - + if (config.SecurityToken) { + meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, config.SecurityToken}); + } + Ydb::Operations::Operation response; NGrpc::TResponseCallback<Ydb::Table::DescribeTableOptionsResponse> responseCb = [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Table::DescribeTableOptionsResponse &&resp) -> void { @@ -872,8 +872,8 @@ public: { NGrpc::TGRpcClientLow clientLow; Ydb::Table::DescribeTableOptionsRequest request; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(ClientConfig); - connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncDescribeTableOptions, meta); + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(ClientConfig); + connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncDescribeTableOptions, meta); } if (!res) { diff --git a/ydb/core/driver_lib/cli_config_base/ya.make b/ydb/core/driver_lib/cli_config_base/ya.make index 62a23895b4..feb09c8e7b 100644 --- a/ydb/core/driver_lib/cli_config_base/ya.make +++ b/ydb/core/driver_lib/cli_config_base/ya.make @@ -1,6 +1,6 @@ LIBRARY() -OWNER(g:kikimr) +OWNER(g:kikimr) SRCS( config_base.h diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 9b22de9b9c..6766dd3171 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -73,7 +73,7 @@ protected: TString Rack; ui32 Body; ui32 GRpcPort; - ui32 GRpcsPort; + ui32 GRpcsPort; TString GRpcPublicHost; ui32 GRpcPublicPort; ui32 GRpcsPublicPort; @@ -120,7 +120,7 @@ protected: Rack = ""; Body = 0; GRpcPort = 0; - GRpcsPort = 0; + GRpcsPort = 0; GRpcPublicHost = ""; GRpcPublicPort = 0; GRpcsPublicPort = 0; @@ -198,7 +198,7 @@ protected: config.Opts->AddLongOption("grpc-file", "gRPC config file").OptionalArgument("PATH"); config.Opts->AddLongOption("tenant-pool-file", "Tenant Pool service config file").OptionalArgument("PATH"); config.Opts->AddLongOption("grpc-port", "enable gRPC server on port").RequiredArgument("PORT").StoreResult(&GRpcPort); - config.Opts->AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT").StoreResult(&GRpcsPort); + config.Opts->AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT").StoreResult(&GRpcsPort); config.Opts->AddLongOption("grpc-public-host", "set public gRPC host for discovery").RequiredArgument("HOST").StoreResult(&GRpcPublicHost); config.Opts->AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT").StoreResult(&GRpcPublicPort); config.Opts->AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT").StoreResult(&GRpcsPublicPort); @@ -531,11 +531,11 @@ protected: conf.SetStartGRpcProxy(true); conf.SetPort(GRpcPort); } - if (GRpcsPort) { - auto& conf = *AppConfig.MutableGRpcConfig(); - conf.SetStartGRpcProxy(true); + if (GRpcsPort) { + auto& conf = *AppConfig.MutableGRpcConfig(); + conf.SetStartGRpcProxy(true); conf.SetSslPort(GRpcsPort); - } + } if (GRpcPublicHost) { auto& conf = *AppConfig.MutableGRpcConfig(); conf.SetPublicHost(GRpcPublicHost); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp index 1f1d406e00..db44a3a516 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp @@ -4,7 +4,7 @@ #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_cms_v1.grpc.pb.h> @@ -69,7 +69,7 @@ public: << " (" << response.GetStatus().GetReason() << ")" << Endl; } - virtual void PrintResponse(const Ydb::Operations::Operation &response) + virtual void PrintResponse(const Ydb::Operations::Operation &response) { if (response.status() == Ydb::StatusIds::SUCCESS) Cout << "OK" << Endl; @@ -149,7 +149,7 @@ public: int Run(TConfig &config) override { - Ydb::Operations::Operation response; + Ydb::Operations::Operation response; int res; res = DoGRpcRequest<TService, TRequest, TResponse, TFunction> @@ -164,11 +164,11 @@ public: }; class TClientCommandTenantList - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::ListDatabasesRequest, - Ydb::Cms::ListDatabasesResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncListDatabases), - &Ydb::Cms::V1::CmsService::Stub::AsyncListDatabases> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::ListDatabasesRequest, + Ydb::Cms::ListDatabasesResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncListDatabases), + &Ydb::Cms::V1::CmsService::Stub::AsyncListDatabases> { public: TClientCommandTenantList() : TTenantClientGRpcCommand("list", {}, "List existing databases") @@ -176,12 +176,12 @@ public: using TTenantClientGRpcCommand::PrintResponse; - void PrintResponse(const Ydb::Operations::Operation &response) override + void PrintResponse(const Ydb::Operations::Operation &response) override { if (response.status() != Ydb::StatusIds::SUCCESS) { TTenantClientGRpcCommand::PrintResponse(response); } else { - Ydb::Cms::ListDatabasesResult result; + Ydb::Cms::ListDatabasesResult result; Y_VERIFY(response.result().UnpackTo(&result)); Cout << "Databases:" << Endl; @@ -192,11 +192,11 @@ public: }; class TClientCommandTenantOptions - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, Ydb::Cms::DescribeDatabaseOptionsRequest, Ydb::Cms::DescribeDatabaseOptionsResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncDescribeDatabaseOptions), - &Ydb::Cms::V1::CmsService::Stub::AsyncDescribeDatabaseOptions> { + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncDescribeDatabaseOptions), + &Ydb::Cms::V1::CmsService::Stub::AsyncDescribeDatabaseOptions> { public: TClientCommandTenantOptions() : TTenantClientGRpcCommand("options", {}, "Describe available database options") @@ -236,11 +236,11 @@ public: }; class TClientCommandTenantStatus - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::GetDatabaseStatusRequest, - Ydb::Cms::GetDatabaseStatusResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncGetDatabaseStatus), - &Ydb::Cms::V1::CmsService::Stub::AsyncGetDatabaseStatus> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::GetDatabaseStatusRequest, + Ydb::Cms::GetDatabaseStatusResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncGetDatabaseStatus), + &Ydb::Cms::V1::CmsService::Stub::AsyncGetDatabaseStatus> { public: TClientCommandTenantStatus() : TTenantClientGRpcCommand("status", {}, "Get database status") @@ -260,12 +260,12 @@ public: using TTenantClientGRpcCommand::PrintResponse; - void PrintResponse(const Ydb::Operations::Operation &response) override + void PrintResponse(const Ydb::Operations::Operation &response) override { if (response.status() != Ydb::StatusIds::SUCCESS) { TTenantClientGRpcCommand::PrintResponse(response); } else { - Ydb::Cms::GetDatabaseStatusResult result; + Ydb::Cms::GetDatabaseStatusResult result; Y_VERIFY(response.result().UnpackTo(&result)); // type -> <required, allocated> THashMap<TString, std::pair<ui64, ui64>> pools; @@ -308,11 +308,11 @@ public: }; class TClientCommandTenantCreate - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::CreateDatabaseRequest, - Ydb::Cms::CreateDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncCreateDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncCreateDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::CreateDatabaseRequest, + Ydb::Cms::CreateDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncCreateDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncCreateDatabase> { TVector<TString> Attributes; bool Serverless = false; @@ -347,7 +347,7 @@ public: TVector<TString> items = StringSplitter(attr).Split('=').ToList<TString>(); if (items.size() != 2) ythrow yexception() << "bad format in attr '" + attr + "'"; - (*GRpcRequest.mutable_attributes())[items[0]] = items[1]; + (*GRpcRequest.mutable_attributes())[items[0]] = items[1]; } if (Serverless) { @@ -369,13 +369,13 @@ public: }; class TClientCommandTenantRemove - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::RemoveDatabaseRequest, - Ydb::Cms::RemoveDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncRemoveDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncRemoveDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::RemoveDatabaseRequest, + Ydb::Cms::RemoveDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncRemoveDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncRemoveDatabase> { private: - Ydb::Cms::RemoveDatabaseRequest Request; + Ydb::Cms::RemoveDatabaseRequest Request; bool Force; public: @@ -413,11 +413,11 @@ public: }; class TClientCommandTenantAddUnits - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::AlterDatabaseRequest, - Ydb::Cms::AlterDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::AlterDatabaseRequest, + Ydb::Cms::AlterDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { public: TClientCommandTenantAddUnits() : TTenantClientGRpcCommand("add", {}, "Add computational units for database") @@ -448,11 +448,11 @@ public: }; class TClientCommandTenantRemoveUnits - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::AlterDatabaseRequest, - Ydb::Cms::AlterDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::AlterDatabaseRequest, + Ydb::Cms::AlterDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { public: TClientCommandTenantRemoveUnits() : TTenantClientGRpcCommand("remove", {}, "Remove computational units from database") @@ -483,11 +483,11 @@ public: }; class TClientCommandTenantRegisterUnits - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::AlterDatabaseRequest, - Ydb::Cms::AlterDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::AlterDatabaseRequest, + Ydb::Cms::AlterDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { public: TClientCommandTenantRegisterUnits() : TTenantClientGRpcCommand("register", {}, "Register computational units for database") @@ -524,11 +524,11 @@ public: }; class TClientCommandTenantDeregisterUnits - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::AlterDatabaseRequest, - Ydb::Cms::AlterDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::AlterDatabaseRequest, + Ydb::Cms::AlterDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { public: TClientCommandTenantDeregisterUnits() : TTenantClientGRpcCommand("deregister", {}, "Deregister computational units for database") @@ -563,11 +563,11 @@ public: }; class TClientCommandTenantAddPools - : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, - Ydb::Cms::AlterDatabaseRequest, - Ydb::Cms::AlterDatabaseResponse, - decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), - &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { + : public TTenantClientGRpcCommand<Ydb::Cms::V1::CmsService, + Ydb::Cms::AlterDatabaseRequest, + Ydb::Cms::AlterDatabaseResponse, + decltype(&Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase), + &Ydb::Cms::V1::CmsService::Stub::AsyncAlterDatabase> { public: TClientCommandTenantAddPools() : TTenantClientGRpcCommand("add", {}, "Add storage resources for database") diff --git a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp index e432d19b2b..b4429d5721 100644 --- a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp @@ -76,8 +76,8 @@ int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv) { auto txRes = response.GetMiniKQLCompileResults(); if (txRes.ProgramCompileErrorsSize() > 0) { - TIssues errors; - NYql::IssuesFromMessage(txRes.GetProgramCompileErrors(), errors); + TIssues errors; + NYql::IssuesFromMessage(txRes.GetProgramCompileErrors(), errors); Cerr << "Program compile errors:\n"; if (config.PathToBinPgm) { errors.PrintTo(Cerr); @@ -88,8 +88,8 @@ int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv) { } if (txRes.ParamsCompileErrorsSize() > 0) { - TIssues errors; - NYql::IssuesFromMessage(txRes.GetParamsCompileErrors(), errors); + TIssues errors; + NYql::IssuesFromMessage(txRes.GetParamsCompileErrors(), errors); Cerr << "Params compile errors:\n"; if (config.PathToBinParams) { errors.PrintTo(Cerr); diff --git a/ydb/core/driver_lib/cli_utils/ya.make b/ydb/core/driver_lib/cli_utils/ya.make index 26169f056d..cb1f9670be 100644 --- a/ydb/core/driver_lib/cli_utils/ya.make +++ b/ydb/core/driver_lib/cli_utils/ya.make @@ -1,6 +1,6 @@ LIBRARY(cli_utils) -OWNER(g:kikimr) +OWNER(g:kikimr) SRCS( cli.cpp diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index d646b78b41..bb68656291 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -61,7 +61,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("memorylog-file", "set buffer size for memory log").OptionalArgument("PATH"); opts.AddLongOption("grpc-file", "gRPC config file").OptionalArgument("PATH"); opts.AddLongOption("grpc-port", "enable gRPC server on port").RequiredArgument("PORT"); - opts.AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT"); + opts.AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT"); opts.AddLongOption("grpc-public-host", "set public gRPC host for discovery").RequiredArgument("HOST"); opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT"); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT"); @@ -147,12 +147,12 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu conf.SetPort(FromString<ui16>(res.Get("grpc-port"))); } - if (res.Has("grpcs-port")) { - auto& conf = *Config.AppConfig.MutableGRpcConfig(); - conf.SetStartGRpcProxy(true); - conf.SetSslPort(FromString<ui16>(res.Get("grpcs-port"))); - } - + if (res.Has("grpcs-port")) { + auto& conf = *Config.AppConfig.MutableGRpcConfig(); + conf.SetStartGRpcProxy(true); + conf.SetSslPort(FromString<ui16>(res.Get("grpcs-port"))); + } + if (res.Has("grpc-public-host")) { auto& conf = *Config.AppConfig.MutableGRpcConfig(); conf.SetPublicHost(res.Get("grpc-public-host")); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 2b81cdcb6b..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -46,7 +46,7 @@ #include <ydb/core/grpc_services/grpc_mon.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> - + #include <ydb/core/kesus/proxy/proxy.h> #include <ydb/core/kesus/tablet/tablet.h> @@ -1505,14 +1505,14 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se TActorSetupCmd(cache, TMailboxType::ReadAsFilled, appData->UserPoolId)); } } - - if (!IsServiceInitialized(setup, NGRpcService::CreateGRpcRequestProxyId())) { + + if (!IsServiceInitialized(setup, NGRpcService::CreateGRpcRequestProxyId())) { auto grpcReqProxy = NGRpcService::CreateGRpcRequestProxy(Config); setup->LocalServices.push_back(std::pair<TActorId, - TActorSetupCmd>(NGRpcService::CreateGRpcRequestProxyId(), - TActorSetupCmd(grpcReqProxy, TMailboxType::ReadAsFilled, + TActorSetupCmd>(NGRpcService::CreateGRpcRequestProxyId(), + TActorSetupCmd(grpcReqProxy, TMailboxType::ReadAsFilled, appData->UserPoolId))); - } + } if (!IsServiceInitialized(setup, NKesus::MakeKesusProxyServiceId())) { if (IActor* proxy = NKesus::CreateKesusProxyService()) { diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index 6a78b478ea..f0a9315685 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -17,10 +17,10 @@ // allocator info #include <library/cpp/malloc/api/malloc.h> -#ifndef _win_ -#include <sys/mman.h> -#endif - +#ifndef _win_ +#include <sys/mman.h> +#endif + namespace NKikimr { int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories) { @@ -52,9 +52,9 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> } int Main(int argc, char **argv, std::shared_ptr<TModuleFactories> factories) { -#ifndef _win_ - mlockall(MCL_CURRENT); -#endif +#ifndef _win_ + mlockall(MCL_CURRENT); +#endif using namespace NLastGetopt; using TDriverModeParser = TCliCommands<EDriverMode>; diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 7178889f7a..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -100,7 +100,7 @@ #include <ydb/services/rate_limiter/grpc_service.h> #include <ydb/services/discovery/grpc_service.h> #include <ydb/services/yq/grpc_service.h> - + #include <ydb/core/yq/libs/init/init.h> #include <library/cpp/logger/global/global.h> @@ -330,9 +330,9 @@ TKikimrRunner::~TKikimrRunner() { ActorSystem->Stop(); // After that stop sending any requests to actors // by destroing grpc subsystem. - for (auto& serv : GRpcServers) { - serv.second.Destroy(); - } + for (auto& serv : GRpcServers) { + serv.second.Destroy(); + } ActorSystem.Destroy(); } @@ -464,11 +464,11 @@ void TKikimrRunner::InitializeMessageBus( } } -static TString ReadFile(const TString& fileName) { - TFileInput f(fileName); - return f.ReadAll(); -} - +static TString ReadFile(const TString& fileName) { + TFileInput f(fileName); + return f.ReadAll(); +} + void TKikimrRunner::InitializeGracefulShutdown(const TKikimrRunConfig& runConfig) { Y_UNUSED(runConfig); GracefulShutdownSupported = true; @@ -739,14 +739,14 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { opts.SetWorkerThreads(grpcConfig.GetWorkerThreads()); opts.SetGRpcMemoryQuotaBytes(grpcConfig.GetGRpcMemoryQuotaBytes()); opts.SetMaxMessageSize(grpcConfig.HasMaxMessageSize() ? grpcConfig.GetMaxMessageSize() : DEFAULT_GRPC_MESSAGE_SIZE_LIMIT); - opts.SetMaxGlobalRequestInFlight(grpcConfig.GetMaxInFlight()); + opts.SetMaxGlobalRequestInFlight(grpcConfig.GetMaxInFlight()); opts.SetLogger(NGrpc::CreateActorSystemLogger(*ActorSystem.Get(), NKikimrServices::GRPC_SERVER)); - if (appConfig.HasDomainsConfig() && - appConfig.GetDomainsConfig().HasSecurityConfig() && - appConfig.GetDomainsConfig().GetSecurityConfig().HasEnforceUserTokenRequirement()) { + if (appConfig.HasDomainsConfig() && + appConfig.GetDomainsConfig().HasSecurityConfig() && + appConfig.GetDomainsConfig().GetSecurityConfig().HasEnforceUserTokenRequirement()) { opts.SetUseAuth(appConfig.GetDomainsConfig().GetSecurityConfig().GetEnforceUserTokenRequirement()); - } + } if (grpcConfig.HasKeepAliveEnable()) { if (grpcConfig.GetKeepAliveEnable()) { @@ -764,22 +764,22 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { } NGrpc::TServerOptions sslOpts = opts; - if (grpcConfig.HasSslPort() && grpcConfig.GetSslPort()) { - Y_VERIFY(grpcConfig.HasCA(), "CA not set"); - Y_VERIFY(grpcConfig.HasCert(), "Cert not set"); - Y_VERIFY(grpcConfig.HasKey(), "Key not set"); - sslOpts.SetPort(grpcConfig.GetSslPort()); + if (grpcConfig.HasSslPort() && grpcConfig.GetSslPort()) { + Y_VERIFY(grpcConfig.HasCA(), "CA not set"); + Y_VERIFY(grpcConfig.HasCert(), "Cert not set"); + Y_VERIFY(grpcConfig.HasKey(), "Key not set"); + sslOpts.SetPort(grpcConfig.GetSslPort()); NGrpc::TSslData sslData; - sslData.Root = ReadFile(grpcConfig.GetCA()); - sslData.Cert = ReadFile(grpcConfig.GetCert()); - sslData.Key = ReadFile(grpcConfig.GetKey()); - sslOpts.SetSslData(sslData); - + sslData.Root = ReadFile(grpcConfig.GetCA()); + sslData.Cert = ReadFile(grpcConfig.GetCert()); + sslData.Key = ReadFile(grpcConfig.GetKey()); + sslOpts.SetSslData(sslData); + GRpcServers.push_back({ "grpcs", new NGrpc::TGRpcServer(sslOpts) }); fillFn(grpcConfig, *GRpcServers.back().second, sslOpts); - } - + } + if (grpcConfig.GetPort()) { GRpcServers.push_back({ "grpc", new NGrpc::TGRpcServer(opts) }); @@ -797,8 +797,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { GRpcServers.push_back({ "grpc", new NGrpc::TGRpcServer(xopts) }); fillFn(ex, *GRpcServers.back().second, xopts); - } - + } + if (ex.HasSslPort() && ex.GetSslPort()) { NGrpc::TServerOptions xopts = opts; xopts.SetPort(ex.GetSslPort()); @@ -1363,18 +1363,18 @@ void TKikimrRunner::KikimrStart() { ActorSystem->Start(); } - for (auto& server : GRpcServers) { - if (server.second) { - server.second->Start(); - - TString endpoint; - if (server.second->GetHost() != "[::]") { - endpoint = server.second->GetHost(); - } - endpoint += Sprintf(":%d", server.second->GetPort()); - ActorSystem->Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ActorSystem->NodeId), - new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateAddEndpoint(server.first, endpoint)); - } + for (auto& server : GRpcServers) { + if (server.second) { + server.second->Start(); + + TString endpoint; + if (server.second->GetHost() != "[::]") { + endpoint = server.second->GetHost(); + } + endpoint += Sprintf(":%d", server.second->GetPort()); + ActorSystem->Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ActorSystem->NodeId), + new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateAddEndpoint(server.first, endpoint)); + } } if (SqsHttp) { @@ -1457,20 +1457,20 @@ void TKikimrRunner::KikimrStop(bool graceful) { SqsHttp.Destroy(); } - // stop processing grpc requests/response - we must stop feeding ActorSystem - for (auto& server : GRpcServers) { - if (server.second) { - server.second->Stop(); - } - } - + // stop processing grpc requests/response - we must stop feeding ActorSystem + for (auto& server : GRpcServers) { + if (server.second) { + server.second->Stop(); + } + } + if (ActorSystem) { ActorSystem->Stop(); } - for (auto& server : GRpcServers) { - server.second.Destroy(); - } + for (auto& server : GRpcServers) { + server.second.Destroy(); + } if (Bus) { Bus->Stop(); diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 49a26fb902..f410ef42f2 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -157,8 +157,8 @@ END() RECURSE_FOR_TESTS( ut ) - -RECURSE_ROOT_RELATIVE( + +RECURSE_ROOT_RELATIVE( ydb/core ydb/services -) +) diff --git a/ydb/core/engine/kikimr_program_builder.cpp b/ydb/core/engine/kikimr_program_builder.cpp index 53ef2bf628..fd32a772ee 100644 --- a/ydb/core/engine/kikimr_program_builder.cpp +++ b/ydb/core/engine/kikimr_program_builder.cpp @@ -57,7 +57,7 @@ TTableRangeOptions::TTableRangeOptions(const TTypeEnvironment& env) : ItemsLimit(BuildDataLiteral(NUdf::TUnboxedValuePod((ui64)0), NUdf::EDataSlot::Uint64, env), true) , BytesLimit(BuildDataLiteral(NUdf::TUnboxedValuePod((ui64)0), NUdf::EDataSlot::Uint64, env), true) , Flags(BuildDataLiteral(NUdf::TUnboxedValuePod((ui32)TReadRangeOptions::TFlags::Default), NUdf::EDataSlot::Uint32, env), true) - , Reverse(BuildDataLiteral(NUdf::TUnboxedValuePod(false), NUdf::EDataSlot::Bool, env), true) + , Reverse(BuildDataLiteral(NUdf::TUnboxedValuePod(false), NUdf::EDataSlot::Bool, env), true) { } @@ -75,22 +75,22 @@ TRuntimeNode TParametersBuilder::Build() { return TRuntimeNode(StructBuilder.Build(), true); } -static TRuntimeNode DoRewriteNullType( - TRuntimeNode value, +static TRuntimeNode DoRewriteNullType( + TRuntimeNode value, NUdf::TDataTypeId expectedType, - const TInternName& nullInternName, - const TTypeEnvironment& env) -{ - if (!value.IsImmediate()) { - auto& callable = static_cast<TCallable&>(*value.GetNode()); - auto callableName = callable.GetType()->GetNameStr(); - if (callableName == nullInternName) { - return TRuntimeNode(BuildEmptyOptionalDataLiteral(expectedType, env), true); - } - } - return value; -} - + const TInternName& nullInternName, + const TTypeEnvironment& env) +{ + if (!value.IsImmediate()) { + auto& callable = static_cast<TCallable&>(*value.GetNode()); + auto callableName = callable.GetType()->GetNameStr(); + if (callableName == nullInternName) { + return TRuntimeNode(BuildEmptyOptionalDataLiteral(expectedType, env), true); + } + } + return value; +} + TUpdateRowBuilder::TUpdateRowBuilder(const TTypeEnvironment& env) : Builder(env) , Env(env) @@ -103,7 +103,7 @@ void TUpdateRowBuilder::SetColumn( TRuntimeNode value) { bool isOptional; - value = DoRewriteNullType(value, expectedType, NullInternName, Env); + value = DoRewriteNullType(value, expectedType, NullInternName, Env); TDataType* dataType = UnpackOptionalData(value, isOptional); MKQL_ENSURE(expectedType == dataType->GetSchemeType(), "Mismatch of column type expectedType = " << expectedType @@ -179,50 +179,50 @@ TKikimrProgramBuilder::TKikimrProgramBuilder( { UseNullType = false; NullInternName = Env.InternName(TStringBuf("Null")); - - std::array<TType*, 3> tupleTypes; - tupleTypes[0] = NewDataType(NUdf::TDataType<ui64>::Id); - tupleTypes[1] = NewDataType(NUdf::TDataType<ui64>::Id); - tupleTypes[2] = NewDataType(NUdf::TDataType<ui64>::Id); - TableIdType = TTupleType::Create(tupleTypes.size(), tupleTypes.data(), Env); + + std::array<TType*, 3> tupleTypes; + tupleTypes[0] = NewDataType(NUdf::TDataType<ui64>::Id); + tupleTypes[1] = NewDataType(NUdf::TDataType<ui64>::Id); + tupleTypes[2] = NewDataType(NUdf::TDataType<ui64>::Id); + TableIdType = TTupleType::Create(tupleTypes.size(), tupleTypes.data(), Env); } -TRuntimeNode TKikimrProgramBuilder::RewriteNullType( - TRuntimeNode value, +TRuntimeNode TKikimrProgramBuilder::RewriteNullType( + TRuntimeNode value, NUdf::TDataTypeId expectedType) const -{ - return DoRewriteNullType(value, expectedType, NullInternName, Env); -} - +{ + return DoRewriteNullType(value, expectedType, NullInternName, Env); +} + TVector<TRuntimeNode> TKikimrProgramBuilder::FixKeysType( const TArrayRef<const ui32>& keyTypes, - const TKeyColumnValues& row) const + const TKeyColumnValues& row) const { MKQL_ENSURE(!row.empty(), "Expected at least 1 key column"); MKQL_ENSURE(keyTypes.size() == row.size(), "Mismatch of key types count"); TVector<TRuntimeNode> tmp(row.size()); for (ui32 i = 0; i < row.size(); ++i) { - tmp[i] = RewriteNullType(row[i], keyTypes[i]); + tmp[i] = RewriteNullType(row[i], keyTypes[i]); bool isOptional; - TDataType* dataType = UnpackOptionalData(tmp[i], isOptional); + TDataType* dataType = UnpackOptionalData(tmp[i], isOptional); MKQL_ENSURE(keyTypes[i] == dataType->GetSchemeType(), - "Mismatch of key column type, expected: " - << keyTypes[i] << ", but got: " << dataType->GetSchemeType()); + "Mismatch of key column type, expected: " + << keyTypes[i] << ", but got: " << dataType->GetSchemeType()); } - return tmp; -} + return tmp; +} TRuntimeNode TKikimrProgramBuilder::ReadTarget(const TReadTarget& target) { MKQL_ENSURE(!target.HasSnapshotTime(), "Snapshots are not supported"); return NewDataLiteral((ui32)target.GetMode()); } -TRuntimeNode TKikimrProgramBuilder::SelectRow( - const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, - const TArrayRef<const TSelectColumn>& columns, - const TKeyColumnValues& row, const TReadTarget& target) -{ +TRuntimeNode TKikimrProgramBuilder::SelectRow( + const TTableId& tableId, + const TArrayRef<const ui32>& keyTypes, + const TArrayRef<const TSelectColumn>& columns, + const TKeyColumnValues& row, const TReadTarget& target) +{ return SelectRow(tableId, keyTypes, columns, row, ReadTarget(target)); } @@ -235,18 +235,18 @@ TRuntimeNode TKikimrProgramBuilder::SelectRow( MKQL_ENSURE(AS_TYPE(TDataType, readTarget)->GetSchemeType() == NUdf::TDataType<ui32>::Id, "Read target must be ui32"); - auto rows = FixKeysType(keyTypes, row); - + auto rows = FixKeysType(keyTypes, row); + TRuntimeNode tags; auto rowType = ValidateColumns(columns, tags, this); TType* optType = NewOptionalType(rowType); TCallableBuilder builder(Env, "SelectRow", optType); - - builder.Add(BuildTableId(tableId)); + + builder.Add(BuildTableId(tableId)); builder.Add(TRuntimeNode(rowType, true)); builder.Add(tags); - builder.Add(NewTuple(TKeyColumnValues(rows))); + builder.Add(NewTuple(TKeyColumnValues(rows))); builder.Add(readTarget); return TRuntimeNode(builder.Build(), false); @@ -262,22 +262,22 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( return SelectRange(tableId, keyTypes, columns, options, ReadTarget(target)); } -static TRuntimeNode BuildOptionList(const TArrayRef<bool>& arr, TProgramBuilder& pBuilder) { - TListLiteralBuilder builder(pBuilder.GetTypeEnvironment(), pBuilder.NewDataType(NUdf::TDataType<bool>::Id)); - for (auto& item : arr) { - builder.Add(pBuilder.NewDataLiteral(item)); - } - return TRuntimeNode(builder.Build(), true); -} - -static bool AtLeastOneFlagSet(const TArrayRef<bool>& arr) { - for (bool flag : arr) { - if (flag) - return true; - } - return false; -} - +static TRuntimeNode BuildOptionList(const TArrayRef<bool>& arr, TProgramBuilder& pBuilder) { + TListLiteralBuilder builder(pBuilder.GetTypeEnvironment(), pBuilder.NewDataType(NUdf::TDataType<bool>::Id)); + for (auto& item : arr) { + builder.Add(pBuilder.NewDataLiteral(item)); + } + return TRuntimeNode(builder.Build(), true); +} + +static bool AtLeastOneFlagSet(const TArrayRef<bool>& arr) { + for (bool flag : arr) { + if (flag) + return true; + } + return false; +} + TRuntimeNode TKikimrProgramBuilder::SelectRange( const TTableId& tableId, const TArrayRef<const ui32>& keyTypes, @@ -303,17 +303,17 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( TDataType* dataFrom = nullptr; TDataType* dataTo = nullptr; if (i < options.FromColumns.size()) { - from[i] = RewriteNullType(options.FromColumns[i], keyTypes[i]); - + from[i] = RewriteNullType(options.FromColumns[i], keyTypes[i]); + bool isOptionalFrom; - dataFrom = UnpackOptionalData(from[i], isOptionalFrom); + dataFrom = UnpackOptionalData(from[i], isOptionalFrom); } if (i < options.ToColumns.size()) { - to[i] = RewriteNullType(options.ToColumns[i], keyTypes[i]); - + to[i] = RewriteNullType(options.ToColumns[i], keyTypes[i]); + bool isOptionalTo; - dataTo = UnpackOptionalData(to[i], isOptionalTo); + dataTo = UnpackOptionalData(to[i], isOptionalTo); } MKQL_ENSURE(dataFrom || dataTo, "Invalid key tuple values"); @@ -335,9 +335,9 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( TType* listType = NewListType(rowType); TDataType* boolType = TDataType::Create(NUdf::TDataType<bool>::Id, Env); - TRuntimeNode skipNullKeys = BuildOptionList(options.SkipNullKeys, *this); - TRuntimeNode forbidNullArgsFrom = BuildOptionList(options.ForbidNullArgsFrom, *this); - TRuntimeNode forbidNullArgsTo = BuildOptionList(options.ForbidNullArgsTo, *this); + TRuntimeNode skipNullKeys = BuildOptionList(options.SkipNullKeys, *this); + TRuntimeNode forbidNullArgsFrom = BuildOptionList(options.ForbidNullArgsFrom, *this); + TRuntimeNode forbidNullArgsTo = BuildOptionList(options.ForbidNullArgsTo, *this); TStructTypeBuilder returnTypeBuilder(Env); returnTypeBuilder.Reserve(2); @@ -345,25 +345,25 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( returnTypeBuilder.Add(TStringBuf("Truncated"), boolType); auto returnType = returnTypeBuilder.Build(); TCallableBuilder builder(Env, "SelectRange", returnType); - - builder.Add(BuildTableId(tableId)); + + builder.Add(BuildTableId(tableId)); builder.Add(TRuntimeNode(rowType, true)); builder.Add(tags); - builder.Add(NewTuple(TKeyColumnValues(from))); - builder.Add(NewTuple(TKeyColumnValues(to))); + builder.Add(NewTuple(TKeyColumnValues(from))); + builder.Add(NewTuple(TKeyColumnValues(to))); builder.Add(options.Flags); builder.Add(options.ItemsLimit); builder.Add(options.BytesLimit); builder.Add(readTarget); builder.Add(skipNullKeys); - builder.Add(options.Reverse); - - // Compat with older kikimr - // The following 'if' should be removed when all deployed versions have support for 13 parameters. - if (AtLeastOneFlagSet(options.ForbidNullArgsFrom) || - AtLeastOneFlagSet(options.ForbidNullArgsTo)) { - builder.Add(forbidNullArgsFrom); - builder.Add(forbidNullArgsTo); + builder.Add(options.Reverse); + + // Compat with older kikimr + // The following 'if' should be removed when all deployed versions have support for 13 parameters. + if (AtLeastOneFlagSet(options.ForbidNullArgsFrom) || + AtLeastOneFlagSet(options.ForbidNullArgsTo)) { + builder.Add(forbidNullArgsFrom); + builder.Add(forbidNullArgsTo); } return TRuntimeNode(builder.Build(), false); @@ -375,12 +375,12 @@ TRuntimeNode TKikimrProgramBuilder::UpdateRow( const TKeyColumnValues& row, TUpdateRowBuilder& update) { - auto rows = FixKeysType(keyTypes, row); + auto rows = FixKeysType(keyTypes, row); TCallableBuilder builder(Env, "UpdateRow", Env.GetTypeOfVoid()); - - builder.Add(BuildTableId(tableId)); - builder.Add(NewTuple(TKeyColumnValues(rows))); + + builder.Add(BuildTableId(tableId)); + builder.Add(NewTuple(TKeyColumnValues(rows))); builder.Add(update.Build()); return TRuntimeNode(builder.Build(), false); } @@ -390,12 +390,12 @@ TRuntimeNode TKikimrProgramBuilder::EraseRow( const TArrayRef<const ui32>& keyTypes, const TKeyColumnValues& row) { - auto rows = FixKeysType(keyTypes, row); + auto rows = FixKeysType(keyTypes, row); TCallableBuilder builder(Env, "EraseRow", Env.GetTypeOfVoid()); - - builder.Add(BuildTableId(tableId)); - builder.Add(NewTuple(TKeyColumnValues(rows))); + + builder.Add(BuildTableId(tableId)); + builder.Add(NewTuple(TKeyColumnValues(rows))); return TRuntimeNode(builder.Build(), false); } @@ -657,19 +657,19 @@ TRuntimeNode TKikimrProgramBuilder::SetResult(const TStringBuf& label, TRuntimeN TRuntimeNode TKikimrProgramBuilder::NewDataLiteral(const std::pair<ui64, ui64>& data) const { return TRuntimeNode(BuildDataLiteral(NUdf::TStringRef(reinterpret_cast<const char*>(&data), sizeof(data)), LegacyPairUi64Ui64, Env), true); } - -TRuntimeNode TKikimrProgramBuilder::BuildTableId(const TTableId& tableId) const { - if (tableId.SchemaVersion) { - std::array<TRuntimeNode, 3> items; - items[0] = TProgramBuilder::NewDataLiteral<ui64>(tableId.PathId.OwnerId); - items[1] = TProgramBuilder::NewDataLiteral<ui64>(tableId.PathId.LocalPathId); - items[2] = TProgramBuilder::NewDataLiteral<ui64>(tableId.SchemaVersion); - return TRuntimeNode(TTupleLiteral::Create(items.size(), items.data(), TableIdType, Env), true); - } else { - // temporary compatibility (KIKIMR-8446) - return NewDataLiteral({tableId.PathId.OwnerId, tableId.PathId.LocalPathId}); - } -} - + +TRuntimeNode TKikimrProgramBuilder::BuildTableId(const TTableId& tableId) const { + if (tableId.SchemaVersion) { + std::array<TRuntimeNode, 3> items; + items[0] = TProgramBuilder::NewDataLiteral<ui64>(tableId.PathId.OwnerId); + items[1] = TProgramBuilder::NewDataLiteral<ui64>(tableId.PathId.LocalPathId); + items[2] = TProgramBuilder::NewDataLiteral<ui64>(tableId.SchemaVersion); + return TRuntimeNode(TTupleLiteral::Create(items.size(), items.data(), TableIdType, Env), true); + } else { + // temporary compatibility (KIKIMR-8446) + return NewDataLiteral({tableId.PathId.OwnerId, tableId.PathId.LocalPathId}); + } +} + } // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/engine/kikimr_program_builder.h b/ydb/core/engine/kikimr_program_builder.h index ee77acade8..717132c0b9 100644 --- a/ydb/core/engine/kikimr_program_builder.h +++ b/ydb/core/engine/kikimr_program_builder.h @@ -64,8 +64,8 @@ struct TTableRangeOptions TKeyColumnValues FromColumns; TKeyColumnValues ToColumns; TArrayRef<bool> SkipNullKeys; - TArrayRef<bool> ForbidNullArgsFrom; - TArrayRef<bool> ForbidNullArgsTo; + TArrayRef<bool> ForbidNullArgsFrom; + TArrayRef<bool> ForbidNullArgsTo; TRuntimeNode Reverse; // default = <unset> }; @@ -101,7 +101,7 @@ public: private: TStructLiteralBuilder Builder; const TTypeEnvironment& Env; - TInternName NullInternName; + TInternName NullInternName; }; class TParametersBuilder @@ -233,15 +233,15 @@ public: using TProgramBuilder::NewDataLiteral; private: TRuntimeNode NewDataLiteral(const std::pair<ui64, ui64>& data) const; - TRuntimeNode BuildTableId(const TTableId& tableId) const; + TRuntimeNode BuildTableId(const TTableId& tableId) const; TVector<TRuntimeNode> FixKeysType( - const TArrayRef<const ui32>& keyTypes, - const TKeyColumnValues& row) const; - TRuntimeNode RewriteNullType( - TRuntimeNode value, + const TArrayRef<const ui32>& keyTypes, + const TKeyColumnValues& row) const; + TRuntimeNode RewriteNullType( + TRuntimeNode value, NUdf::TDataTypeId expectedType) const; - TInternName NullInternName; - TTupleType* TableIdType; + TInternName NullInternName; + TTupleType* TableIdType; }; } // namespace NMiniKQL diff --git a/ydb/core/engine/kikimr_program_builder_ut.cpp b/ydb/core/engine/kikimr_program_builder_ut.cpp index 12f19bcbe4..efa073ae2f 100644 --- a/ydb/core/engine/kikimr_program_builder_ut.cpp +++ b/ydb/core/engine/kikimr_program_builder_ut.cpp @@ -103,7 +103,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Erase); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -147,7 +147,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Erase); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveTo); @@ -189,7 +189,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Erase); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveTo); @@ -231,7 +231,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -286,7 +286,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Update); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -346,7 +346,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Update); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveTo); @@ -401,7 +401,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -451,7 +451,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -503,7 +503,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -558,7 +558,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveTo); @@ -611,7 +611,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(tableKeys[0]->Range.InclusiveTo); @@ -663,7 +663,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { explorer.Walk(pgm, env); TVector<THolder<TKeyDesc>> tableKeys = ExtractTableKeys(explorer, env); UNIT_ASSERT_VALUES_EQUAL(tableKeys.size(), 1); - UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); + UNIT_ASSERT(tableKeys[0]->TableId.HasSamePath(TTableId(1, 2))); UNIT_ASSERT(tableKeys[0]->RowOperation == TKeyDesc::ERowOperation::Read); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveFrom); UNIT_ASSERT(!tableKeys[0]->Range.InclusiveTo); diff --git a/ydb/core/engine/minikql/flat_local_tx_minikql.h b/ydb/core/engine/minikql/flat_local_tx_minikql.h index 6ef7a77fcb..0feabf20e6 100644 --- a/ydb/core/engine/minikql/flat_local_tx_minikql.h +++ b/ydb/core/engine/minikql/flat_local_tx_minikql.h @@ -86,7 +86,7 @@ class TFlatLocalMiniKQL : public NTabletFlatExecutor::ITransaction { TAutoPtr<NKikimrMiniKQL::TResult> EngineEvaluatedResponse; ui64 PageFaultCount = 0; - bool ParseProgram(TStringBuf program, NYql::TIssues &errors, NYql::TExprContainer &expr) { + bool ParseProgram(TStringBuf program, NYql::TIssues &errors, NYql::TExprContainer &expr) { NYql::TAstParseResult astResult = NYql::ParseAst(program); if (!astResult.Root) { errors = astResult.Issues; @@ -95,7 +95,7 @@ class TFlatLocalMiniKQL : public NTabletFlatExecutor::ITransaction { expr.Context.IssueManager.AddIssues(astResult.Issues); if (!NYql::CompileExpr(*astResult.Root, expr.Root, expr.Context, nullptr)) { - errors = expr.Context.IssueManager.GetIssues(); + errors = expr.Context.IssueManager.GetIssues(); return false; } @@ -187,7 +187,7 @@ class TFlatLocalMiniKQL : public NTabletFlatExecutor::ITransaction { record.SetStatus(NKikimrProto::OK); compileResults->SetCompiledProgram(ProgramCompileResult->CompiledProgram); } else { - NYql::IssuesToMessage(ProgramCompileResult->Errors, compileResults->MutableProgramCompileErrors()); + NYql::IssuesToMessage(ProgramCompileResult->Errors, compileResults->MutableProgramCompileErrors()); record.SetStatus(NKikimrProto::ERROR); } } else { diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index 0d3b7c1e39..36c35f32cc 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -752,41 +752,41 @@ private: NUdf::TUnboxedValue List; }; -static NUdf::TUnboxedValue CreateEmptyRange(const THolderFactory& holderFactory) { - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto res = holderFactory.CreateDirectArrayHolder(4, itemsPtr); - // Empty list (data container) - itemsPtr[0] = NUdf::TUnboxedValue(holderFactory.GetEmptyContainer()); - // Truncated flag - itemsPtr[1] = NUdf::TUnboxedValuePod(false); - // First key - itemsPtr[2] = NUdf::TUnboxedValuePod::Zero(); - // Size - itemsPtr[3] = NUdf::TUnboxedValuePod((ui64)0); - - return res; -} - -static TSmallVec<bool> CreateBoolVec(const TListLiteral* list) { - TSmallVec<bool> res; - if (list) { - res.reserve(list->GetItemsCount()); - for (ui32 i = 0; i < list->GetItemsCount(); ++i) { - res.push_back(AS_VALUE(TDataLiteral, list->GetItems()[i])->AsValue().Get<bool>()); - } - } - - while (!res.empty() && !res.back()) { - res.pop_back(); - } - - return res; -} - +static NUdf::TUnboxedValue CreateEmptyRange(const THolderFactory& holderFactory) { + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto res = holderFactory.CreateDirectArrayHolder(4, itemsPtr); + // Empty list (data container) + itemsPtr[0] = NUdf::TUnboxedValue(holderFactory.GetEmptyContainer()); + // Truncated flag + itemsPtr[1] = NUdf::TUnboxedValuePod(false); + // First key + itemsPtr[2] = NUdf::TUnboxedValuePod::Zero(); + // Size + itemsPtr[3] = NUdf::TUnboxedValuePod((ui64)0); + + return res; +} + +static TSmallVec<bool> CreateBoolVec(const TListLiteral* list) { + TSmallVec<bool> res; + if (list) { + res.reserve(list->GetItemsCount()); + for (ui32 i = 0; i < list->GetItemsCount(); ++i) { + res.push_back(AS_VALUE(TDataLiteral, list->GetItems()[i])->AsValue().Get<bool>()); + } + } + + while (!res.empty() && !res.back()) { + res.pop_back(); + } + + return res; +} + NUdf::TUnboxedValue TEngineHost::SelectRange(const TTableId& tableId, const TTableRange& range, TStructLiteral* columnIds, TListLiteral* skipNullKeys, TStructType* returnType, const TReadTarget& readTarget, - ui64 itemsLimit, ui64 bytesLimit, bool reverse, std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, - const THolderFactory& holderFactory) + ui64 itemsLimit, ui64 bytesLimit, bool reverse, std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, + const THolderFactory& holderFactory) { // It is assumed that returnType has form: // struct< @@ -820,26 +820,26 @@ NUdf::TUnboxedValue TEngineHost::SelectRange(const TTableId& tableId, const TTab TSmallVec<NTable::TTag> systemColumnTags; AnalyzeRowType(columnIds, tags, systemColumnTags); - TSmallVec<bool> skipNullKeysFlags = CreateBoolVec(skipNullKeys); - TSmallVec<bool> forbidNullArgsFrom = CreateBoolVec(forbidNullArgs.first); - TSmallVec<bool> forbidNullArgsTo = CreateBoolVec(forbidNullArgs.second); - - for (size_t keyIdx = 0; keyIdx < forbidNullArgsFrom.size(); keyIdx++) { - if (!forbidNullArgsFrom[keyIdx]) - continue; - - if (keyIdx < range.From.size() && range.From[keyIdx].IsNull()) { - return CreateEmptyRange(holderFactory); + TSmallVec<bool> skipNullKeysFlags = CreateBoolVec(skipNullKeys); + TSmallVec<bool> forbidNullArgsFrom = CreateBoolVec(forbidNullArgs.first); + TSmallVec<bool> forbidNullArgsTo = CreateBoolVec(forbidNullArgs.second); + + for (size_t keyIdx = 0; keyIdx < forbidNullArgsFrom.size(); keyIdx++) { + if (!forbidNullArgsFrom[keyIdx]) + continue; + + if (keyIdx < range.From.size() && range.From[keyIdx].IsNull()) { + return CreateEmptyRange(holderFactory); } } - for (size_t keyIdx = 0; keyIdx < forbidNullArgsTo.size(); keyIdx++) { - if (!forbidNullArgsTo[keyIdx]) - continue; - - if (keyIdx < range.To.size() && range.To[keyIdx].IsNull()) { - return CreateEmptyRange(holderFactory); - } + for (size_t keyIdx = 0; keyIdx < forbidNullArgsTo.size(); keyIdx++) { + if (!forbidNullArgsTo[keyIdx]) + continue; + + if (keyIdx < range.To.size() && range.To[keyIdx].IsNull()) { + return CreateEmptyRange(holderFactory); + } } Counters.NSelectRange++; @@ -928,10 +928,10 @@ bool TEngineHost::IsMyKey(const TTableId& tableId, const TArrayRef<const TCell>& return true; } -ui64 TEngineHost::GetTableSchemaVersion(const TTableId&) const { - return 0; -} - +ui64 TEngineHost::GetTableSchemaVersion(const TTableId&) const { + return 0; +} + ui64 TEngineHost::LocalTableId(const TTableId& tableId) const { return tableId.PathId.LocalPathId; } diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h index 8b3fa95e7f..012ee6891b 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.h +++ b/ydb/core/engine/minikql/minikql_engine_host.h @@ -110,14 +110,14 @@ public: NUdf::TUnboxedValue SelectRange(const TTableId& tableId, const TTableRange& range, TStructLiteral* columnIds, TListLiteral* skipNullKeys, TStructType* returnType, const TReadTarget& readTarget, ui64 itemsLimit, ui64 bytesLimit, bool reverse, - std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) override; + std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) override; void UpdateRow(const TTableId& tableId, const TArrayRef<const TCell>& row, const TArrayRef<const TUpdateCommand>& commands) override; void EraseRow(const TTableId& tableId, const TArrayRef<const TCell>& row) override; bool IsPathErased(const TTableId& tableId) const override; bool IsMyKey(const TTableId& tableId, const TArrayRef<const TCell>& row) const override; - ui64 GetTableSchemaVersion(const TTableId&) const override; + ui64 GetTableSchemaVersion(const TTableId&) const override; void SetPeriodicCallback(TPeriodicCallback&& callback) override; void ExecPeriodicCallback() { if (PeriodicCallback) { PeriodicCallback();} } diff --git a/ydb/core/engine/mkql_engine_flat.cpp b/ydb/core/engine/mkql_engine_flat.cpp index 71b8c02ced..c7fe388eef 100644 --- a/ydb/core/engine/mkql_engine_flat.cpp +++ b/ydb/core/engine/mkql_engine_flat.cpp @@ -481,8 +481,8 @@ public: } } - TTablePathHashSet srcTables; - TTablePathHashSet dstTables; + TTablePathHashSet srcTables; + TTablePathHashSet dstTables; for (auto& readset : readsets) { srcTables.emplace(tableMap[readset.first]); dstTables.emplace(tableMap[readset.second]); @@ -980,7 +980,7 @@ public: if (!CheckValidUint8InKey(*desc)) { return EResult::ProgramError; } - + validationInfo.Keys.emplace_back(TValidatedKey(std::move(desc), true)); } } diff --git a/ydb/core/engine/mkql_engine_flat_extfunc.cpp b/ydb/core/engine/mkql_engine_flat_extfunc.cpp index 850362aa71..414d3aeeb4 100644 --- a/ydb/core/engine/mkql_engine_flat_extfunc.cpp +++ b/ydb/core/engine/mkql_engine_flat_extfunc.cpp @@ -972,7 +972,7 @@ NUdf::TUnboxedValue PerformLocalSelectRow(TCallable& callable, IEngineFlatHost& NUdf::TUnboxedValue PerformLocalSelectRange(TCallable& callable, IEngineFlatHost& engineHost, const THolderFactory& holderFactory, const TTypeEnvironment& env) { - MKQL_ENSURE(callable.GetInputsCount() >= 9 && callable.GetInputsCount() <= 13, "Expected 9 to 13 args"); + MKQL_ENSURE(callable.GetInputsCount() >= 9 && callable.GetInputsCount() <= 13, "Expected 9 to 13 args"); auto tableNode = callable.GetInput(0); const auto tableId = ExtractTableId(tableNode); ui32 flags = AS_VALUE(TDataLiteral, callable.GetInput(5))->AsValue().Get<ui32>(); @@ -1000,15 +1000,15 @@ NUdf::TUnboxedValue PerformLocalSelectRange(TCallable& callable, IEngineFlatHost reverse = AS_VALUE(TDataLiteral, callable.GetInput(10))->AsValue().Get<bool>(); } - std::pair<TListLiteral*, TListLiteral*> forbidNullArgs{nullptr, nullptr}; - if (callable.GetInputsCount() > 12) { - forbidNullArgs.first = AS_VALUE(TListLiteral, callable.GetInput(11)); - forbidNullArgs.second = AS_VALUE(TListLiteral, callable.GetInput(12)); - } - + std::pair<TListLiteral*, TListLiteral*> forbidNullArgs{nullptr, nullptr}; + if (callable.GetInputsCount() > 12) { + forbidNullArgs.first = AS_VALUE(TListLiteral, callable.GetInput(11)); + forbidNullArgs.second = AS_VALUE(TListLiteral, callable.GetInput(12)); + } + return engineHost.SelectRange(tableId, range, AS_VALUE(TStructLiteral, callable.GetInput(2)), skipNullKeys, AS_TYPE(TStructType, returnType), - ExtractFlatReadTarget(callable.GetInput(8)), itemsLimit, bytesLimit, reverse, forbidNullArgs, holderFactory); + ExtractFlatReadTarget(callable.GetInput(8)), itemsLimit, bytesLimit, reverse, forbidNullArgs, holderFactory); } TStructType* GetTxLockType(const TTypeEnvironment& env, bool v2) { diff --git a/ydb/core/engine/mkql_engine_flat_host.h b/ydb/core/engine/mkql_engine_flat_host.h index 301c0bd8f2..0593fb3a9b 100644 --- a/ydb/core/engine/mkql_engine_flat_host.h +++ b/ydb/core/engine/mkql_engine_flat_host.h @@ -50,7 +50,7 @@ public: virtual NUdf::TUnboxedValue SelectRange(const TTableId& tableId, const TTableRange& range, TStructLiteral* columnIds, TListLiteral* skipNullKeys, TStructType* returnType, const TReadTarget& readTarget, ui64 itemsLimit, ui64 bytesLimit, bool reverse, - std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) = 0; + std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) = 0; // Updates the single row. Column in commands must be unique. virtual void UpdateRow(const TTableId& tableId, const TArrayRef<const TCell>& row, @@ -64,9 +64,9 @@ public: // Returns whether row belong this shard. virtual bool IsMyKey(const TTableId& tableId, const TArrayRef<const TCell>& row) const = 0; - - // Returns schema version for given tableId - virtual ui64 GetTableSchemaVersion(const TTableId& tableId) const = 0; + + // Returns schema version for given tableId + virtual ui64 GetTableSchemaVersion(const TTableId& tableId) const = 0; // Set callback for periodic execution virtual void SetPeriodicCallback(TPeriodicCallback&& callback) = 0; diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp index 8473eca1d0..e2b12e2d7a 100644 --- a/ydb/core/engine/mkql_engine_flat_ut.cpp +++ b/ydb/core/engine/mkql_engine_flat_ut.cpp @@ -290,54 +290,54 @@ namespace { } Y_UNIT_TEST_SUITE(TMiniKQLEngineFlatTest) { - + template<NUdf::TDataTypeId TKey, NUdf::TDataTypeId TValue> - NKikimrMiniKQL::TResult WriteSomething( - TDriver& driver, - TRuntimeNode key = TRuntimeNode(), - TRuntimeNode value = TRuntimeNode()) - { - auto& pgmBuilder = driver.PgmBuilder; + NKikimrMiniKQL::TResult WriteSomething( + TDriver& driver, + TRuntimeNode key = TRuntimeNode(), + TRuntimeNode value = TRuntimeNode()) + { + auto& pgmBuilder = driver.PgmBuilder; TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)TKey; + keyTypes[0] = (ui32)TKey; TRuntimeNode::TList row(1); - row[0] = key ? key : pgmBuilder.NewNull(); - auto update = pgmBuilder.GetUpdateRowBuilder(); - ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; - - update.SetColumn(columnId, - TValue, - value ? value : pgmBuilder.NewNull()); - - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), - keyTypes, row, update))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - return res; - } - + row[0] = key ? key : pgmBuilder.NewNull(); + auto update = pgmBuilder.GetUpdateRowBuilder(); + ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; + + update.SetColumn(columnId, + TValue, + value ? value : pgmBuilder.NewNull()); + + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), + keyTypes, row, update))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + return res; + } + template<NUdf::TDataTypeId TKey> - NKikimrMiniKQL::TResult SelectNullKey( - TDriver& driver) - { - auto& pgmBuilder = driver.PgmBuilder; + NKikimrMiniKQL::TResult SelectNullKey( + TDriver& driver) + { + auto& pgmBuilder = driver.PgmBuilder; TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)TKey; + keyTypes[0] = (ui32)TKey; TRuntimeNode::TList row(1); - row[0] = pgmBuilder.NewNull(); - + row[0] = pgmBuilder.NewNull(); + TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); - auto value = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes, columns, row); - - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - return res; - } - + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); + auto value = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes, columns, row); + + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + return res; + } + Y_UNIT_TEST(TestEmptyProgram) { TDriver driver; auto& pgmBuilder = driver.PgmBuilder; @@ -594,72 +594,72 @@ Value { } Y_UNIT_TEST(TestSelectRowPayloadNullKey) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - - auto& pgmBuilder = driver.PgmBuilder; + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + + auto& pgmBuilder = driver.PgmBuilder; WriteSomething<NUdf::TDataType<ui32>::Id, NUdf::TDataType<NUdf::TUtf8>::Id>( - driver, - pgmBuilder.NewNull(), + driver, + pgmBuilder.NewNull(), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); - + auto res = SelectNullKey<NUdf::TDataType<ui32>::Id>(driver); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct - Struct { - Member { - Name: "myRes" - Type { - Kind: Optional - Optional { - Item { - Kind: Optional - Optional { - Item { - Kind: Struct - Struct { - Member { - Name: "2" - Type { - Kind: Optional - Optional { - Item { - Kind: Data - Data { - Scheme: 4608 - } - } - } - } - } - } - } - } - } - } - } - } - } -} -Value { - Struct { - Optional { - Optional { - Struct { - Optional { - Text: "qwe" - } - } - } - } - } -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - } - - + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct + Struct { + Member { + Name: "myRes" + Type { + Kind: Optional + Optional { + Item { + Kind: Optional + Optional { + Item { + Kind: Struct + Struct { + Member { + Name: "2" + Type { + Kind: Optional + Optional { + Item { + Kind: Data + Data { + Scheme: 4608 + } + } + } + } + } + } + } + } + } + } + } + } + } +} +Value { + Struct { + Optional { + Optional { + Struct { + Optional { + Text: "qwe" + } + } + } + } + } +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + } + + Y_UNIT_TEST(TestEraseRow) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); @@ -706,85 +706,85 @@ Value { } Y_UNIT_TEST(TestEraseRowNullKey) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - - auto& pgmBuilder = driver.PgmBuilder; - { + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + + auto& pgmBuilder = driver.PgmBuilder; + { WriteSomething<NUdf::TDataType<ui32>::Id, NUdf::TDataType<NUdf::TUtf8>::Id>( - driver, - pgmBuilder.NewNull(), + driver, + pgmBuilder.NewNull(), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); - } - + } + TVector<ui32> keyTypes(1); keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; TRuntimeNode::TList row(1); - row[0] = pgmBuilder.NewNull(); - - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.EraseRow(TTableId(OwnerId, Table1Id), keyTypes, row))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - - { + row[0] = pgmBuilder.NewNull(); + + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.EraseRow(TTableId(OwnerId, Table1Id), keyTypes, row))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + + { auto res = SelectNullKey<NUdf::TDataType<ui32>::Id>(driver); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct - Struct { - Member { - Name: "myRes" - Type { - Kind: Optional - Optional { - Item { - Kind: Optional - Optional { - Item { - Kind: Struct - Struct { - Member { - Name: "2" - Type { - Kind: Optional - Optional { - Item { - Kind: Data - Data { - Scheme: 4608 - } - } - } - } - } - } - } - } - } - } - } - } - } -} -Value { - Struct { - Optional { - } - } -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - - } - } - + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct + Struct { + Member { + Name: "myRes" + Type { + Kind: Optional + Optional { + Item { + Kind: Optional + Optional { + Item { + Kind: Struct + Struct { + Member { + Name: "2" + Type { + Kind: Optional + Optional { + Item { + Kind: Data + Data { + Scheme: 4608 + } + } + } + } + } + } + } + } + } + } + } + } + } +} +Value { + Struct { + Optional { + } + } +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + + } + } + Y_UNIT_TEST(TestUpdateRowNotExistWithoutColumns) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); @@ -867,34 +867,34 @@ Value { } Y_UNIT_TEST(TestUpdateRowNotExistSetPayloadNullValue) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - - auto& pgmBuilder = driver.PgmBuilder; + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + + auto& pgmBuilder = driver.PgmBuilder; auto res = WriteSomething<NUdf::TDataType<ui32>::Id, NUdf::TDataType<NUdf::TUtf8>::Id>( - driver, - pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42)); - - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - - { - driver.ShardDbState.BeginTransaction(Shard1); - NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); - auto selectRes = db.Table<Schema1::Table1>() - .Key(ui32(42)) - .Select<Schema1::Table1::Value>(); - - UNIT_ASSERT(selectRes.IsReady() && selectRes.IsValid()); - UNIT_ASSERT(!selectRes.HaveValue<Schema1::Table1::Value>()); - driver.ShardDbState.CommitTransaction(Shard1); - } - } - + driver, + pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42)); + + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + + { + driver.ShardDbState.BeginTransaction(Shard1); + NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); + auto selectRes = db.Table<Schema1::Table1>() + .Key(ui32(42)) + .Select<Schema1::Table1::Value>(); + + UNIT_ASSERT(selectRes.IsReady() && selectRes.IsValid()); + UNIT_ASSERT(!selectRes.HaveValue<Schema1::Table1::Value>()); + driver.ShardDbState.CommitTransaction(Shard1); + } + } + Y_UNIT_TEST(TestUpdateRowNotExistErasePayload) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); @@ -1110,77 +1110,77 @@ Value { } Y_UNIT_TEST(TestSelectRangeFullWithoutColumnsNotExistsNullKey) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - auto& pgmBuilder = driver.PgmBuilder; + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - auto options = pgmBuilder.GetDefaultTableRangeOptions(); + auto options = pgmBuilder.GetDefaultTableRangeOptions(); TVector<ui32> keyTypes(1); keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; TRuntimeNode::TList rowFrom(1); - rowFrom[0] = pgmBuilder.NewNull(); - options.FromColumns = rowFrom; - options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeTermValue); - auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct - Struct { - Member { - Name: "myRes" - Type { - Kind: Optional - Optional { - Item { - Kind: Struct - Struct { - Member { - Name: "List" - Type { - Kind: List - List { - Item { - Kind: Struct - } - } - } - } - Member { - Name: "Truncated" - Type { - Kind: Data - Data { - Scheme: 6 - } - } - } - } - } - } - } - } - } -} -Value { - Struct { - Optional { - Struct { - } - Struct { - Bool: false - } - } - } -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - } - - + rowFrom[0] = pgmBuilder.NewNull(); + options.FromColumns = rowFrom; + options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeTermValue); + auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct + Struct { + Member { + Name: "myRes" + Type { + Kind: Optional + Optional { + Item { + Kind: Struct + Struct { + Member { + Name: "List" + Type { + Kind: List + List { + Item { + Kind: Struct + } + } + } + } + Member { + Name: "Truncated" + Type { + Kind: Data + Data { + Scheme: 6 + } + } + } + } + } + } + } + } + } +} +Value { + Struct { + Optional { + Struct { + } + Struct { + Bool: false + } + } + } +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + } + + Y_UNIT_TEST(TestSelectRangeFullExists) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); @@ -1410,115 +1410,115 @@ Value { } Y_UNIT_TEST(TestSelectRangeFullExistsTruncatedByItemsFromNull) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - - { - driver.ShardDbState.BeginTransaction(Shard1); - NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); - db.Table<Schema1::Table1>() - .Key(ui32(42)) - .Update(NIceDb::TUpdate<Schema1::Table1::Value>("asd")); - db.Table<Schema1::Table1>() - .Key(ui32(43)) - .Update(NIceDb::TUpdate<Schema1::Table1::Value>("qwe")); - - driver.ShardDbState.CommitTransaction(Shard1); - } - - auto& pgmBuilder = driver.PgmBuilder; + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + + { + driver.ShardDbState.BeginTransaction(Shard1); + NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); + db.Table<Schema1::Table1>() + .Key(ui32(42)) + .Update(NIceDb::TUpdate<Schema1::Table1::Value>("asd")); + db.Table<Schema1::Table1>() + .Key(ui32(43)) + .Update(NIceDb::TUpdate<Schema1::Table1::Value>("qwe")); + + driver.ShardDbState.CommitTransaction(Shard1); + } + + auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); - auto options = pgmBuilder.GetDefaultTableRangeOptions(); + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); + auto options = pgmBuilder.GetDefaultTableRangeOptions(); TVector<ui32> keyTypes(1); keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; TRuntimeNode::TList rowFrom(1); - rowFrom[0] = pgmBuilder.NewNull(); - options.FromColumns = rowFrom; - options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeTermValue); - options.ItemsLimit = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(1); - auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct - Struct { - Member { - Name: "myRes" - Type { - Kind: Optional - Optional { - Item { - Kind: Struct - Struct { - Member { - Name: "List" - Type { - Kind: List - List { - Item { - Kind: Struct - Struct { - Member { - Name: "2" - Type { - Kind: Optional - Optional { - Item { - Kind: Data - Data { - Scheme: 4608 - } - } - } - } - } - } - } - } - } - } - Member { - Name: "Truncated" - Type { - Kind: Data - Data { - Scheme: 6 - } - } - } - } - } - } - } - } - } -} -Value { - Struct { - Optional { - Struct { - List { - Struct { - Optional { - Text: "asd" - } - } - } - } - Struct { - Bool: true - } - } - } -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - } - + rowFrom[0] = pgmBuilder.NewNull(); + options.FromColumns = rowFrom; + options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeTermValue); + options.ItemsLimit = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(1); + auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct + Struct { + Member { + Name: "myRes" + Type { + Kind: Optional + Optional { + Item { + Kind: Struct + Struct { + Member { + Name: "List" + Type { + Kind: List + List { + Item { + Kind: Struct + Struct { + Member { + Name: "2" + Type { + Kind: Optional + Optional { + Item { + Kind: Data + Data { + Scheme: 4608 + } + } + } + } + } + } + } + } + } + } + Member { + Name: "Truncated" + Type { + Kind: Data + Data { + Scheme: 6 + } + } + } + } + } + } + } + } + } +} +Value { + Struct { + Optional { + Struct { + List { + Struct { + Optional { + Text: "asd" + } + } + } + } + Struct { + Bool: true + } + } + } +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + } + Y_UNIT_TEST(TestSelectRangeFullExistsTruncatedByBytes) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); @@ -2002,115 +2002,115 @@ Value { } Y_UNIT_TEST(TestSelectRangeNullNull) { - TDriver driver; - driver.ShardDbState.AddShard<Schema1>(Shard1); - - { - driver.ShardDbState.BeginTransaction(Shard1); - NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); - db.Table<Schema1::Table1>() - .Key(ui32(42)) - .Update(NIceDb::TUpdate<Schema1::Table1::Value>("asd")); - db.Table<Schema1::Table1>() - .Key(ui32(43)) - .Update(NIceDb::TUpdate<Schema1::Table1::Value>("qwe")); - db.Table<Schema1::Table1>() - .Key(ui32(44)) - .Update(NIceDb::TUpdate<Schema1::Table1::Value>("zsd")); - - driver.ShardDbState.CommitTransaction(Shard1); - } - - auto& pgmBuilder = driver.PgmBuilder; + TDriver driver; + driver.ShardDbState.AddShard<Schema1>(Shard1); + + { + driver.ShardDbState.BeginTransaction(Shard1); + NIceDb::TNiceDb db(*driver.ShardDbState.Dbs[Shard1]); + db.Table<Schema1::Table1>() + .Key(ui32(42)) + .Update(NIceDb::TUpdate<Schema1::Table1::Value>("asd")); + db.Table<Schema1::Table1>() + .Key(ui32(43)) + .Update(NIceDb::TUpdate<Schema1::Table1::Value>("qwe")); + db.Table<Schema1::Table1>() + .Key(ui32(44)) + .Update(NIceDb::TUpdate<Schema1::Table1::Value>("zsd")); + + driver.ShardDbState.CommitTransaction(Shard1); + } + + auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); - + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType); + TVector<ui32> keyTypes(1); keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); - rowFrom[0] = pgmBuilder.NewNull(); - rowTo[0] = pgmBuilder.NewNull(); - - auto options = pgmBuilder.GetDefaultTableRangeOptions(); - options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::IncludeTermValue); - options.FromColumns = rowFrom; - options.ToColumns = rowTo; - auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); - auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); - - NKikimrMiniKQL::TResult res; - UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); - auto resStr = res.DebugString(); - auto expectedStr = R"___(Type { - Kind: Struct - Struct { - Member { - Name: "myRes" - Type { - Kind: Optional - Optional { - Item { - Kind: Struct - Struct { - Member { - Name: "List" - Type { - Kind: List - List { - Item { - Kind: Struct - Struct { - Member { - Name: "2" - Type { - Kind: Optional - Optional { - Item { - Kind: Data - Data { - Scheme: 4608 - } - } - } - } - } - } - } - } - } - } - Member { - Name: "Truncated" - Type { - Kind: Data - Data { - Scheme: 6 - } - } - } - } - } - } - } - } - } -} -Value { - Struct { - Optional { - Struct { - } - Struct { - Bool: false - } - } - } -} -)___"; - UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); - } - + rowFrom[0] = pgmBuilder.NewNull(); + rowTo[0] = pgmBuilder.NewNull(); + + auto options = pgmBuilder.GetDefaultTableRangeOptions(); + options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::IncludeTermValue); + options.FromColumns = rowFrom; + options.ToColumns = rowTo; + auto value = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); + auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); + + NKikimrMiniKQL::TResult res; + UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); + auto resStr = res.DebugString(); + auto expectedStr = R"___(Type { + Kind: Struct + Struct { + Member { + Name: "myRes" + Type { + Kind: Optional + Optional { + Item { + Kind: Struct + Struct { + Member { + Name: "List" + Type { + Kind: List + List { + Item { + Kind: Struct + Struct { + Member { + Name: "2" + Type { + Kind: Optional + Optional { + Item { + Kind: Data + Data { + Scheme: 4608 + } + } + } + } + } + } + } + } + } + } + Member { + Name: "Truncated" + Type { + Kind: Data + Data { + Scheme: 6 + } + } + } + } + } + } + } + } + } +} +Value { + Struct { + Optional { + Struct { + } + Struct { + Bool: false + } + } + } +} +)___"; + UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); + } + Y_UNIT_TEST(TestSelectRangeToExclusive) { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); diff --git a/ydb/core/engine/mkql_keys.cpp b/ydb/core/engine/mkql_keys.cpp index a7b220e95c..05afb89adc 100644 --- a/ydb/core/engine/mkql_keys.cpp +++ b/ydb/core/engine/mkql_keys.cpp @@ -114,7 +114,7 @@ THolder<TKeyDesc> ExtractSelectRow(TCallable& callable, const TTypeEnvironment& } THolder<TKeyDesc> ExtractSelectRange(TCallable& callable, const TTypeEnvironment& env) { - MKQL_ENSURE(callable.GetInputsCount() >= 9 && callable.GetInputsCount() <= 13, "Expected 9 to 13 args"); + MKQL_ENSURE(callable.GetInputsCount() >= 9 && callable.GetInputsCount() <= 13, "Expected 9 to 13 args"); auto tableId = ExtractTableId(callable.GetInput(0)); auto columnsNode = callable.GetInput(1); MKQL_ENSURE(columnsNode.IsImmediate() && columnsNode.GetNode()->GetType()->IsType(), "Expected struct type"); @@ -320,36 +320,36 @@ TVector<THolder<TKeyDesc>> ExtractTableKeys(TExploringNodeVisitor& explorer, con } TTableId ExtractTableId(const TRuntimeNode& node) { - if (node.GetStaticType()->IsTuple()) { - const TTupleLiteral* tupleNode = AS_VALUE(TTupleLiteral, node); - MKQL_ENSURE(tupleNode->GetValuesCount() >= 3u, "Unexpected number of tuple items"); - const ui64 ownerId = AS_VALUE(TDataLiteral, tupleNode->GetValue(0))->AsValue().Get<ui64>(); - const ui64 localPathId = AS_VALUE(TDataLiteral, tupleNode->GetValue(1))->AsValue().Get<ui64>(); - const ui64 schemaVersion = AS_VALUE(TDataLiteral, tupleNode->GetValue(2))->AsValue().Get<ui64>(); - return TTableId(ownerId, localPathId, schemaVersion); - } else { - const TStringBuf buffer(AS_VALUE(TDataLiteral, node)->AsValue().AsStringRef()); - using TPair = std::pair<ui64, ui64>; - MKQL_ENSURE(sizeof(TPair) == buffer.size(), "Wrong table id buffer size."); - const auto pair = reinterpret_cast<const TPair*>(buffer.data()); - return TTableId(pair->first, pair->second); - } + if (node.GetStaticType()->IsTuple()) { + const TTupleLiteral* tupleNode = AS_VALUE(TTupleLiteral, node); + MKQL_ENSURE(tupleNode->GetValuesCount() >= 3u, "Unexpected number of tuple items"); + const ui64 ownerId = AS_VALUE(TDataLiteral, tupleNode->GetValue(0))->AsValue().Get<ui64>(); + const ui64 localPathId = AS_VALUE(TDataLiteral, tupleNode->GetValue(1))->AsValue().Get<ui64>(); + const ui64 schemaVersion = AS_VALUE(TDataLiteral, tupleNode->GetValue(2))->AsValue().Get<ui64>(); + return TTableId(ownerId, localPathId, schemaVersion); + } else { + const TStringBuf buffer(AS_VALUE(TDataLiteral, node)->AsValue().AsStringRef()); + using TPair = std::pair<ui64, ui64>; + MKQL_ENSURE(sizeof(TPair) == buffer.size(), "Wrong table id buffer size."); + const auto pair = reinterpret_cast<const TPair*>(buffer.data()); + return TTableId(pair->first, pair->second); + } +} + +void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, + const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env) +{ + for (ui32 i = 0; i < rowIndices.size(); ++i) { + auto rowIndex = rowIndices[i]; + + auto value = row.GetElement(rowIndex); + auto type = rowTypes[rowIndex]; + + // NOTE: We have to copy values here as some values inlined in TUnboxedValue + // cannot be be inlined in TCell. So set copy = true. + cells[i] = MakeCell(type, value, env, /* copy */true); + } } -void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, - const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env) -{ - for (ui32 i = 0; i < rowIndices.size(); ++i) { - auto rowIndex = rowIndices[i]; - - auto value = row.GetElement(rowIndex); - auto type = rowTypes[rowIndex]; - - // NOTE: We have to copy values here as some values inlined in TUnboxedValue - // cannot be be inlined in TCell. So set copy = true. - cells[i] = MakeCell(type, value, env, /* copy */true); - } } - } -} diff --git a/ydb/core/engine/mkql_keys.h b/ydb/core/engine/mkql_keys.h index 9fdfe27f64..635ad3ce14 100644 --- a/ydb/core/engine/mkql_keys.h +++ b/ydb/core/engine/mkql_keys.h @@ -45,8 +45,8 @@ THolder<TKeyDesc> ExtractTableKey(TCallable& callable, const TTableStrings& stri TVector<THolder<TKeyDesc>> ExtractTableKeys(TExploringNodeVisitor& explorer, const TTypeEnvironment& env); TTableId ExtractTableId(const TRuntimeNode& node); TCell MakeCell(NUdf::TDataTypeId typeId, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy = true); -void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, - const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env); +void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, + const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env); } } diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp index 3e8fc4991e..3dd053e372 100644 --- a/ydb/core/engine/mkql_proto.cpp +++ b/ydb/core/engine/mkql_proto.cpp @@ -16,130 +16,130 @@ namespace NMiniKQL { namespace { Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(TType* type, const Ydb::Value& value) { - auto dataType = static_cast<TDataType*>(type); - switch (dataType->GetSchemeType()) { - case NUdf::TDataType<bool>::Id: - return NUdf::TUnboxedValuePod(value.bool_value()); - case NUdf::TDataType<ui8>::Id: - return NUdf::TUnboxedValuePod(ui8(value.uint32_value())); - case NUdf::TDataType<i8>::Id: - return NUdf::TUnboxedValuePod(i8(value.int32_value())); - case NUdf::TDataType<ui16>::Id: - return NUdf::TUnboxedValuePod(ui16(value.uint32_value())); - case NUdf::TDataType<i16>::Id: - return NUdf::TUnboxedValuePod(i16(value.int32_value())); - case NUdf::TDataType<i32>::Id: - return NUdf::TUnboxedValuePod(value.int32_value()); - case NUdf::TDataType<ui32>::Id: - return NUdf::TUnboxedValuePod(value.uint32_value()); - case NUdf::TDataType<i64>::Id: - return NUdf::TUnboxedValuePod(value.int64_value()); - case NUdf::TDataType<ui64>::Id: - return NUdf::TUnboxedValuePod(value.uint64_value()); - case NUdf::TDataType<float>::Id: - return NUdf::TUnboxedValuePod(value.float_value()); - case NUdf::TDataType<double>::Id: - return NUdf::TUnboxedValuePod(value.double_value()); - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: - return MakeString(value.text_value()); - case NUdf::TDataType<NUdf::TDate>::Id: - return NUdf::TUnboxedValuePod(ui16(value.uint32_value())); - case NUdf::TDataType<NUdf::TDatetime>::Id: - return NUdf::TUnboxedValuePod(value.uint32_value()); - case NUdf::TDataType<NUdf::TTimestamp>::Id: - return NUdf::TUnboxedValuePod(value.uint64_value()); - case NUdf::TDataType<NUdf::TInterval>::Id: - return NUdf::TUnboxedValuePod(value.int64_value()); + auto dataType = static_cast<TDataType*>(type); + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + return NUdf::TUnboxedValuePod(value.bool_value()); + case NUdf::TDataType<ui8>::Id: + return NUdf::TUnboxedValuePod(ui8(value.uint32_value())); + case NUdf::TDataType<i8>::Id: + return NUdf::TUnboxedValuePod(i8(value.int32_value())); + case NUdf::TDataType<ui16>::Id: + return NUdf::TUnboxedValuePod(ui16(value.uint32_value())); + case NUdf::TDataType<i16>::Id: + return NUdf::TUnboxedValuePod(i16(value.int32_value())); + case NUdf::TDataType<i32>::Id: + return NUdf::TUnboxedValuePod(value.int32_value()); + case NUdf::TDataType<ui32>::Id: + return NUdf::TUnboxedValuePod(value.uint32_value()); + case NUdf::TDataType<i64>::Id: + return NUdf::TUnboxedValuePod(value.int64_value()); + case NUdf::TDataType<ui64>::Id: + return NUdf::TUnboxedValuePod(value.uint64_value()); + case NUdf::TDataType<float>::Id: + return NUdf::TUnboxedValuePod(value.float_value()); + case NUdf::TDataType<double>::Id: + return NUdf::TUnboxedValuePod(value.double_value()); + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: + return MakeString(value.text_value()); + case NUdf::TDataType<NUdf::TDate>::Id: + return NUdf::TUnboxedValuePod(ui16(value.uint32_value())); + case NUdf::TDataType<NUdf::TDatetime>::Id: + return NUdf::TUnboxedValuePod(value.uint32_value()); + case NUdf::TDataType<NUdf::TTimestamp>::Id: + return NUdf::TUnboxedValuePod(value.uint64_value()); + case NUdf::TDataType<NUdf::TInterval>::Id: + return NUdf::TUnboxedValuePod(value.int64_value()); case NUdf::TDataType<NUdf::TJsonDocument>::Id: return ValueFromString(NUdf::EDataSlot::JsonDocument, value.text_value()); case NUdf::TDataType<NUdf::TDyNumber>::Id: return ValueFromString(NUdf::EDataSlot::DyNumber, value.text_value()); - default: - return MakeString(value.bytes_value()); - } - } - + default: + return MakeString(value.bytes_value()); + } + } + } NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, const THolderFactory& factory) { - switch (type->GetKind()) { - case TType::EKind::Void: - return NUdf::TUnboxedValuePod::Void(); - + switch (type->GetKind()) { + case TType::EKind::Void: + return NUdf::TUnboxedValuePod::Void(); + case TType::EKind::Null: case TType::EKind::EmptyList: case TType::EKind::EmptyDict: return NUdf::TUnboxedValue(); - case TType::EKind::Data: + case TType::EKind::Data: return HandleKindDataImport(type, value); - - case TType::EKind::Optional: { - auto optionalType = static_cast<TOptionalType*>(type); + + case TType::EKind::Optional: { + auto optionalType = static_cast<TOptionalType*>(type); switch (value.value_case()) { case Ydb::Value::kNestedValue: - return ImportValueFromProto(optionalType->GetItemType(), value.nested_value(), factory).MakeOptional(); + return ImportValueFromProto(optionalType->GetItemType(), value.nested_value(), factory).MakeOptional(); case Ydb::Value::kNullFlagValue: - return NUdf::TUnboxedValue(); - default: - return ImportValueFromProto(optionalType->GetItemType(), value, factory).MakeOptional(); - } - } - - case TType::EKind::List: { - auto listType = static_cast<TListType*>(type); - auto itemType = listType->GetItemType(); - const auto& list = value.items(); - NUdf::TUnboxedValue *items = nullptr; - auto array = factory.CreateDirectArrayHolder(list.size(), items); - for (const auto& x : list) { - *items++ = ImportValueFromProto(itemType, x, factory); - } - - return std::move(array); - } - - case TType::EKind::Struct: { - auto structType = static_cast<TStructType*>(type); - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto res = factory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto memberType = structType->GetMemberType(index); - itemsPtr[index] = ImportValueFromProto(memberType, value.items(index), factory); - } - - return std::move(res); - } - - case TType::EKind::Tuple: { - auto tupleType = static_cast<TTupleType*>(type); - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto res = factory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - auto elementType = tupleType->GetElementType(index); - itemsPtr[index] = ImportValueFromProto(elementType, value.items(index), factory); - } - - return std::move(res); - } - - case TType::EKind::Dict: { - auto dictType = static_cast<TDictType*>(type); - auto keyType = dictType->GetKeyType(); - auto payloadType = dictType->GetPayloadType(); + return NUdf::TUnboxedValue(); + default: + return ImportValueFromProto(optionalType->GetItemType(), value, factory).MakeOptional(); + } + } + + case TType::EKind::List: { + auto listType = static_cast<TListType*>(type); + auto itemType = listType->GetItemType(); + const auto& list = value.items(); + NUdf::TUnboxedValue *items = nullptr; + auto array = factory.CreateDirectArrayHolder(list.size(), items); + for (const auto& x : list) { + *items++ = ImportValueFromProto(itemType, x, factory); + } + + return std::move(array); + } + + case TType::EKind::Struct: { + auto structType = static_cast<TStructType*>(type); + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto res = factory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + auto memberType = structType->GetMemberType(index); + itemsPtr[index] = ImportValueFromProto(memberType, value.items(index), factory); + } + + return std::move(res); + } + + case TType::EKind::Tuple: { + auto tupleType = static_cast<TTupleType*>(type); + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto res = factory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + auto elementType = tupleType->GetElementType(index); + itemsPtr[index] = ImportValueFromProto(elementType, value.items(index), factory); + } + + return std::move(res); + } + + case TType::EKind::Dict: { + auto dictType = static_cast<TDictType*>(type); + auto keyType = dictType->GetKeyType(); + auto payloadType = dictType->GetPayloadType(); auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed); - - for (const auto& x : value.pairs()) { - dictBuilder->Add( - ImportValueFromProto(keyType, x.key(), factory), - ImportValueFromProto(payloadType, x.payload(), factory) - ); - } - - return dictBuilder->Build(); - } - + + for (const auto& x : value.pairs()) { + dictBuilder->Add( + ImportValueFromProto(keyType, x.key(), factory), + ImportValueFromProto(payloadType, x.payload(), factory) + ); + } + + return dictBuilder->Build(); + } + case TType::EKind::Variant: { auto variantType = static_cast<TVariantType*>(type); auto index = value.variant_index(); @@ -154,12 +154,12 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c return std::move(unboxedValue); } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } -} - - + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } +} + + template <typename ValType> class TAlmostDoneTypeValue : public TRawTypeValue { public: diff --git a/ydb/core/engine/mkql_proto.h b/ydb/core/engine/mkql_proto.h index 0e587366d5..510ff15991 100644 --- a/ydb/core/engine/mkql_proto.h +++ b/ydb/core/engine/mkql_proto.h @@ -10,7 +10,7 @@ namespace NKikimr { namespace NMiniKQL { -class THolderFactory; +class THolderFactory; NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, const THolderFactory& factory); diff --git a/ydb/core/engine/mkql_proto_ut.cpp b/ydb/core/engine/mkql_proto_ut.cpp index 86a4d23d03..c6e00e947c 100644 --- a/ydb/core/engine/mkql_proto_ut.cpp +++ b/ydb/core/engine/mkql_proto_ut.cpp @@ -19,376 +19,376 @@ namespace NMiniKQL { Y_UNIT_TEST_SUITE(TMiniKQLProtoTestYdb) { Y_UNIT_TEST(TestExportVoidTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewVoid(); - return pgmReturn; - }, "void_type: NULL_VALUE\n"); - } - + auto pgmReturn = pgmBuilder.NewVoid(); + return pgmReturn; + }, "void_type: NULL_VALUE\n"); + } + Y_UNIT_TEST(TestExportDataTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); - return pgmReturn; - }, - "type_id: INT32\n"); - } - - Y_UNIT_TEST(TestExportDecimalTypeYdb) { - TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); + return pgmReturn; + }, + "type_id: INT32\n"); + } + + Y_UNIT_TEST(TestExportDecimalTypeYdb) { + TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { NYql::NDecimal::TInt128 x; ui64* p = (ui64*)&x; p[0] = 1; p[1] = 0; auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, - "decimal_type {\n" - " precision: 21\n" - " scale: 8\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportUuidTypeYdb) { - TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, + "decimal_type {\n" + " precision: 21\n" + " scale: 8\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportUuidTypeYdb) { + TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Uuid>("\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv); - return pgmReturn; - }, - "type_id: UUID\n"); - } - + return pgmReturn; + }, + "type_id: UUID\n"); + } + Y_UNIT_TEST(TestExportOptionalTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<i32>(42)); - return pgmReturn; - }, - "optional_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - "}\n"); - } - + auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<i32>(42)); + return pgmReturn; + }, + "optional_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(TestExportListTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<i32>(42)); - return pgmReturn; - }, - "list_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - "}\n"); - } - + auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<i32>(42)); + return pgmReturn; + }, + "list_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(TestExportTupleTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { TRuntimeNode::TList items; - items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); - items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - auto pgmReturn = pgmBuilder.NewTuple(items); - return pgmReturn; - }, - "tuple_type {\n" - " elements {\n" - " type_id: INT32\n" - " }\n" - " elements {\n" - " type_id: STRING\n" - " }\n" - "}\n"); - } - + items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); + items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + auto pgmReturn = pgmBuilder.NewTuple(items); + return pgmReturn; + }, + "tuple_type {\n" + " elements {\n" + " type_id: INT32\n" + " }\n" + " elements {\n" + " type_id: STRING\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(TestExportStructTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - "struct_type {\n" - " members {\n" - " name: \"x\"\n" - " type {\n" - " type_id: INT32\n" - " }\n" - " }\n" - " members {\n" - " name: \"y\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" - "}\n"); - } - + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + "struct_type {\n" + " members {\n" + " name: \"x\"\n" + " type {\n" + " type_id: INT32\n" + " }\n" + " }\n" + " members {\n" + " name: \"y\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(TestExportDictTypeYdb) { TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), - pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); - TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; - items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewDict(dictType, items); - return pgmReturn; - }, - "dict_type {\n" - " key {\n" - " type_id: INT32\n" - " }\n" - " payload {\n" - " type_id: STRING\n" - " }\n" - "}\n"); - } - - Y_UNIT_TEST(TestExportVariantTupleTypeYdb) { - const TString expected = R"___(variant_type { - tuple_items { - elements { - type_id: UINT64 - } - elements { - type_id: UINT32 - } - } -} -)___"; - TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { - std::vector<TType*> tupleElemenTypes; - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - 1, - pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); - return pgmReturn; - }, - expected); - } - -Y_UNIT_TEST(TestExportVariantStructTypeYdb) { - const TString expected = R"___(variant_type { - struct_items { - members { - name: "a" - type { - type_id: UINT64 - } - } - members { - name: "b" - type { - type_id: UINT32 - } - } - } -} -)___"; - - TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { + auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; + items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewDict(dictType, items); + return pgmReturn; + }, + "dict_type {\n" + " key {\n" + " type_id: INT32\n" + " }\n" + " payload {\n" + " type_id: STRING\n" + " }\n" + "}\n"); + } + + Y_UNIT_TEST(TestExportVariantTupleTypeYdb) { + const TString expected = R"___(variant_type { + tuple_items { + elements { + type_id: UINT64 + } + elements { + type_id: UINT32 + } + } +} +)___"; + TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + 1, + pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); + return pgmReturn; + }, + expected); + } + +Y_UNIT_TEST(TestExportVariantStructTypeYdb) { + const TString expected = R"___(variant_type { + struct_items { + members { + name: "a" + type { + type_id: UINT64 + } + } + members { + name: "b" + type { + type_id: UINT32 + } + } + } +} +)___"; + + TestExportType<Ydb::Type>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TType*>> structElemenTypes; structElemenTypes.push_back({"a", pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); structElemenTypes.push_back({"b", pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)}); - TType* structType = pgmBuilder.NewStructType(structElemenTypes); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - "b", - pgmBuilder.NewVariantType(structType)); - return pgmReturn; - }, - expected); - } - + TType* structType = pgmBuilder.NewStructType(structElemenTypes); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + "b", + pgmBuilder.NewVariantType(structType)); + return pgmReturn; + }, + expected); + } + Y_UNIT_TEST(TestExportVoidYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewVoid(); - return pgmReturn; - }, ""); - } - + auto pgmReturn = pgmBuilder.NewVoid(); + return pgmReturn; + }, ""); + } + Y_UNIT_TEST(TestExportBoolYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral(true); - return pgmReturn; - }, "bool_value: true\n"); - } - + auto pgmReturn = pgmBuilder.NewDataLiteral(true); + return pgmReturn; + }, "bool_value: true\n"); + } + Y_UNIT_TEST(TestExportIntegralYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); - return pgmReturn; - }, "int32_value: 42\n"); - } - + auto pgmReturn = pgmBuilder.NewDataLiteral<i32>(42); + return pgmReturn; + }, "int32_value: 42\n"); + } + Y_UNIT_TEST(TestExportDoubleYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral(3.5); - return pgmReturn; - }, "double_value: 3.5\n"); - } - - Y_UNIT_TEST(TestExportStringYdb) { + auto pgmReturn = pgmBuilder.NewDataLiteral(3.5); + return pgmReturn; + }, "double_value: 3.5\n"); + } + + Y_UNIT_TEST(TestExportStringYdb) { + TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { + auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + return pgmReturn; + }, "bytes_value: \"abc\"\n"); + } + + Y_UNIT_TEST(TestExportUuidYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"); - return pgmReturn; - }, "bytes_value: \"abc\"\n"); - } - - Y_UNIT_TEST(TestExportUuidYdb) { - TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { auto pgmReturn = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Uuid>("\1\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0"sv); - return pgmReturn; - }, "low_128: 1\n" - "high_128: 2\n"); - } - - Y_UNIT_TEST(TestExportDecimalYdb) { - TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, "low_128: 1\n" + "high_128: 2\n"); + } + + Y_UNIT_TEST(TestExportDecimalYdb) { + TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { NYql::NDecimal::TInt128 x; ui64* p = (ui64*)&x; p[0] = 1; p[1] = 0; auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "low_128: 1\n"); - } - - Y_UNIT_TEST(TestExportDecimalNegativeYdb) { - TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, "low_128: 1\n"); + } + + Y_UNIT_TEST(TestExportDecimalNegativeYdb) { + TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { NYql::NDecimal::TInt128 x; ui64* p = (ui64*)&x; p[0] = Max<ui64>(); p[1] = Max<ui64>(); auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "low_128: 18446744073709551615\n" - "high_128: 18446744073709551615\n"); - } - - Y_UNIT_TEST(TestExportDecimalHugeYdb) { - TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { + return pgmReturn; + }, "low_128: 18446744073709551615\n" + "high_128: 18446744073709551615\n"); + } + + Y_UNIT_TEST(TestExportDecimalHugeYdb) { + TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { NYql::NDecimal::TInt128 x; ui64* p = (ui64*)&x; p[0] = 0; p[1] = 1; auto pgmReturn = pgmBuilder.NewDecimalLiteral(x, 21, 8); - return pgmReturn; - }, "low_128: 0\n" - "high_128: 1\n"); - } - + return pgmReturn; + }, "low_128: 0\n" + "high_128: 1\n"); + } + Y_UNIT_TEST(TestExportEmptyOptionalYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); - return pgmReturn; - }, "null_flag_value: NULL_VALUE\n"); - } - + auto pgmReturn = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + return pgmReturn; + }, "null_flag_value: NULL_VALUE\n"); + } + Y_UNIT_TEST(TestExportEmptyOptionalOptionalYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional( - pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); - return pgmReturn; - }, - "nested_value {\n" - " null_flag_value: NULL_VALUE\n" - "}\n"); - } - + auto pgmReturn = pgmBuilder.NewOptional( + pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id)); + return pgmReturn; + }, + "nested_value {\n" + " null_flag_value: NULL_VALUE\n" + "}\n"); + } + Y_UNIT_TEST(TestExportMultipleOptionalNotEmptyYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional( - pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"))); - return pgmReturn; - }, "bytes_value: \"abc\"\n"); - } - + auto pgmReturn = pgmBuilder.NewOptional( + pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc"))); + return pgmReturn; + }, "bytes_value: \"abc\"\n"); + } + Y_UNIT_TEST(TestExportOptionalYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - return pgmReturn; - }, "bytes_value: \"abc\"\n"); - } - + auto pgmReturn = pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + return pgmReturn; + }, "bytes_value: \"abc\"\n"); + } + Y_UNIT_TEST(TestExportListYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - return pgmReturn; - }, - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"); - } - + auto pgmReturn = pgmBuilder.AsList(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + return pgmReturn; + }, + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(TestExportTupleYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { TRuntimeNode::TList items; - items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); - items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); - auto pgmReturn = pgmBuilder.NewTuple(items); - return pgmReturn; - }, - "items {\n" - " int32_value: 42\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"); - } - + items.push_back(pgmBuilder.NewDataLiteral<i32>(42)); + items.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc")); + auto pgmReturn = pgmBuilder.NewTuple(items); + return pgmReturn; + }, + "items {\n" + " int32_value: 42\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(TestExportStructYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - "items {\n" - " int32_value: 42\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"); - } - + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + "items {\n" + " int32_value: 42\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(TestExportDictYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), - pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); - TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; - items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewDict(dictType, items); - return pgmReturn; - }, - "pairs {\n" - " key {\n" - " int32_value: 42\n" - " }\n" - " payload {\n" - " bytes_value: \"abc\"\n" - " }\n" - "}\n"); - } - + auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> items; + items.push_back({ pgmBuilder.NewDataLiteral<i32>(42), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewDict(dictType, items); + return pgmReturn; + }, + "pairs {\n" + " key {\n" + " int32_value: 42\n" + " }\n" + " payload {\n" + " bytes_value: \"abc\"\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(TestExportVariantYdb) { TestExportValue<Ydb::Value>([](TProgramBuilder& pgmBuilder) { - std::vector<TType*> tupleElemenTypes; - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); - tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto pgmReturn = pgmBuilder.NewVariant( - pgmBuilder.NewDataLiteral<ui32>(66), - 1, - pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); - return pgmReturn; - }, - "nested_value {\n" - " uint32_value: 66\n" - "}\n" - "variant_index: 1\n"); - } - + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto pgmReturn = pgmBuilder.NewVariant( + pgmBuilder.NewDataLiteral<ui32>(66), + 1, + pgmBuilder.NewVariantType(pgmBuilder.NewTupleType(tupleElemenTypes))); + return pgmReturn; + }, + "nested_value {\n" + " uint32_value: 66\n" + "}\n" + "variant_index: 1\n"); + } + TString DoTestCellsFromTuple(const TConstArrayRef<NScheme::TTypeId>& types, TString paramsProto) { NKikimrMiniKQL::TParams params; bool parseOk = ::google::protobuf::TextFormat::ParseFromString(paramsProto, ¶ms); diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 986a116216..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -1,36 +1,36 @@ -#pragma once - -#include <grpc++/support/byte_buffer.h> -#include <grpc++/support/slice.h> +#pragma once + +#include <grpc++/support/byte_buffer.h> +#include <grpc++/support/slice.h> #include <library/cpp/grpc/server/grpc_request_base.h> #include <library/cpp/string_utils/quote/quote.h> - + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/api/protos/ydb_common.pb.h> - + #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue_manager.h> - + #include <ydb/core/grpc_streaming/grpc_streaming.h> #include <ydb/core/tx/scheme_board/events.h> #include <ydb/core/base/events.h> - -namespace NKikimrConfig { -class TAppConfig; -} - -namespace NKikimr { - -namespace NSchemeCache { -struct TSchemeCacheNavigate; -} - + +namespace NKikimrConfig { +class TAppConfig; +} + +namespace NKikimr { + +namespace NSchemeCache { +struct TSchemeCacheNavigate; +} + namespace NRpcService { struct TRlPath { @@ -40,30 +40,30 @@ struct TRlPath { } -namespace NGRpcService { - -using TYdbIssueMessageType = Ydb::Issue::IssueMessage; - +namespace NGRpcService { + +using TYdbIssueMessageType = Ydb::Issue::IssueMessage; + std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TString& path); -std::pair<TString, TString> SplitPath(const TString& path); +std::pair<TString, TString> SplitPath(const TString& path); void RefreshToken(const TString& token, const TString& database, const TActorContext& ctx, TActorId id); - -struct TRpcServices { - enum EServiceId { - EvMakeDirectory = EventSpaceBegin(TKikimrEvents::ES_GRPC_CALLS), - EvRemoveDirectory, - EvAlterTable, - EvCreateTable, - EvDropTable, - EvCopyTable, + +struct TRpcServices { + enum EServiceId { + EvMakeDirectory = EventSpaceBegin(TKikimrEvents::ES_GRPC_CALLS), + EvRemoveDirectory, + EvAlterTable, + EvCreateTable, + EvDropTable, + EvCopyTable, EvCopyTables, EvListDirectory, EvRenameTables, - EvDescribeTable, - EvDescribePath, - EvGetOperation, - EvCreateSession, - EvDeleteSession, + EvDescribeTable, + EvDescribePath, + EvGetOperation, + EvCreateSession, + EvDeleteSession, EvKeepAlive = EvDeleteSession + 3, EvReadTable = EvKeepAlive + 3, EvGrpcStreamIsReady, @@ -78,7 +78,7 @@ struct TRpcServices { EvRemoveTenant, EvBeginTransaction, EvCommitTransaction, - EvRollbackTransaction, + EvRollbackTransaction, EvModifyPermissions, EvListEndpoints, EvDescribeTenantOptions, @@ -95,7 +95,7 @@ struct TRpcServices { EvS3Listing, EvExplainDataQueryAst, EvReadColumns, - EvBiStreamPing, + EvBiStreamPing, EvRefreshTokenRequest, // internal call EvGetShardLocations, EvExperimentalStreamQuery, @@ -208,250 +208,250 @@ struct TRpcServices { EvListYndxRateLimiterResources, EvDescribeYndxRateLimiterResource, EvAcquireYndxRateLimiterResource, - EvGrpcRuntimeRequest // !!! DO NOT ADD NEW REQUEST !!! - }; - - struct TEvGrpcNextReply : public TEventLocal<TEvGrpcNextReply, TRpcServices::EvGrpcStreamIsReady> { - TEvGrpcNextReply(size_t left) - : LeftInQueue(left) - {} - const size_t LeftInQueue; - }; + EvGrpcRuntimeRequest // !!! DO NOT ADD NEW REQUEST !!! + }; + + struct TEvGrpcNextReply : public TEventLocal<TEvGrpcNextReply, TRpcServices::EvGrpcStreamIsReady> { + TEvGrpcNextReply(size_t left) + : LeftInQueue(left) + {} + const size_t LeftInQueue; + }; struct TEvCancelOperation : public TEventLocal<TEvCancelOperation, TRpcServices::EvCancelOperation> {}; struct TEvForgetOperation : public TEventLocal<TEvForgetOperation, TRpcServices::EvForgetOperation> {}; -}; - +}; + // Should be specialized for real responses template <class T> void FillYdbStatus(T& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status); - -class TProtoResponseHelper { -public: - template<typename T, typename C> - static void SendProtoResponse(const T& r, Ydb::StatusIds::StatusCode status, C& ctx) { - T* resp = google::protobuf::Arena::CreateMessage<T>(ctx->GetArena()); - resp->CopyFrom(r); - ctx->Reply(resp, status); - } -}; - -class IRequestCtxBase { -public: - virtual ~IRequestCtxBase() = default; - // Returns client provided database name - virtual const TMaybe<TString> GetDatabaseName() const = 0; - // Returns "internal" token (result of ticket parser authentication) - virtual const TString& GetInternalToken() const = 0; - // Reply using YDB status code - virtual void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) = 0; - // Reply using "transport error code" - virtual void ReplyWithRpcStatus(grpc::StatusCode code, const TString& msg = "") = 0; - // Return address of the peer - virtual TString GetPeerName() const = 0; - // Return deadile of request execution, calculated from client timeout by grpc - virtual TInstant GetDeadline() const = 0; + +class TProtoResponseHelper { +public: + template<typename T, typename C> + static void SendProtoResponse(const T& r, Ydb::StatusIds::StatusCode status, C& ctx) { + T* resp = google::protobuf::Arena::CreateMessage<T>(ctx->GetArena()); + resp->CopyFrom(r); + ctx->Reply(resp, status); + } +}; + +class IRequestCtxBase { +public: + virtual ~IRequestCtxBase() = default; + // Returns client provided database name + virtual const TMaybe<TString> GetDatabaseName() const = 0; + // Returns "internal" token (result of ticket parser authentication) + virtual const TString& GetInternalToken() const = 0; + // Reply using YDB status code + virtual void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) = 0; + // Reply using "transport error code" + virtual void ReplyWithRpcStatus(grpc::StatusCode code, const TString& msg = "") = 0; + // Return address of the peer + virtual TString GetPeerName() const = 0; + // Return deadile of request execution, calculated from client timeout by grpc + virtual TInstant GetDeadline() const = 0; // Meta value from request virtual const TMaybe<TString> GetPeerMetaValues(const TString&) const = 0; - // Returns path and resource for rate limiter - virtual TMaybe<NRpcService::TRlPath> GetRlPath() const = 0; - // Raise issue on the context - virtual void RaiseIssue(const NYql::TIssue& issue) = 0; - virtual void RaiseIssues(const NYql::TIssues& issues) = 0; - virtual TMaybe<TString> GetTraceId() const = 0; - virtual const TString& GetRequestName() const = 0; -}; - -class TRespHookCtx : public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TRespHookCtx>; - using TContextPtr = TIntrusivePtr<NGrpc::IRequestContextBase>; - using TMessage = NProtoBuf::Message; - TRespHookCtx(TContextPtr ctx, TMessage* data, const TString& requestName, ui64 ru, ui32 status) - : Ctx_(ctx) - , RespData_(data) - , RequestName_(requestName) - , Ru_(ru) - , Status_(status) - {} - - void Pass() { - Ctx_->Reply(RespData_, Status_); - } - - ui64 GetConsumedRu() const { - return Ru_; - } - - const TString& GetRequestName() const { - return RequestName_; - } - -private: + // Returns path and resource for rate limiter + virtual TMaybe<NRpcService::TRlPath> GetRlPath() const = 0; + // Raise issue on the context + virtual void RaiseIssue(const NYql::TIssue& issue) = 0; + virtual void RaiseIssues(const NYql::TIssues& issues) = 0; + virtual TMaybe<TString> GetTraceId() const = 0; + virtual const TString& GetRequestName() const = 0; +}; + +class TRespHookCtx : public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TRespHookCtx>; + using TContextPtr = TIntrusivePtr<NGrpc::IRequestContextBase>; + using TMessage = NProtoBuf::Message; + TRespHookCtx(TContextPtr ctx, TMessage* data, const TString& requestName, ui64 ru, ui32 status) + : Ctx_(ctx) + , RespData_(data) + , RequestName_(requestName) + , Ru_(ru) + , Status_(status) + {} + + void Pass() { + Ctx_->Reply(RespData_, Status_); + } + + ui64 GetConsumedRu() const { + return Ru_; + } + + const TString& GetRequestName() const { + return RequestName_; + } + +private: TIntrusivePtr<NGrpc::IRequestContextBase> Ctx_; - TMessage* RespData_; //Allocated on arena owned by implementation of IRequestContextBase - const TString RequestName_; - const ui64 Ru_; - const ui32 Status_; -}; - -using TRespHook = std::function<void(TRespHookCtx::TPtr ctx)>; - -enum class TRateLimiterMode : ui8 { - Off = 0, - Rps = 1, - Ru = 2, - RuOnProgress = 3, -}; - -class ICheckerIface; - -// The way to pass some common data to request processing -class IFacilityProvider { -public: - virtual const NKikimrConfig::TAppConfig& GetAppConfig() const = 0; -}; - -struct TRequestAuxSettings { - TRateLimiterMode RlMode = TRateLimiterMode::Off; + TMessage* RespData_; //Allocated on arena owned by implementation of IRequestContextBase + const TString RequestName_; + const ui64 Ru_; + const ui32 Status_; +}; + +using TRespHook = std::function<void(TRespHookCtx::TPtr ctx)>; + +enum class TRateLimiterMode : ui8 { + Off = 0, + Rps = 1, + Ru = 2, + RuOnProgress = 3, +}; + +class ICheckerIface; + +// The way to pass some common data to request processing +class IFacilityProvider { +public: + virtual const NKikimrConfig::TAppConfig& GetAppConfig() const = 0; +}; + +struct TRequestAuxSettings { + TRateLimiterMode RlMode = TRateLimiterMode::Off; void (*CustomAttributeProcessor)(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData, ICheckerIface*) = nullptr; -}; - -// grpc_request_proxy part -// The interface is used to perform authentication and check database access right -class IRequestProxyCtx : public virtual IRequestCtxBase { -public: - virtual ~IRequestProxyCtx() = default; - virtual const TMaybe<TString> GetYdbToken() const = 0; +}; + +// grpc_request_proxy part +// The interface is used to perform authentication and check database access right +class IRequestProxyCtx : public virtual IRequestCtxBase { +public: + virtual ~IRequestProxyCtx() = default; + virtual const TMaybe<TString> GetYdbToken() const = 0; virtual void UpdateAuthState(NGrpc::TAuthState::EAuthState state) = 0; - virtual void SetInternalToken(const TString& token) = 0; + virtual void SetInternalToken(const TString& token) = 0; virtual const NGrpc::TAuthState& GetAuthState() const = 0; - virtual void ReplyUnauthenticated(const TString& msg = "") = 0; - virtual void ReplyUnavaliable() = 0; + virtual void ReplyUnauthenticated(const TString& msg = "") = 0; + virtual void ReplyUnavaliable() = 0; virtual bool Validate(TString& error) = 0; virtual void UseDatabase(const TString& database) = 0; - - // This method allows to set hook for unary call. - // The hook will be called at the reply time - // TRespHookCtx::Ptr will be passed in to the hook, it is allow - // to store the ctx somwhere to delay reply and then call Pass() to send response. - virtual void SetRespHook(TRespHook&& hook) = 0; - virtual void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) = 0; - virtual TRateLimiterMode GetRlMode() const = 0; + + // This method allows to set hook for unary call. + // The hook will be called at the reply time + // TRespHookCtx::Ptr will be passed in to the hook, it is allow + // to store the ctx somwhere to delay reply and then call Pass() to send response. + virtual void SetRespHook(TRespHook&& hook) = 0; + virtual void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) = 0; + virtual TRateLimiterMode GetRlMode() const = 0; virtual bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData, - ICheckerIface* iface) = 0; - - // Pass request for next processing - virtual void Pass(const IFacilityProvider& facility) = 0; - -}; - -// Request context -// The interface is used for rpc_ request actors -class IRequestCtx : public virtual IRequestCtxBase { - friend class TProtoResponseHelper; -public: - virtual void SetClientLostAction(std::function<void()>&& cb) = 0; - virtual const google::protobuf::Message* GetRequest() const = 0; - virtual google::protobuf::Arena* GetArena() = 0; - virtual const TMaybe<TString> GetRequestType() const = 0; - virtual void SetRuHeader(ui64 ru) = 0; + ICheckerIface* iface) = 0; + + // Pass request for next processing + virtual void Pass(const IFacilityProvider& facility) = 0; + +}; + +// Request context +// The interface is used for rpc_ request actors +class IRequestCtx : public virtual IRequestCtxBase { + friend class TProtoResponseHelper; +public: + virtual void SetClientLostAction(std::function<void()>&& cb) = 0; + virtual const google::protobuf::Message* GetRequest() const = 0; + virtual google::protobuf::Arena* GetArena() = 0; + virtual const TMaybe<TString> GetRequestType() const = 0; + virtual void SetRuHeader(ui64 ru) = 0; virtual void AddServerHint(const TString& hint) = 0; - virtual void SetCostInfo(float consumed_units) = 0; - -private: - virtual void Reply(NProtoBuf::Message* resp, ui32 status = 0) = 0; -}; - -class IRequestOpCtx : public IRequestCtx { -public: - virtual void SendOperation(const Ydb::Operations::Operation& operation) = 0; - virtual void SendResult(const google::protobuf::Message& result, - Ydb::StatusIds::StatusCode status) = 0; - // Legacy, do not use for modern code - virtual void SendResult(const google::protobuf::Message& result, Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message) = 0; - // Legacy, do not use for modern code - virtual void SendResult(Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message) = 0; -}; - -class IRequestNoOpCtx : public IRequestCtx { -}; - -struct TCommonResponseFillerImpl { - template <typename T> - static void FillImpl(T& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { - resp.set_status(status); - NYql::IssuesToMessage(issues, resp.mutable_issues()); - } -}; - -template <typename TResp, bool IsOperation = true> -struct TCommonResponseFiller : private TCommonResponseFillerImpl { - static void Fill(TResp& resp, const NYql::TIssues& issues, Ydb::CostInfo* costInfo, Ydb::StatusIds::StatusCode status) { - auto& operation = *resp.mutable_operation(); - operation.set_ready(true); - if (costInfo) { - operation.mutable_cost_info()->Swap(costInfo); - } - FillImpl(operation, issues, status); - } -}; - -template <typename TResp> -struct TCommonResponseFiller<TResp, false> : private TCommonResponseFillerImpl { - static void Fill(TResp& resp, const NYql::TIssues& issues, Ydb::CostInfo*, Ydb::StatusIds::StatusCode status) { - FillImpl(resp, issues, status); - } -}; - -class TRefreshTokenImpl - : public IRequestProxyCtx - , public TEventLocal<TRefreshTokenImpl, TRpcServices::EvRefreshTokenRequest> { -public: + virtual void SetCostInfo(float consumed_units) = 0; + +private: + virtual void Reply(NProtoBuf::Message* resp, ui32 status = 0) = 0; +}; + +class IRequestOpCtx : public IRequestCtx { +public: + virtual void SendOperation(const Ydb::Operations::Operation& operation) = 0; + virtual void SendResult(const google::protobuf::Message& result, + Ydb::StatusIds::StatusCode status) = 0; + // Legacy, do not use for modern code + virtual void SendResult(const google::protobuf::Message& result, Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message) = 0; + // Legacy, do not use for modern code + virtual void SendResult(Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message) = 0; +}; + +class IRequestNoOpCtx : public IRequestCtx { +}; + +struct TCommonResponseFillerImpl { + template <typename T> + static void FillImpl(T& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { + resp.set_status(status); + NYql::IssuesToMessage(issues, resp.mutable_issues()); + } +}; + +template <typename TResp, bool IsOperation = true> +struct TCommonResponseFiller : private TCommonResponseFillerImpl { + static void Fill(TResp& resp, const NYql::TIssues& issues, Ydb::CostInfo* costInfo, Ydb::StatusIds::StatusCode status) { + auto& operation = *resp.mutable_operation(); + operation.set_ready(true); + if (costInfo) { + operation.mutable_cost_info()->Swap(costInfo); + } + FillImpl(operation, issues, status); + } +}; + +template <typename TResp> +struct TCommonResponseFiller<TResp, false> : private TCommonResponseFillerImpl { + static void Fill(TResp& resp, const NYql::TIssues& issues, Ydb::CostInfo*, Ydb::StatusIds::StatusCode status) { + FillImpl(resp, issues, status); + } +}; + +class TRefreshTokenImpl + : public IRequestProxyCtx + , public TEventLocal<TRefreshTokenImpl, TRpcServices::EvRefreshTokenRequest> { +public: TRefreshTokenImpl(const TString& token, const TString& database, TActorId from) - : Token_(token) - , Database_(database) - , From_(from) - , State_(true) - { } - - const TMaybe<TString> GetYdbToken() const override { - return Token_; - } - + : Token_(token) + , Database_(database) + , From_(from) + , State_(true) + { } + + const TMaybe<TString> GetYdbToken() const override { + return Token_; + } + void UpdateAuthState(NGrpc::TAuthState::EAuthState state) override { - State_.State = state; - } - - void SetInternalToken(const TString& token) override { - InternalToken_ = token; - } - - const TMaybe<TString> GetDatabaseName() const override { - return Database_; - } - + State_.State = state; + } + + void SetInternalToken(const TString& token) override { + InternalToken_ = token; + } + + const TMaybe<TString> GetDatabaseName() const override { + return Database_; + } + const NGrpc::TAuthState& GetAuthState() const override { - return State_; - } - - const TString& GetInternalToken() const override { - return InternalToken_; - } - + return State_; + } + + const TString& GetInternalToken() const override { + return InternalToken_; + } + TString GetPeerName() const override { return {}; } void SetRlPath(TMaybe<NRpcService::TRlPath>&&) override {} - - TMaybe<NRpcService::TRlPath> GetRlPath() const override { - return Nothing(); - } - + + TMaybe<NRpcService::TRlPath> GetRlPath() const override { + return Nothing(); + } + const TMaybe<TString> GetPeerMetaValues(const TString&) const override { Y_FAIL("Unimplemented"); return TMaybe<TString>{}; @@ -461,20 +461,20 @@ public: Y_FAIL("Unimplemented"); } - void ReplyUnauthenticated(const TString&) override; - void ReplyUnavaliable() override; - void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode) override { - Y_FAIL("Unimplemented"); - } - - void RaiseIssue(const NYql::TIssue& issue) override { - IssueManager_.RaiseIssue(issue); - } - - void RaiseIssues(const NYql::TIssues& issues) override { - IssueManager_.RaiseIssues(issues); - } - + void ReplyUnauthenticated(const TString&) override; + void ReplyUnavaliable() override; + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode) override { + Y_FAIL("Unimplemented"); + } + + void RaiseIssue(const NYql::TIssue& issue) override { + IssueManager_.RaiseIssue(issue); + } + + void RaiseIssues(const NYql::TIssues& issues) override { + IssueManager_.RaiseIssues(issues); + } + bool Validate(TString&) override { return true; } @@ -484,152 +484,152 @@ public: } TActorId GetFromId() const { - return From_; - } - - const TString& GetRequestName() const override { - static TString str = "refresh token internal request"; - return str; - } - - TMaybe<TString> GetTraceId() const override { - return {}; - } - - TMaybe<TString> GetSdkBuildInfo() const { - return {}; - } - - TMaybe<TString> GetGrpcUserAgent() const { - return {}; - } - - TInstant GetDeadline() const override { - return TInstant::Max(); - } - + return From_; + } + + const TString& GetRequestName() const override { + static TString str = "refresh token internal request"; + return str; + } + + TMaybe<TString> GetTraceId() const override { + return {}; + } + + TMaybe<TString> GetSdkBuildInfo() const { + return {}; + } + + TMaybe<TString> GetGrpcUserAgent() const { + return {}; + } + + TInstant GetDeadline() const override { + return TInstant::Max(); + } + void SetRespHook(TRespHook&&) override { /* do nothing */} - - TRateLimiterMode GetRlMode() const override { + + TRateLimiterMode GetRlMode() const override { return TRateLimiterMode::Off; } bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { - return false; - } - - void Pass(const IFacilityProvider&) override { - Y_FAIL("unimplemented"); - } - -private: - const TString Token_; - const TString Database_; + return false; + } + + void Pass(const IFacilityProvider&) override { + Y_FAIL("unimplemented"); + } + +private: + const TString Token_; + const TString Database_; const TActorId From_; NGrpc::TAuthState State_; - TString InternalToken_; - NYql::TIssueManager IssueManager_; -}; - -inline TMaybe<TString> ToMaybe(const TVector<TStringBuf>& vec) { - if (vec.empty()) { - return {}; - } - return TString{vec[0]}; -} - -template<ui32 TRpcId, typename TReq, typename TResp> -class TGRpcRequestBiStreamWrapper : - public IRequestProxyCtx, - public TEventLocal<TGRpcRequestBiStreamWrapper<TRpcId, TReq, TResp>, TRpcId> { -public: - using TRequest = TReq; - using TResponse = TResp; - using IStreamCtx = NGRpcServer::IGRpcStreamingContext<TRequest, TResponse>; - TGRpcRequestBiStreamWrapper(TIntrusivePtr<IStreamCtx> ctx) - : Ctx_(ctx) - { } - - TRateLimiterMode GetRlMode() const override { - return TRateLimiterMode::Off; - } - + TString InternalToken_; + NYql::TIssueManager IssueManager_; +}; + +inline TMaybe<TString> ToMaybe(const TVector<TStringBuf>& vec) { + if (vec.empty()) { + return {}; + } + return TString{vec[0]}; +} + +template<ui32 TRpcId, typename TReq, typename TResp> +class TGRpcRequestBiStreamWrapper : + public IRequestProxyCtx, + public TEventLocal<TGRpcRequestBiStreamWrapper<TRpcId, TReq, TResp>, TRpcId> { +public: + using TRequest = TReq; + using TResponse = TResp; + using IStreamCtx = NGRpcServer::IGRpcStreamingContext<TRequest, TResponse>; + TGRpcRequestBiStreamWrapper(TIntrusivePtr<IStreamCtx> ctx) + : Ctx_(ctx) + { } + + TRateLimiterMode GetRlMode() const override { + return TRateLimiterMode::Off; + } + bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { - return false; - } - - const TMaybe<TString> GetYdbToken() const override { - const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_AUTH_TICKET_HEADER); - if (res.empty()) { - return {}; - } + return false; + } + + const TMaybe<TString> GetYdbToken() const override { + const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_AUTH_TICKET_HEADER); + if (res.empty()) { + return {}; + } return TString{res[0]}; - } - - const TMaybe<TString> GetDatabaseName() const override { - const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_DATABASE_HEADER); - if (res.empty()) { - return {}; - } - return CGIUnescapeRet(res[0]); - } - + } + + const TMaybe<TString> GetDatabaseName() const override { + const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_DATABASE_HEADER); + if (res.empty()) { + return {}; + } + return CGIUnescapeRet(res[0]); + } + void UpdateAuthState(NGrpc::TAuthState::EAuthState state) override { - auto& s = Ctx_->GetAuthState(); - s.State = state; - } - + auto& s = Ctx_->GetAuthState(); + s.State = state; + } + const NGrpc::TAuthState& GetAuthState() const override { - return Ctx_->GetAuthState(); - } - + return Ctx_->GetAuthState(); + } + void ReplyWithRpcStatus(grpc::StatusCode, const TString&) override { Y_FAIL("Unimplemented"); } - void ReplyUnauthenticated(const TString& in) override { - const TString message = in.empty() ? TString("unauthenticated") : TString("unauthenticated, ") + in; - Ctx_->Finish(grpc::Status(grpc::StatusCode::UNAUTHENTICATED, message)); - } - - void ReplyUnavaliable() override { + void ReplyUnauthenticated(const TString& in) override { + const TString message = in.empty() ? TString("unauthenticated") : TString("unauthenticated, ") + in; + Ctx_->Finish(grpc::Status(grpc::StatusCode::UNAUTHENTICATED, message)); + } + + void ReplyUnavaliable() override { Ctx_->Attach(TActorId()); - TResponse resp; - FillYdbStatus(resp, IssueManager_.GetIssues(), Ydb::StatusIds::UNAVAILABLE); - Ctx_->WriteAndFinish(std::move(resp), grpc::Status()); - } - - void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { + TResponse resp; + FillYdbStatus(resp, IssueManager_.GetIssues(), Ydb::StatusIds::UNAVAILABLE); + Ctx_->WriteAndFinish(std::move(resp), grpc::Status()); + } + + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { Ctx_->Attach(TActorId()); - TResponse resp; - FillYdbStatus(resp, IssueManager_.GetIssues(), status); - Ctx_->WriteAndFinish(std::move(resp), grpc::Status()); - } - - void RaiseIssue(const NYql::TIssue& issue) override { - IssueManager_.RaiseIssue(issue); - } - - void RaiseIssues(const NYql::TIssues& issues) override { - IssueManager_.RaiseIssues(issues); - } - - void SetInternalToken(const TString& token) override { - InternalToken_ = token; - } - - void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) override { - RlPath_ = std::move(path); - } - - TMaybe<NRpcService::TRlPath> GetRlPath() const override { - return RlPath_; - } - - const TString& GetInternalToken() const override { - return InternalToken_; - } - + TResponse resp; + FillYdbStatus(resp, IssueManager_.GetIssues(), status); + Ctx_->WriteAndFinish(std::move(resp), grpc::Status()); + } + + void RaiseIssue(const NYql::TIssue& issue) override { + IssueManager_.RaiseIssue(issue); + } + + void RaiseIssues(const NYql::TIssues& issues) override { + IssueManager_.RaiseIssues(issues); + } + + void SetInternalToken(const TString& token) override { + InternalToken_ = token; + } + + void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) override { + RlPath_ = std::move(path); + } + + TMaybe<NRpcService::TRlPath> GetRlPath() const override { + return RlPath_; + } + + const TString& GetInternalToken() const override { + return InternalToken_; + } + TString GetPeerName() const override { return Ctx_->GetPeerName(); } @@ -642,58 +642,58 @@ public: Ctx_->UseDatabase(database); } - IStreamCtx* GetStreamCtx() { - return Ctx_.Get(); - } - - const TString& GetRequestName() const override { - return TRequest::descriptor()->name(); - } - - TMaybe<TString> GetTraceId() const override { + IStreamCtx* GetStreamCtx() { + return Ctx_.Get(); + } + + const TString& GetRequestName() const override { + return TRequest::descriptor()->name(); + } + + TMaybe<TString> GetTraceId() const override { return GetPeerMetaValues(NYdb::YDB_TRACE_ID_HEADER); - } - - const TMaybe<TString> GetSdkBuildInfo() const { + } + + const TMaybe<TString> GetSdkBuildInfo() const { return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); - } - + } + const TMaybe<TString> GetRequestType() const { return GetPeerMetaValues(NYdb::YDB_REQUEST_TYPE_HEADER); } - TInstant GetDeadline() const override { - return TInstant::Max(); - } - - const TMaybe<TString> GetGrpcUserAgent() const { + TInstant GetDeadline() const override { + return TInstant::Max(); + } + + const TMaybe<TString> GetGrpcUserAgent() const { return GetPeerMetaValues(NGrpc::GRPC_USER_AGENT_HEADER); - } - + } + const TMaybe<TString> GetPeerMetaValues(const TString& key) const override { return ToMaybe(Ctx_->GetPeerMetaValues(key)); } void RefreshToken(const TString& token, const TActorContext& ctx, TActorId id) { - NGRpcService::RefreshToken(token, GetDatabaseName().GetOrElse(""), ctx, id); - } - + NGRpcService::RefreshToken(token, GetDatabaseName().GetOrElse(""), ctx, id); + } + void SetRespHook(TRespHook&&) override { - /* cannot add hook to bidirect streaming */ - Y_FAIL("Unimplemented"); - } - - void Pass(const IFacilityProvider&) override { - Y_FAIL("unimplemented"); - } - -private: - TIntrusivePtr<IStreamCtx> Ctx_; - TString InternalToken_; - NYql::TIssueManager IssueManager_; - TMaybe<NRpcService::TRlPath> RlPath_; -}; - + /* cannot add hook to bidirect streaming */ + Y_FAIL("Unimplemented"); + } + + void Pass(const IFacilityProvider&) override { + Y_FAIL("unimplemented"); + } + +private: + TIntrusivePtr<IStreamCtx> Ctx_; + TString InternalToken_; + NYql::TIssueManager IssueManager_; + TMaybe<NRpcService::TRlPath> RlPath_; +}; + template <typename TDerived> class TGrpcResponseSenderImpl : public IRequestOpCtx { public: @@ -756,105 +756,105 @@ private: TDerived* Derived() noexcept { return static_cast<TDerived*>(this); } }; -class TEvProxyRuntimeEvent - : public IRequestProxyCtx - , public TEventLocal<TEvProxyRuntimeEvent, TRpcServices::EvGrpcRuntimeRequest> -{ -public: - const TMaybe<TString> GetSdkBuildInfo() const { - return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); - } - - const TMaybe<TString> GetGrpcUserAgent() const { - return GetPeerMetaValues(NGrpc::GRPC_USER_AGENT_HEADER); - } -}; - -template<ui32 TRpcId, typename TDerived> -class TEvProxyLegacyEvent - : public IRequestProxyCtx - , public TEventLocal<TDerived, TRpcId> -{ -public: - const TMaybe<TString> GetSdkBuildInfo() const { - return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); - } - - const TMaybe<TString> GetGrpcUserAgent() const { - return GetPeerMetaValues(NGrpc::GRPC_USER_AGENT_HEADER); - } -}; - -template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, typename TDerived> +class TEvProxyRuntimeEvent + : public IRequestProxyCtx + , public TEventLocal<TEvProxyRuntimeEvent, TRpcServices::EvGrpcRuntimeRequest> +{ +public: + const TMaybe<TString> GetSdkBuildInfo() const { + return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); + } + + const TMaybe<TString> GetGrpcUserAgent() const { + return GetPeerMetaValues(NGrpc::GRPC_USER_AGENT_HEADER); + } +}; + +template<ui32 TRpcId, typename TDerived> +class TEvProxyLegacyEvent + : public IRequestProxyCtx + , public TEventLocal<TDerived, TRpcId> +{ +public: + const TMaybe<TString> GetSdkBuildInfo() const { + return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); + } + + const TMaybe<TString> GetGrpcUserAgent() const { + return GetPeerMetaValues(NGrpc::GRPC_USER_AGENT_HEADER); + } +}; + +template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, typename TDerived> class TGRpcRequestWrapperImpl : public std::conditional_t<IsOperation, TGrpcResponseSenderImpl<TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TDerived>>, IRequestNoOpCtx>, - public std::conditional_t<TRpcId == TRpcServices::EvGrpcRuntimeRequest, - TEvProxyRuntimeEvent, - TEvProxyLegacyEvent<TRpcId, TDerived>> -{ - friend class TProtoResponseHelper; + public std::conditional_t<TRpcId == TRpcServices::EvGrpcRuntimeRequest, + TEvProxyRuntimeEvent, + TEvProxyLegacyEvent<TRpcId, TDerived>> +{ + friend class TProtoResponseHelper; friend class TGrpcResponseSenderImpl<TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TDerived>>; -public: - using TRequest = TReq; - using TResponse = TResp; - +public: + using TRequest = TReq; + using TResponse = TResp; + TGRpcRequestWrapperImpl(NGrpc::IRequestContextBase* ctx) - : Ctx_(ctx) - { } - - const TMaybe<TString> GetYdbToken() const override { - const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_AUTH_TICKET_HEADER); - if (res.empty()) { - return {}; - } + : Ctx_(ctx) + { } + + const TMaybe<TString> GetYdbToken() const override { + const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_AUTH_TICKET_HEADER); + if (res.empty()) { + return {}; + } return TString{res[0]}; - } - - const TMaybe<TString> GetDatabaseName() const override { - const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_DATABASE_HEADER); - if (res.empty()) { - return {}; - } - return CGIUnescapeRet(res[0]); - } - + } + + const TMaybe<TString> GetDatabaseName() const override { + const auto& res = Ctx_->GetPeerMetaValues(NYdb::YDB_DATABASE_HEADER); + if (res.empty()) { + return {}; + } + return CGIUnescapeRet(res[0]); + } + void UpdateAuthState(NGrpc::TAuthState::EAuthState state) override { - auto& s = Ctx_->GetAuthState(); - s.State = state; - } - + auto& s = Ctx_->GetAuthState(); + s.State = state; + } + const NGrpc::TAuthState& GetAuthState() const override { - return Ctx_->GetAuthState(); - } - + return Ctx_->GetAuthState(); + } + void ReplyWithRpcStatus(grpc::StatusCode code, const TString& reason) override { Ctx_->ReplyError(code, reason); } - void ReplyUnauthenticated(const TString& in) override { - Ctx_->ReplyUnauthenticated(in); - } - - void SetInternalToken(const TString& token) override { - InternalToken_ = token; - } - + void ReplyUnauthenticated(const TString& in) override { + Ctx_->ReplyUnauthenticated(in); + } + + void SetInternalToken(const TString& token) override { + InternalToken_ = token; + } + void AddServerHint(const TString& hint) override { Ctx_->AddTrailingMetadata(NYdb::YDB_SERVER_HINTS, hint); } - void SetRuHeader(ui64 ru) override { - Ru = ru; - auto ruStr = Sprintf("%lu", ru); - Ctx_->AddTrailingMetadata(NYdb::YDB_CONSUMED_UNITS_HEADER, ruStr); - } - - const TString& GetInternalToken() const override { - return InternalToken_; - } - + void SetRuHeader(ui64 ru) override { + Ru = ru; + auto ruStr = Sprintf("%lu", ru); + Ctx_->AddTrailingMetadata(NYdb::YDB_CONSUMED_UNITS_HEADER, ruStr); + } + + const TString& GetInternalToken() const override { + return InternalToken_; + } + const TMaybe<TString> GetPeerMetaValues(const TString& key) const override { return ToMaybe(Ctx_->GetPeerMetaValues(key)); } @@ -867,18 +867,18 @@ public: Ctx_->UseDatabase(database); } - void ReplyUnavaliable() override { - TResponse* resp = CreateResponseMessage(); - TCommonResponseFiller<TResp, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, Ydb::StatusIds::UNAVAILABLE); - Reply(resp, Ydb::StatusIds::UNAVAILABLE); + void ReplyUnavaliable() override { + TResponse* resp = CreateResponseMessage(); + TCommonResponseFiller<TResp, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, Ydb::StatusIds::UNAVAILABLE); + Reply(resp, Ydb::StatusIds::UNAVAILABLE); + } + + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { + TResponse* resp = CreateResponseMessage(); + TCommonResponseFiller<TResponse, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, status); + Reply(resp, status); } - void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { - TResponse* resp = CreateResponseMessage(); - TCommonResponseFiller<TResponse, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, status); - Reply(resp, status); - } - TString GetPeerName() const override { return Ctx_->GetPeer(); } @@ -887,211 +887,211 @@ public: return Ctx_->SslServer(); } - template<typename T> - static const TRequest* GetProtoRequest(const T& req) { - auto request = dynamic_cast<const TRequest*>(req->GetRequest()); - Y_VERIFY(request != nullptr, "Wrong using of TGRpcRequestWrapper"); - return request; - } - - const TRequest* GetProtoRequest() const { - return GetProtoRequest(this); - } - + template<typename T> + static const TRequest* GetProtoRequest(const T& req) { + auto request = dynamic_cast<const TRequest*>(req->GetRequest()); + Y_VERIFY(request != nullptr, "Wrong using of TGRpcRequestWrapper"); + return request; + } + + const TRequest* GetProtoRequest() const { + return GetProtoRequest(this); + } + TMaybe<TString> GetTraceId() const override { return GetPeerMetaValues(NYdb::YDB_TRACE_ID_HEADER); - } - - const TMaybe<TString> GetSdkBuildInfo() const { + } + + const TMaybe<TString> GetSdkBuildInfo() const { return GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); - } - - TInstant GetDeadline() const override { - return Ctx_->Deadline(); - } - + } + + TInstant GetDeadline() const override { + return Ctx_->Deadline(); + } + const TMaybe<TString> GetRequestType() const override { return GetPeerMetaValues(NYdb::YDB_REQUEST_TYPE_HEADER); } void SendSerializedResult(TString&& in, Ydb::StatusIds::StatusCode status) { - // res->data() pointer is used inside grpc code. - // So this object should be destroyed during grpc_slice destroying routine - auto res = new TString; - res->swap(in); - - static auto freeResult = [](void* p) -> void { - TString* toDelete = reinterpret_cast<TString*>(p); - delete toDelete; - }; - - grpc_slice slice = grpc_slice_new_with_user_data( - (void*)(res->data()), res->size(), freeResult, res); - grpc::Slice sl = grpc::Slice(slice, grpc::Slice::STEAL_REF); - auto data = grpc::ByteBuffer(&sl, 1); + // res->data() pointer is used inside grpc code. + // So this object should be destroyed during grpc_slice destroying routine + auto res = new TString; + res->swap(in); + + static auto freeResult = [](void* p) -> void { + TString* toDelete = reinterpret_cast<TString*>(p); + delete toDelete; + }; + + grpc_slice slice = grpc_slice_new_with_user_data( + (void*)(res->data()), res->size(), freeResult, res); + grpc::Slice sl = grpc::Slice(slice, grpc::Slice::STEAL_REF); + auto data = grpc::ByteBuffer(&sl, 1); Ctx_->Reply(&data, status); - } - + } + void SetCostInfo(float consumed_units) override { - CostInfo = google::protobuf::Arena::CreateMessage<Ydb::CostInfo>(GetArena()); - CostInfo->set_consumed_units(consumed_units); - } - - const TString& GetRequestName() const override { - return TRequest::descriptor()->name(); - } - - google::protobuf::Arena* GetArena() override { - return Ctx_->GetArena(); - } - - //! Allocate Result message using protobuf arena allocator - //! The memory will be freed automaticaly after destroying - //! corresponding request. - //! Do not call delete for objects allocated here! - template<typename TResult, typename T> - static TResult* AllocateResult(T& ctx) { - return google::protobuf::Arena::CreateMessage<TResult>(ctx->GetArena()); - } - - template<typename Tcb> - void SetStreamingNotify(Tcb&& cb) { - Ctx_->SetNextReplyCallback(cb); - } - - void SetClientLostAction(std::function<void()>&& cb) override { + CostInfo = google::protobuf::Arena::CreateMessage<Ydb::CostInfo>(GetArena()); + CostInfo->set_consumed_units(consumed_units); + } + + const TString& GetRequestName() const override { + return TRequest::descriptor()->name(); + } + + google::protobuf::Arena* GetArena() override { + return Ctx_->GetArena(); + } + + //! Allocate Result message using protobuf arena allocator + //! The memory will be freed automaticaly after destroying + //! corresponding request. + //! Do not call delete for objects allocated here! + template<typename TResult, typename T> + static TResult* AllocateResult(T& ctx) { + return google::protobuf::Arena::CreateMessage<TResult>(ctx->GetArena()); + } + + template<typename Tcb> + void SetStreamingNotify(Tcb&& cb) { + Ctx_->SetNextReplyCallback(cb); + } + + void SetClientLostAction(std::function<void()>&& cb) override { auto shutdown = [cb{move(cb)}](const NGrpc::IRequestContextBase::TAsyncFinishResult& future) mutable { - Y_ASSERT(future.HasValue()); + Y_ASSERT(future.HasValue()); if (future.GetValue() == NGrpc::IRequestContextBase::EFinishStatus::CANCEL) { - cb(); - } - }; - Ctx_->GetFinishFuture().Subscribe(std::move(shutdown)); - } - - void FinishStream() { - Ctx_->FinishStreamingOk(); - } - - void RaiseIssue(const NYql::TIssue& issue) override { - IssueManager.RaiseIssue(issue); - } - - void RaiseIssues(const NYql::TIssues& issues) override { - IssueManager.RaiseIssues(issues); - } - - const google::protobuf::Message* GetRequest() const override { - return Ctx_->GetRequest(); - } - - void SetRespHook(TRespHook&& hook) override { - RespHook = std::move(hook); - } - - void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) override { - RlPath = std::move(path); - } - - TMaybe<NRpcService::TRlPath> GetRlPath() const override { - return RlPath; - } - - void Pass(const IFacilityProvider&) override { - Y_FAIL("unimplemented"); - } - -private: - void Reply(NProtoBuf::Message *resp, ui32 status) override { - if (RespHook) { - TRespHook hook = std::move(RespHook); - return hook(MakeIntrusive<TRespHookCtx>(Ctx_, resp, GetRequestName(), Ru, status)); - } - return Ctx_->Reply(resp, status); - } - -private: - TResponse* CreateResponseMessage() { - return google::protobuf::Arena::CreateMessage<TResponse>(Ctx_->GetArena()); - } - + cb(); + } + }; + Ctx_->GetFinishFuture().Subscribe(std::move(shutdown)); + } + + void FinishStream() { + Ctx_->FinishStreamingOk(); + } + + void RaiseIssue(const NYql::TIssue& issue) override { + IssueManager.RaiseIssue(issue); + } + + void RaiseIssues(const NYql::TIssues& issues) override { + IssueManager.RaiseIssues(issues); + } + + const google::protobuf::Message* GetRequest() const override { + return Ctx_->GetRequest(); + } + + void SetRespHook(TRespHook&& hook) override { + RespHook = std::move(hook); + } + + void SetRlPath(TMaybe<NRpcService::TRlPath>&& path) override { + RlPath = std::move(path); + } + + TMaybe<NRpcService::TRlPath> GetRlPath() const override { + return RlPath; + } + + void Pass(const IFacilityProvider&) override { + Y_FAIL("unimplemented"); + } + +private: + void Reply(NProtoBuf::Message *resp, ui32 status) override { + if (RespHook) { + TRespHook hook = std::move(RespHook); + return hook(MakeIntrusive<TRespHookCtx>(Ctx_, resp, GetRequestName(), Ru, status)); + } + return Ctx_->Reply(resp, status); + } + +private: + TResponse* CreateResponseMessage() { + return google::protobuf::Arena::CreateMessage<TResponse>(Ctx_->GetArena()); + } + TIntrusivePtr<NGrpc::IRequestContextBase> Ctx_; - TString InternalToken_; - NYql::TIssueManager IssueManager; - Ydb::CostInfo* CostInfo = nullptr; - ui64 Ru = 0; - TRespHook RespHook; - TMaybe<NRpcService::TRlPath> RlPath; -}; - -class IFacilityProvider; - -template <typename TReq, typename TResp, bool IsOperation> -class TGrpcRequestCall - : public TGRpcRequestWrapperImpl< - TRpcServices::EvGrpcRuntimeRequest, TReq, TResp, IsOperation, TGrpcRequestCall<TReq, TResp, IsOperation>> { - typedef typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type TRequestIface; -public: - static constexpr bool IsOp = IsOperation; - TGrpcRequestCall(NGrpc::IRequestContextBase* ctx, - void (*cb)(std::unique_ptr<TRequestIface>, const IFacilityProvider&), TRequestAuxSettings auxSettings = {}) - : TGRpcRequestWrapperImpl< - TRpcServices::EvGrpcRuntimeRequest, TReq, TResp, IsOperation, - TGrpcRequestCall<TReq, TResp, IsOperation>>(ctx) - , PassMethod(cb) - , AuxSettings(std::move(auxSettings)) - { } - - void Pass(const IFacilityProvider& facility) override { - PassMethod(std::move(std::unique_ptr<TRequestIface>(this)), facility); - } - - TRateLimiterMode GetRlMode() const override { - return AuxSettings.RlMode; - } - + TString InternalToken_; + NYql::TIssueManager IssueManager; + Ydb::CostInfo* CostInfo = nullptr; + ui64 Ru = 0; + TRespHook RespHook; + TMaybe<NRpcService::TRlPath> RlPath; +}; + +class IFacilityProvider; + +template <typename TReq, typename TResp, bool IsOperation> +class TGrpcRequestCall + : public TGRpcRequestWrapperImpl< + TRpcServices::EvGrpcRuntimeRequest, TReq, TResp, IsOperation, TGrpcRequestCall<TReq, TResp, IsOperation>> { + typedef typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type TRequestIface; +public: + static constexpr bool IsOp = IsOperation; + TGrpcRequestCall(NGrpc::IRequestContextBase* ctx, + void (*cb)(std::unique_ptr<TRequestIface>, const IFacilityProvider&), TRequestAuxSettings auxSettings = {}) + : TGRpcRequestWrapperImpl< + TRpcServices::EvGrpcRuntimeRequest, TReq, TResp, IsOperation, + TGrpcRequestCall<TReq, TResp, IsOperation>>(ctx) + , PassMethod(cb) + , AuxSettings(std::move(auxSettings)) + { } + + void Pass(const IFacilityProvider& facility) override { + PassMethod(std::move(std::unique_ptr<TRequestIface>(this)), facility); + } + + TRateLimiterMode GetRlMode() const override { + return AuxSettings.RlMode; + } + bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData, - ICheckerIface* iface) override - { - if (!AuxSettings.CustomAttributeProcessor) { - return false; - } else { - AuxSettings.CustomAttributeProcessor(schemeData, iface); - return true; - } - } -private: - void (*PassMethod)(std::unique_ptr<TRequestIface>, const IFacilityProvider&); - const TRequestAuxSettings AuxSettings; -}; - -template <typename TReq, typename TResp> -using TGrpcRequestOperationCall = TGrpcRequestCall<TReq, TResp, true>; - -template <typename TReq, typename TResp> -using TGrpcRequestNoOperationCall = TGrpcRequestCall<TReq, TResp, false>; - -template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> + ICheckerIface* iface) override + { + if (!AuxSettings.CustomAttributeProcessor) { + return false; + } else { + AuxSettings.CustomAttributeProcessor(schemeData, iface); + return true; + } + } +private: + void (*PassMethod)(std::unique_ptr<TRequestIface>, const IFacilityProvider&); + const TRequestAuxSettings AuxSettings; +}; + +template <typename TReq, typename TResp> +using TGrpcRequestOperationCall = TGrpcRequestCall<TReq, TResp, true>; + +template <typename TReq, typename TResp> +using TGrpcRequestNoOperationCall = TGrpcRequestCall<TReq, TResp, false>; + +template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> class TGRpcRequestWrapper : - public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, - TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { + public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, + TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { public: - static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); - static constexpr bool IsOp = IsOperation; - static constexpr TRateLimiterMode RateLimitMode = RlMode; - + static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); + static constexpr bool IsOp = IsOperation; + static constexpr TRateLimiterMode RateLimitMode = RlMode; + TGRpcRequestWrapper(NGrpc::IRequestContextBase* ctx) - : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, - TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) + : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, + TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) { } - - TRateLimiterMode GetRlMode() const override { - return RateLimitMode; - } - + + TRateLimiterMode GetRlMode() const override { + return RateLimitMode; + } + bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { - return false; - } + return false; + } }; template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation = true, TRateLimiterMode RlMode = TRateLimiterMode::Off> @@ -1106,44 +1106,44 @@ public: : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestWrapperNoAuth<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) { } - TRateLimiterMode GetRlMode() const override { + TRateLimiterMode GetRlMode() const override { return RateLimitMode; } bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { - return false; - } - + return false; + } + const NGrpc::TAuthState& GetAuthState() const override { static NGrpc::TAuthState noAuthState(false); return noAuthState; } }; -template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> +template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> class TGRpcRequestValidationWrapper : - public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { + public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { public: - static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); - static constexpr bool IsOp = IsOperation; - static constexpr TRateLimiterMode RateLimitMode = RlMode; + static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); + static constexpr bool IsOp = IsOperation; + static constexpr TRateLimiterMode RateLimitMode = RlMode; TGRpcRequestValidationWrapper(NGrpc::IRequestContextBase* ctx) - : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) + : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) { } - TRateLimiterMode GetRlMode() const override { - return RateLimitMode; - } - + TRateLimiterMode GetRlMode() const override { + return RateLimitMode; + } + bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { - return false; - } - + return false; + } + bool Validate(TString& error) override { return this->GetProtoRequest()->validate(error); } }; -} // namespace NGRpcService -} // namespace NKikimr +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_helper.cpp b/ydb/core/grpc_services/grpc_helper.cpp index 2d90998beb..b1ef484cc7 100644 --- a/ydb/core/grpc_services/grpc_helper.cpp +++ b/ydb/core/grpc_services/grpc_helper.cpp @@ -1,8 +1,8 @@ -#include "grpc_helper.h" - -namespace NKikimr { -namespace NGRpcService { - +#include "grpc_helper.h" + +namespace NKikimr { +namespace NGRpcService { + //using namespace NActors; NGrpc::IGRpcRequestLimiterPtr TCreateLimiterCB::operator()(const char* serviceName, const char* requestName, i64 limit) const { @@ -41,5 +41,5 @@ NGrpc::IGRpcRequestLimiterPtr TInFlightLimiterRegistry::RegisterRequestType(TStr return PerTypeLimiters[name]; } -} // namespace NGRpcService -} // namespace NKikimr +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_helper.h b/ydb/core/grpc_services/grpc_helper.h index 5277101a43..9a2a673c7b 100644 --- a/ydb/core/grpc_services/grpc_helper.h +++ b/ydb/core/grpc_services/grpc_helper.h @@ -1,15 +1,15 @@ -#pragma once +#pragma once #include "defs.h" #include "grpc_mon.h" - + #include <ydb/core/control/immediate_control_board_impl.h> #include <ydb/core/grpc_services/counters/counters.h> #include <library/cpp/grpc/server/grpc_request.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + class TInFlightLimiterRegistry : public TThrRefBase { private: TIntrusivePtr<NKikimr::TControlBoard> Icb; @@ -42,6 +42,6 @@ inline TCreateLimiterCB CreateLimiterCb(TIntrusivePtr<TInFlightLimiterRegistry> template <typename TIn, typename TOut, typename TService, typename TInProtoPrinter=google::protobuf::TextFormat::Printer, typename TOutProtoPrinter=google::protobuf::TextFormat::Printer> using TGRpcRequest = NGrpc::TGRpcRequest<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter>; - -} // namespace NGRpcService -} // namespace NKikimr + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_mon.cpp b/ydb/core/grpc_services/grpc_mon.cpp index 366678be80..cd15157b05 100644 --- a/ydb/core/grpc_services/grpc_mon.cpp +++ b/ydb/core/grpc_services/grpc_mon.cpp @@ -20,17 +20,17 @@ struct TEvGrpcMon { static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_MON), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_MON)"); struct TEvReportPeer: public TEventLocal<TEvReportPeer, EvReportPeer> { - explicit TEvReportPeer(const TString& name) + explicit TEvReportPeer(const TString& name) : Name(name) {} - TEvReportPeer(const TString& name, const TString& buildInfo) - : Name(name) - , SdkBuildInfo(buildInfo) - {} - - const TString Name; - const TString SdkBuildInfo; + TEvReportPeer(const TString& name, const TString& buildInfo) + : Name(name) + , SdkBuildInfo(buildInfo) + {} + + const TString Name; + const TString SdkBuildInfo; }; }; @@ -62,7 +62,7 @@ private: if (it == Peers.End()) { TPeerInfo val; val.LastRequest = now; - val.SdkBuildInfo = info.SdkBuildInfo; + val.SdkBuildInfo = info.SdkBuildInfo; Peers.Insert(info.Name, val); } else { it->LastRequest = now; @@ -77,7 +77,7 @@ private: TABLER() { TABLEH() { str << "Address";} TABLEH() { str << "Last Request";} - TABLEH() { str << "Sdk BuildInfo";} + TABLEH() { str << "Sdk BuildInfo";} } } TABLEBODY() { @@ -85,7 +85,7 @@ private: TABLER() { TABLED() { str << p.Key(); } TABLED() { str << p.Value().LastRequest.ToRfc822StringLocal(); } - TABLED() { str << p.Value().SdkBuildInfo; } + TABLED() { str << p.Value().SdkBuildInfo; } } } } @@ -98,7 +98,7 @@ private: private: struct TPeerInfo { TInstant LastRequest; - TString SdkBuildInfo; + TString SdkBuildInfo; }; TLRUCache<TString, TPeerInfo> Peers; @@ -113,12 +113,12 @@ IActor* CreateGrpcMonService() { return new TGrpcMon; } -void ReportGrpcReqToMon(NActors::TActorSystem& actorSystem, const TString& fromAddress) { +void ReportGrpcReqToMon(NActors::TActorSystem& actorSystem, const TString& fromAddress) { actorSystem.Send(GrpcMonServiceId(), new TEvGrpcMon::TEvReportPeer(fromAddress)); } -void ReportGrpcReqToMon(NActors::TActorSystem& actorSystem, const TString& fromAddress, const TString& buildInfo) { - actorSystem.Send(GrpcMonServiceId(), new TEvGrpcMon::TEvReportPeer(fromAddress, buildInfo)); -} - +void ReportGrpcReqToMon(NActors::TActorSystem& actorSystem, const TString& fromAddress, const TString& buildInfo) { + actorSystem.Send(GrpcMonServiceId(), new TEvGrpcMon::TEvReportPeer(fromAddress, buildInfo)); +} + }} diff --git a/ydb/core/grpc_services/grpc_mon.h b/ydb/core/grpc_services/grpc_mon.h index 2a41bf1356..f599676977 100644 --- a/ydb/core/grpc_services/grpc_mon.h +++ b/ydb/core/grpc_services/grpc_mon.h @@ -10,7 +10,7 @@ namespace NGRpcService { TActorId GrpcMonServiceId(); IActor* CreateGrpcMonService(); -void ReportGrpcReqToMon(NActors::TActorSystem&, const TString& fromAddress); -void ReportGrpcReqToMon(NActors::TActorSystem&, const TString& fromAddress, const TString& buildInfo); +void ReportGrpcReqToMon(NActors::TActorSystem&, const TString& fromAddress); +void ReportGrpcReqToMon(NActors::TActorSystem&, const TString& fromAddress, const TString& buildInfo); }} diff --git a/ydb/core/grpc_services/grpc_request_check_actor.h b/ydb/core/grpc_services/grpc_request_check_actor.h index ba78026cf1..04c72abe6d 100644 --- a/ydb/core/grpc_services/grpc_request_check_actor.h +++ b/ydb/core/grpc_services/grpc_request_check_actor.h @@ -19,10 +19,10 @@ namespace NKikimr { namespace NGRpcService { template <typename TEvent> -class TGrpcRequestCheckActor - : public TActorBootstrappedSecureRequest<TGrpcRequestCheckActor<TEvent>> - , public ICheckerIface -{ +class TGrpcRequestCheckActor + : public TActorBootstrappedSecureRequest<TGrpcRequestCheckActor<TEvent>> + , public ICheckerIface +{ using TSelf = TGrpcRequestCheckActor<TEvent>; using TBase = TActorBootstrappedSecureRequest<TGrpcRequestCheckActor>; public: @@ -46,27 +46,27 @@ public: CheckedDatabaseName_ = CanonizePath(schemeData.GetPath()); if (!GrpcRequestBaseCtx_->TryCustomAttributeProcess(schemeData, this)) { ProcessCommonAttributes(schemeData); - } - } - + } + } + void ProcessCommonAttributes(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData) { - static std::vector<TString> allowedAttributes = {"folder_id", "service_account_id", "database_id"}; - TVector<std::pair<TString, TString>> attributes; + static std::vector<TString> allowedAttributes = {"folder_id", "service_account_id", "database_id"}; + TVector<std::pair<TString, TString>> attributes; attributes.reserve(schemeData.GetPathDescription().UserAttributesSize()); for (const auto& attr : schemeData.GetPathDescription().GetUserAttributes()) { if (std::find(allowedAttributes.begin(), allowedAttributes.end(), attr.GetKey()) != allowedAttributes.end()) { attributes.emplace_back(attr.GetKey(), attr.GetValue()); } } - if (!attributes.empty()) { - SetEntries({{GetPermissions(), attributes}}); - } + if (!attributes.empty()) { + SetEntries({{GetPermissions(), attributes}}); + } + } + + void SetEntries(const TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry>& entries) { + TBase::SetEntries(entries); } - void SetEntries(const TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry>& entries) { - TBase::SetEntries(entries); - } - void InitializeAttributes(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData); TGrpcRequestCheckActor( @@ -104,7 +104,7 @@ public: } } - // Simple rps limitation + // Simple rps limitation static NRpcService::TRlConfig rpsRlConfig( "serverless_rt_coordination_node_path", "serverless_rt_base_resource_rps", @@ -115,7 +115,7 @@ public: } ); - // Limitation RU for unary calls in time of response + // Limitation RU for unary calls in time of response static NRpcService::TRlConfig ruRlConfig( "serverless_rt_coordination_node_path", "serverless_rt_base_resource_ru", @@ -128,18 +128,18 @@ public: } ); - // Limitation ru for calls with internall rl support (read table) - static NRpcService::TRlConfig ruRlProgressConfig( - "serverless_rt_coordination_node_path", - "serverless_rt_base_resource_ru", - { - NRpcService::TRlConfig::TOnReqAction { - 1 - } - } - ); - - + // Limitation ru for calls with internall rl support (read table) + static NRpcService::TRlConfig ruRlProgressConfig( + "serverless_rt_coordination_node_path", + "serverless_rt_base_resource_ru", + { + NRpcService::TRlConfig::TOnReqAction { + 1 + } + } + ); + + auto rlMode = Request_->Get()->GetRlMode(); switch (rlMode) { case TRateLimiterMode::Rps: @@ -148,20 +148,20 @@ public: case TRateLimiterMode::Ru: RlConfig = &ruRlConfig; break; - case TRateLimiterMode::RuOnProgress: - RlConfig = &ruRlProgressConfig; + case TRateLimiterMode::RuOnProgress: + RlConfig = &ruRlProgressConfig; break; case TRateLimiterMode::Off: break; - } + } if (!RlConfig) { // No rate limit config for this request return SetTokenAndDie(CheckedDatabaseName_); } else { THashMap<TString, TString> attributes; - for (const auto& [attrName, attrValue] : Attributes_) { - attributes[attrName] = attrValue; + for (const auto& [attrName, attrValue] : Attributes_) { + attributes[attrName] = attrValue; } return ProcessRateLimit(attributes, ctx); } @@ -174,10 +174,10 @@ public: ReplyBackAndDie(); } - void SetRlPath(TMaybe<NRpcService::TRlPath>&& rlPath) { - GrpcRequestBaseCtx_->SetRlPath(std::move(rlPath)); - } - + void SetRlPath(TMaybe<NRpcService::TRlPath>&& rlPath) { + GrpcRequestBaseCtx_->SetRlPath(std::move(rlPath)); + } + STATEFN(DbAccessStateFunc) { switch (ev->GetTypeRewrite()) { hFunc(TEvents::TEvPoisonPill, HandlePoison); @@ -196,17 +196,17 @@ private: } void ProcessOnRequest(Ydb::RateLimiter::AcquireResourceRequest&& req, const TActorContext& ctx) { - auto time = TInstant::Now(); - auto cb = [this, time](Ydb::RateLimiter::AcquireResourceResponse resp) { - TDuration delay = TInstant::Now() - time; + auto time = TInstant::Now(); + auto cb = [this, time](Ydb::RateLimiter::AcquireResourceResponse resp) { + TDuration delay = TInstant::Now() - time; switch (resp.operation().status()) { case Ydb::StatusIds::SUCCESS: - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Request delayed for " << delay << " by ratelimiter"); + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Request delayed for " << delay << " by ratelimiter"); SetTokenAndDie(CheckedDatabaseName_); break; case Ydb::StatusIds::TIMEOUT: Counters_->IncDatabaseRateLimitedCounter(); - LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Throughput limit exceeded"); + LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Throughput limit exceeded"); ReplyOverloadedAndDie(MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, "Throughput limit exceeded")); break; default: @@ -264,13 +264,13 @@ private: void ProcessRateLimit(const THashMap<TString, TString>& attributes, const TActorContext& ctx) { // Match rate limit config and database attributes - auto rlPath = NRpcService::Match(*RlConfig, attributes); - if (!rlPath) { + auto rlPath = NRpcService::Match(*RlConfig, attributes); + if (!rlPath) { return SetTokenAndDie(CheckedDatabaseName_); } else { - auto actions = NRpcService::MakeRequests(*RlConfig, rlPath.GetRef()); - SetRlPath(std::move(rlPath)); - + auto actions = NRpcService::MakeRequests(*RlConfig, rlPath.GetRef()); + SetRlPath(std::move(rlPath)); + Ydb::RateLimiter::AcquireResourceRequest req; bool hasOnReqAction = false; for (auto& action : actions) { @@ -397,15 +397,15 @@ private: IRequestProxyCtx* GrpcRequestBaseCtx_; NRpcService::TRlConfig* RlConfig = nullptr; bool SkipCheckConnectRigths_ = false; - std::vector<std::pair<TString, TString>> Attributes_; + std::vector<std::pair<TString, TString>> Attributes_; }; // default behavior - attributes in schema template <typename TEvent> void TGrpcRequestCheckActor<TEvent>::InitializeAttributes(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData) { - for (const auto& attr : schemeData.GetPathDescription().GetUserAttributes()) { - Attributes_.emplace_back(std::make_pair(attr.GetKey(), attr.GetValue())); - } + for (const auto& attr : schemeData.GetPathDescription().GetUserAttributes()) { + Attributes_.emplace_back(std::make_pair(attr.GetKey(), attr.GetValue())); + } InitializeAttributesFromSchema(schemeData); } @@ -443,9 +443,9 @@ IActor* CreateGrpcRequestCheckActor( TAutoPtr<TEventHandle<TEvent>> request, TGrpcProxyCounters::TPtr counters, bool skipCheckConnectRigths) { - + return new TGrpcRequestCheckActor<TEvent>(owner, schemeData, std::move(securityObject), std::move(request), counters, skipCheckConnectRigths); } } -} +} diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp index 5dcc084734..a1e7accb87 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.cpp +++ b/ydb/core/grpc_services/grpc_request_proxy.cpp @@ -1,9 +1,9 @@ #include "grpc_proxy_counters.h" #include "grpc_request_check_actor.h" -#include "grpc_request_proxy.h" -#include "local_rate_limiter.h" -#include "operation_helpers.h" - +#include "grpc_request_proxy.h" +#include "local_rate_limiter.h" +#include "operation_helpers.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> @@ -11,37 +11,37 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/scheme_board/scheme_board.h> -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; - +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; + static const ui32 MAX_DEFERRED_EVENTS_PER_DATABASE = 100; - + TString DatabaseFromDomain(const TAppData* appdata = AppData()) { - auto dinfo = appdata->DomainsInfo; - if (!dinfo) - ythrow yexception() << "Invalid DomainsInfo ptr"; - if (dinfo->Domains.empty() || dinfo->Domains.size() != 1) - ythrow yexception() << "Unexpected domains container size: " - << dinfo->Domains.size(); - if (!dinfo->Domains.begin()->second) - ythrow yexception() << "Empty domain params"; - - return TString("/") + dinfo->Domains.begin()->second->Name; -} - -struct TDatabaseInfo { - enum class TDatabaseType { + auto dinfo = appdata->DomainsInfo; + if (!dinfo) + ythrow yexception() << "Invalid DomainsInfo ptr"; + if (dinfo->Domains.empty() || dinfo->Domains.size() != 1) + ythrow yexception() << "Unexpected domains container size: " + << dinfo->Domains.size(); + if (!dinfo->Domains.begin()->second) + ythrow yexception() << "Empty domain params"; + + return TString("/") + dinfo->Domains.begin()->second->Name; +} + +struct TDatabaseInfo { + enum class TDatabaseType { Root, - Tenant, - Serverless - }; + Tenant, + Serverless + }; NKikimrTenantPool::EState State; TDatabaseType DatabaseType; THolder<TSchemeBoardEvents::TEvNotifyUpdate> SchemeBoardResult; TIntrusivePtr<TSecurityObject> SecurityObject; - + bool IsDatabaseReady() const { if (SchemeBoardResult == nullptr) { return false; @@ -59,10 +59,10 @@ struct TDatabaseInfo { } } return true; - } + } }; - - + + struct TEventReqHolder { TEventReqHolder(TAutoPtr<IEventHandle> ev, IRequestProxyCtx* ctx) : Ev(std::move(ev)) @@ -77,7 +77,7 @@ class TGRpcRequestProxyImpl , public TGRpcRequestProxy { using TBase = TActorBootstrapped<TGRpcRequestProxyImpl>; -public: +public: explicit TGRpcRequestProxyImpl(const NKikimrConfig::TAppConfig& appConfig) : AppConfig(appConfig) {} @@ -99,9 +99,9 @@ private: void HandleSchemeBoard(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev, const TActorContext& ctx); void HandleSchemeBoard(TSchemeBoardEvents::TEvNotifyDelete::TPtr& ev); void ReplayEvents(const TString& databaseName, const TActorContext& ctx); - + static bool IsAuthStateOK(const IRequestProxyCtx& ctx); - + template <typename TEvent> void Handle(TAutoPtr<TEventHandle<TEvent>>& event, const TActorContext& ctx) { IRequestProxyCtx* requestBaseCtx = event->Get(); @@ -110,23 +110,23 @@ private: const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_API_VALIDATION_ERROR, validationError); requestBaseCtx->RaiseIssue(issue); requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); - } else { + } else { TGRpcRequestProxy::Handle(event, ctx); - } - } - - void Handle(TEvProxyRuntimeEvent::TPtr& event, const TActorContext&) { - IRequestProxyCtx* requestBaseCtx = event->Get(); - TString validationError; - if (!requestBaseCtx->Validate(validationError)) { - const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_API_VALIDATION_ERROR, validationError); - requestBaseCtx->RaiseIssue(issue); - requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); - } else { - event->Release().Release()->Pass(*this); - } - } - + } + } + + void Handle(TEvProxyRuntimeEvent::TPtr& event, const TActorContext&) { + IRequestProxyCtx* requestBaseCtx = event->Get(); + TString validationError; + if (!requestBaseCtx->Validate(validationError)) { + const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_API_VALIDATION_ERROR, validationError); + requestBaseCtx->RaiseIssue(issue); + requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); + } else { + event->Release().Release()->Pass(*this); + } + } + void Handle(TRefreshTokenImpl::TPtr& event, const TActorContext& ctx) { const auto record = event->Get(); ctx.Send(record->GetFromId(), new TGRpcRequestProxy::TEvRefreshTokenResponse { @@ -134,8 +134,8 @@ private: record->GetInternalToken(), record->GetAuthState().State == NGrpc::TAuthState::EAuthState::AS_UNAVAILABLE, NYql::TIssues()}); - } - + } + // returns true and defer event if no updates for given database // otherwice returns false and leave event untouched template <typename TEvent> @@ -143,22 +143,22 @@ private: std::deque<TEventReqHolder>& queue = DeferredEvents[database]; if (queue.size() >= MAX_DEFERRED_EVENTS_PER_DATABASE) { return false; - } - + } + if (queue.empty()) { DoStartUpdate(database); } queue.push_back(TEventReqHolder(ev.Release(), reqCtx)); return true; - } - + } + template <typename TEvent> void PreHandle(TAutoPtr<TEventHandle<TEvent>>& event, const TActorContext& ctx) { IRequestProxyCtx* requestBaseCtx = event->Get(); - + LogRequest(event); - + if (!SchemeCache) { const TString error = "Grpc proxy is not ready to accept request, no proxy service"; LOG_ERROR_S(ctx, NKikimrServices::GRPC_SERVER, error); @@ -167,17 +167,17 @@ private: requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); return; } - + if (IsAuthStateOK(*requestBaseCtx)) { Handle(event, ctx); return; } auto state = requestBaseCtx->GetAuthState(); - + if (state.State == NGrpc::TAuthState::AS_FAIL) { requestBaseCtx->ReplyUnauthenticated(); - return; + return; } if (state.State == NGrpc::TAuthState::AS_UNAVAILABLE) { @@ -188,14 +188,14 @@ private: requestBaseCtx->ReplyUnavaliable(); return; } - + TString databaseName; const TDatabaseInfo* database = nullptr; bool skipResourceCheck = false; // do not check connect rights for the deprecated requests without database // remove this along with AllowYdbRequestsWithoutDatabase flag bool skipCheckConnectRigths = false; - + if (state.State == NGrpc::TAuthState::AS_NOT_PERFORMED) { const auto& maybeDatabaseName = requestBaseCtx->GetDatabaseName(); if (maybeDatabaseName && !maybeDatabaseName.GetRef().empty()) { @@ -203,13 +203,13 @@ private: } else { if (!AllowYdbRequestsWithoutDatabase) { requestBaseCtx->ReplyUnauthenticated("Requests without specified database is not allowed"); - return; + return; } else { databaseName = RootDatabase; skipResourceCheck = true; skipCheckConnectRigths = true; - } - } + } + } auto it = Databases.find(databaseName); if (it != Databases.end() && it->second.IsDatabaseReady()) { database = &it->second; @@ -222,11 +222,11 @@ private: const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_DB_NOT_READY, error); requestBaseCtx->RaiseIssue(issue); requestBaseCtx->ReplyUnavaliable(); - } + } return; - } - } - + } + } + if (database) { if (database->SchemeBoardResult) { const auto& domain = database->SchemeBoardResult->DescribeSchemeResult.GetPathDescription().GetDomainDescription(); @@ -250,28 +250,28 @@ private: requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); return; } - + Register(CreateGrpcRequestCheckActor<TEvent>(SelfId(), database->SchemeBoardResult->DescribeSchemeResult, database->SecurityObject, event.Release(), Counters, skipCheckConnectRigths)); return; - } - + } + // in case we somehow skipped all auth checks const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_TXPROXY_ERROR, "Can't authenticate request"); requestBaseCtx->RaiseIssue(issue); requestBaseCtx->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); - } - + } + void ForgetDatabase(const TString& database); void SubscribeToDatabase(const TString& database); void DoStartUpdate(const TString& database); bool DeferAndStartUpdate(const TString& database, TAutoPtr<IEventHandle>& ev, IRequestProxyCtx*); - + const NKikimrConfig::TAppConfig& GetAppConfig() const override { return AppConfig; - } - + } + virtual void PassAway() override { for (auto& [database, queue] : DeferredEvents) { for (TEventReqHolder& req : queue) { @@ -287,17 +287,17 @@ private: std::unordered_map<TString, TDatabaseInfo> Databases; std::unordered_map<TString, std::deque<TEventReqHolder>> DeferredEvents; // Events deferred to handle after getting database info std::unordered_map<TString, TActorId> Subscribers; - THashSet<TSubDomainKey> SubDomainKeys; - bool AllowYdbRequestsWithoutDatabase = true; + THashSet<TSubDomainKey> SubDomainKeys; + bool AllowYdbRequestsWithoutDatabase = true; NKikimrConfig::TAppConfig AppConfig; TActorId SchemeCache; - bool DynamicNode = false; + bool DynamicNode = false; TString RootDatabase; - TIntrusivePtr<TGrpcProxyCounters> Counters; -}; - -void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { - ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe()); + TIntrusivePtr<TGrpcProxyCounters> Counters; +}; + +void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { + ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe()); AllowYdbRequestsWithoutDatabase = AppData(ctx)->FeatureFlags.GetAllowYdbRequestsWithoutDatabase(); // Subscribe for TableService config changes @@ -306,17 +306,17 @@ void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}), IEventHandle::FlagTrackDelivery); - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); - - auto nodeID = SelfId().NodeId(); - - if (AppData()->DynamicNameserviceConfig) { - DynamicNode = nodeID > AppData()->DynamicNameserviceConfig->MaxStaticNodeId; - LOG_NOTICE(ctx, NKikimrServices::GRPC_SERVER, "Grpc request proxy started, nodeid# %d, serve as %s node", - nodeID, (DynamicNode ? "dynamic" : "static")); - } - - Counters = MakeIntrusive<TGrpcProxyCounters>(AppData()->Counters); + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); + + auto nodeID = SelfId().NodeId(); + + if (AppData()->DynamicNameserviceConfig) { + DynamicNode = nodeID > AppData()->DynamicNameserviceConfig->MaxStaticNodeId; + LOG_NOTICE(ctx, NKikimrServices::GRPC_SERVER, "Grpc request proxy started, nodeid# %d, serve as %s node", + nodeID, (DynamicNode ? "dynamic" : "static")); + } + + Counters = MakeIntrusive<TGrpcProxyCounters>(AppData()->Counters); RootDatabase = DatabaseFromDomain(); TDatabaseInfo& database = Databases[RootDatabase]; @@ -324,9 +324,9 @@ void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { database.State = NKikimrTenantPool::EState::TENANT_OK; DoStartUpdate(RootDatabase); - Become(&TThis::StateFunc); -} - + Become(&TThis::StateFunc); +} + void TGRpcRequestProxyImpl::ReplayEvents(const TString& databaseName, const TActorContext& ctx) { auto itDeferredEvents = DeferredEvents.find(databaseName); if (itDeferredEvents != DeferredEvents.end()) { @@ -343,11 +343,11 @@ void TGRpcRequestProxyImpl::ReplayEvents(const TString& databaseName, const TAct } } -void TGRpcRequestProxyImpl::HandlePoolStatus(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev, const TActorContext& ctx) { - const auto &event = ev->Get()->Record; +void TGRpcRequestProxyImpl::HandlePoolStatus(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev, const TActorContext& ctx) { + const auto &event = ev->Get()->Record; LOG_INFO_S(ctx, NKikimrServices::GRPC_SERVER, "Received tenant pool status, serving tenants: " << event.ShortDebugString()); - - SubDomainKeys.clear(); + + SubDomainKeys.clear(); for (auto it = Databases.begin(); it != Databases.end();) { if (it->second.DatabaseType != TDatabaseInfo::TDatabaseType::Root) { it = Databases.erase(it); @@ -355,14 +355,14 @@ void TGRpcRequestProxyImpl::HandlePoolStatus(TEvTenantPool::TEvTenantPoolStatus: ++it; } } - for (const auto& slot : event.GetSlots()) { + for (const auto& slot : event.GetSlots()) { if (slot.GetAssignedTenant().empty()) { continue; - } - const auto& subDomainKey = TSubDomainKey(slot.GetDomainKey()); - if (subDomainKey) { - SubDomainKeys.insert(subDomainKey); - } + } + const auto& subDomainKey = TSubDomainKey(slot.GetDomainKey()); + if (subDomainKey) { + SubDomainKeys.insert(subDomainKey); + } TString databaseName = CanonizePath(slot.GetAssignedTenant()); auto itDatabase = Databases.try_emplace(databaseName); if (itDatabase.second) { @@ -375,18 +375,18 @@ void TGRpcRequestProxyImpl::HandlePoolStatus(TEvTenantPool::TEvTenantPoolStatus: if (database.IsDatabaseReady()) { ReplayEvents(databaseName, ctx); } - } -} - -void TGRpcRequestProxyImpl::HandleRefreshToken(TRefreshTokenImpl::TPtr& ev, const TActorContext& ctx) { - const auto record = ev->Get(); - ctx.Send(record->GetFromId(), new TGRpcRequestProxy::TEvRefreshTokenResponse { + } +} + +void TGRpcRequestProxyImpl::HandleRefreshToken(TRefreshTokenImpl::TPtr& ev, const TActorContext& ctx) { + const auto record = ev->Get(); + ctx.Send(record->GetFromId(), new TGRpcRequestProxy::TEvRefreshTokenResponse { record->GetAuthState().State == NGrpc::TAuthState::EAuthState::AS_OK, - record->GetInternalToken(), + record->GetInternalToken(), record->GetAuthState().State == NGrpc::TAuthState::EAuthState::AS_UNAVAILABLE, - NYql::TIssues()}); -} - + NYql::TIssues()}); +} + void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Subscribed for config changes"); } @@ -401,12 +401,12 @@ void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConsole::TEvConfigNotifica Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); } -void TGRpcRequestProxyImpl::HandleProxyService(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { - SchemeCache = ev->Get()->Services.SchemeCache; - LOG_DEBUG(*TlsActivationContext, NKikimrServices::GRPC_SERVER, - "Got proxy service configuration"); -} - +void TGRpcRequestProxyImpl::HandleProxyService(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { + SchemeCache = ev->Get()->Services.SchemeCache; + LOG_DEBUG(*TlsActivationContext, NKikimrServices::GRPC_SERVER, + "Got proxy service configuration"); +} + void TGRpcRequestProxyImpl::HandleUndelivery(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: @@ -428,25 +428,25 @@ bool TGRpcRequestProxyImpl::IsAuthStateOK(const IRequestProxyCtx& ctx) { const auto& state = ctx.GetAuthState(); return state.State == NGrpc::TAuthState::AS_OK || state.State == NGrpc::TAuthState::AS_FAIL && state.NeedAuth == false || - state.NeedAuth == false && !ctx.GetYdbToken(); -} - + state.NeedAuth == false && !ctx.GetYdbToken(); +} + void TGRpcRequestProxyImpl::HandleSchemeBoard(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev, const TActorContext& ctx) { TString databaseName = ev->Get()->Path; LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "SchemeBoardUpdate " << databaseName); - + // non-serverless databases should be in the cache already auto itDatabase = Databases.try_emplace(CanonizePath(databaseName)); TDatabaseInfo& database = itDatabase.first->second; if (itDatabase.second) { database.State = NKikimrTenantPool::EState::TENANT_OK; database.DatabaseType = TDatabaseInfo::TDatabaseType::Serverless; - } + } database.SchemeBoardResult = ev->Release(); const NKikimrScheme::TEvDescribeSchemeResult& describeScheme(database.SchemeBoardResult->DescribeSchemeResult); database.SecurityObject = new TSecurityObject(describeScheme.GetPathDescription().GetSelf().GetOwner(), describeScheme.GetPathDescription().GetSelf().GetEffectiveACL(), false); - + if (describeScheme.GetPathDescription().HasDomainDescription() && describeScheme.GetPathDescription().GetDomainDescription().HasSecurityState()) { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Updating SecurityState for " << databaseName); @@ -458,29 +458,29 @@ void TGRpcRequestProxyImpl::HandleSchemeBoard(TSchemeBoardEvents::TEvNotifyUpdat LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Can't update SecurityState for " << databaseName << " - no DomainDescription"); } else if (!describeScheme.GetPathDescription().GetDomainDescription().HasSecurityState()) { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Can't update SecurityState for " << databaseName << " - no SecurityState"); - } - } - + } + } + if (database.IsDatabaseReady()) { ReplayEvents(databaseName, ctx); - } -} - + } +} + void TGRpcRequestProxyImpl::HandleSchemeBoard(TSchemeBoardEvents::TEvNotifyDelete::TPtr& ev) { LOG_WARN_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "SchemeBoardDelete " << ev->Get()->Path << " Strong=" << ev->Get()->Strong); - + if (ev->Get()->Strong) { ForgetDatabase(ev->Get()->Path); } -} - +} + void TGRpcRequestProxyImpl::ForgetDatabase(const TString& database) { auto itSubscriber = Subscribers.find(database); if (itSubscriber != Subscribers.end()) { Send(itSubscriber->second, new TEvents::TEvPoisonPill()); Subscribers.erase(itSubscriber); - } + } auto itDeferredEvents = DeferredEvents.find(database); if (itDeferredEvents != DeferredEvents.end()) { auto& queue(itDeferredEvents->second); @@ -490,13 +490,13 @@ void TGRpcRequestProxyImpl::ForgetDatabase(const TString& database) { queue.pop_front(); } DeferredEvents.erase(itDeferredEvents); - } + } Databases.erase(database); } - + void TGRpcRequestProxyImpl::SubscribeToDatabase(const TString& database) { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Subscribe to " << database); - + Y_VERIFY(!AppData()->DomainsInfo->Domains.empty()); auto& domain = AppData()->DomainsInfo->Domains.begin()->second; ui64 domainOwnerId = domain->SchemeRoot; @@ -508,60 +508,60 @@ void TGRpcRequestProxyImpl::SubscribeToDatabase(const TString& database) { Send(itSubscriber.first->second, new TEvents::TEvPoisonPill()); itSubscriber.first->second = subscriberId; } -} - +} + void TGRpcRequestProxyImpl::DoStartUpdate(const TString& database) { // we will receive update (or delete) upon sucessfull subscription SubscribeToDatabase(database); } -template<typename TEvent> -void LogRequest(const TEvent& event) { +template<typename TEvent> +void LogRequest(const TEvent& event) { auto getDebugString = [&event]()->TString { - TStringStream ss; - ss << "Got grpc request# " << event->Get()->GetRequestName(); - ss << ", traceId# " << event->Get()->GetTraceId().GetOrElse("undef"); - ss << ", sdkBuildInfo# " << event->Get()->GetSdkBuildInfo().GetOrElse("undef"); + TStringStream ss; + ss << "Got grpc request# " << event->Get()->GetRequestName(); + ss << ", traceId# " << event->Get()->GetTraceId().GetOrElse("undef"); + ss << ", sdkBuildInfo# " << event->Get()->GetSdkBuildInfo().GetOrElse("undef"); ss << ", state# " << event->Get()->GetAuthState().State; - ss << ", database# " << event->Get()->GetDatabaseName().GetOrElse("undef"); - ss << ", grpcInfo# " << event->Get()->GetGrpcUserAgent().GetOrElse("undef"); - if (event->Get()->GetDeadline() == TInstant::Max()) { - ss << ", timeout# undef"; - } else { - ss << ", timeout# " << event->Get()->GetDeadline() - TInstant::Now(); - } - return ss.Str(); - }; - - if constexpr (std::is_same_v<TEvListEndpointsRequest::TPtr, TEvent>) { - LOG_NOTICE(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "%s", getDebugString().c_str()); + ss << ", database# " << event->Get()->GetDatabaseName().GetOrElse("undef"); + ss << ", grpcInfo# " << event->Get()->GetGrpcUserAgent().GetOrElse("undef"); + if (event->Get()->GetDeadline() == TInstant::Max()) { + ss << ", timeout# undef"; + } else { + ss << ", timeout# " << event->Get()->GetDeadline() - TInstant::Now(); + } + return ss.Str(); + }; + + if constexpr (std::is_same_v<TEvListEndpointsRequest::TPtr, TEvent>) { + LOG_NOTICE(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "%s", getDebugString().c_str()); } else { - LOG_DEBUG(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "%s", getDebugString().c_str()); - } -} - -void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + LOG_DEBUG(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "%s", getDebugString().c_str()); + } +} + +void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { bool handled = true; // handle internal events - switch (ev->GetTypeRewrite()) { - HFunc(TEvTenantPool::TEvTenantPoolStatus, HandlePoolStatus); - hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, HandleProxyService); + switch (ev->GetTypeRewrite()) { + HFunc(TEvTenantPool::TEvTenantPoolStatus, HandlePoolStatus); + hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, HandleProxyService); hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig); hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig); hFunc(TEvents::TEvUndelivered, HandleUndelivery); HFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleSchemeBoard); hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleSchemeBoard); - default: + default: handled = false; break; - } - + } + if (handled) { - return; - } - + return; + } + // handle external events switch (ev->GetTypeRewrite()) { HFunc(TRefreshTokenImpl, PreHandle); @@ -670,17 +670,17 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorCo HFunc(TEvDataStreamsStartStreamEncryptionRequest, PreHandle); HFunc(TEvDataStreamsStopStreamEncryptionRequest, PreHandle); - HFunc(TEvProxyRuntimeEvent, PreHandle); - + HFunc(TEvProxyRuntimeEvent, PreHandle); + default: Y_FAIL("Unknown request: %u\n", ev->GetTypeRewrite()); break; - } -} - + } +} + IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig) { return new TGRpcRequestProxyImpl(appConfig); -} - -} // namespace NGRpcService -} // namespace NKikimr +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index 0edb3667aa..97315f6e9f 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include "grpc_endpoint.h" -#include "rpc_calls.h" - +#include "rpc_calls.h" + #include <library/cpp/actors/core/actor.h> - + #include <util/generic/ptr.h> #include <util/generic/vector.h> @@ -13,40 +13,40 @@ namespace NKikimrConfig { class TAppConfig; } -namespace NKikimr { - -struct TAppData; - -namespace NGRpcService { - -TString DatabaseFromDomain(const TAppData* appdata); +namespace NKikimr { + +struct TAppData; + +namespace NGRpcService { + +TString DatabaseFromDomain(const TAppData* appdata); IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig); - -class TGRpcRequestProxy : public IFacilityProvider { -public: - enum EEv { - EvRefreshTokenResponse = EventSpaceBegin(TKikimrEvents::ES_GRPC_REQUEST_PROXY), - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_REQUEST_PROXY), - "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_REQUEST_PROXY)"); - - struct TEvRefreshTokenResponse : public TEventLocal<TEvRefreshTokenResponse, EvRefreshTokenResponse> { - bool Authenticated; - TString InternalToken; - bool Retryable; - NYql::TIssues Issues; - - TEvRefreshTokenResponse(bool ok, const TString& token, bool retryable, const NYql::TIssues& issues) - : Authenticated(ok) - , InternalToken(token) - , Retryable(retryable) - , Issues(issues) - {} - }; - -protected: + +class TGRpcRequestProxy : public IFacilityProvider { +public: + enum EEv { + EvRefreshTokenResponse = EventSpaceBegin(TKikimrEvents::ES_GRPC_REQUEST_PROXY), + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_REQUEST_PROXY), + "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_GRPC_REQUEST_PROXY)"); + + struct TEvRefreshTokenResponse : public TEventLocal<TEvRefreshTokenResponse, EvRefreshTokenResponse> { + bool Authenticated; + TString InternalToken; + bool Retryable; + NYql::TIssues Issues; + + TEvRefreshTokenResponse(bool ok, const TString& token, bool retryable, const NYql::TIssues& issues) + : Authenticated(ok) + , InternalToken(token) + , Retryable(retryable) + , Issues(issues) + {} + }; + +protected: void Handle(TEvAlterTableRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCreateTableRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDropTableRequest::TPtr& ev, const TActorContext& ctx); @@ -85,7 +85,7 @@ protected: void Handle(TEvGetShardLocationsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvKikhouseDescribeTableRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvS3ListingRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvBiStreamPingRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvBiStreamPingRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvExperimentalStreamQueryRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvStreamPQWriteRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvStreamPQReadRequest::TPtr& ev, const TActorContext& ctx); @@ -153,12 +153,12 @@ protected: void Handle(TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); TActorId DiscoveryCacheActorID; -}; - +}; + inline TActorId CreateGRpcRequestProxyId() { const auto actorId = TActorId(0, "GRpcReqProxy"); - return actorId; -} - -} // namespace NGRpcService -} // namespace NKikimr + return actorId; +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/local_rate_limiter.cpp b/ydb/core/grpc_services/local_rate_limiter.cpp index 935702b180..3a2fcb78cb 100644 --- a/ydb/core/grpc_services/local_rate_limiter.cpp +++ b/ydb/core/grpc_services/local_rate_limiter.cpp @@ -1,154 +1,154 @@ -#include "local_rate_limiter.h" -#include "rpc_common.h" - +#include "local_rate_limiter.h" +#include "rpc_common.h" + #include <ydb/core/grpc_services/local_rpc/local_rpc.h> -namespace NKikimr { -namespace NRpcService { - -TActorId RateLimiterAcquireUseSameMailbox( - const TRlFullPath& fullPath, - ui64 required, - const TDuration& duration, - std::function<void()>&& onSuccess, - std::function<void()>&& onTimeout, - const TActorContext& ctx) -{ - auto cb = [onSuccess{std::move(onSuccess)}, onTimeout{std::move(onTimeout)}] - (const Ydb::RateLimiter::AcquireResourceResponse& resp) - { - switch (resp.operation().status()) { - case Ydb::StatusIds::SUCCESS: - onSuccess(); - break; - case Ydb::StatusIds::TIMEOUT: - onTimeout(); - break; - default: - // Fallback - onSuccess(); - } - }; - - Ydb::RateLimiter::AcquireResourceRequest request; +namespace NKikimr { +namespace NRpcService { + +TActorId RateLimiterAcquireUseSameMailbox( + const TRlFullPath& fullPath, + ui64 required, + const TDuration& duration, + std::function<void()>&& onSuccess, + std::function<void()>&& onTimeout, + const TActorContext& ctx) +{ + auto cb = [onSuccess{std::move(onSuccess)}, onTimeout{std::move(onTimeout)}] + (const Ydb::RateLimiter::AcquireResourceResponse& resp) + { + switch (resp.operation().status()) { + case Ydb::StatusIds::SUCCESS: + onSuccess(); + break; + case Ydb::StatusIds::TIMEOUT: + onTimeout(); + break; + default: + // Fallback + onSuccess(); + } + }; + + Ydb::RateLimiter::AcquireResourceRequest request; SetDuration(duration, *request.mutable_operation_params()->mutable_operation_timeout()); - request.set_coordination_node_path(fullPath.CoordinationNode); - request.set_resource_path(fullPath.ResourcePath); - request.set_required(required); - return RateLimiterAcquireUseSameMailbox( - std::move(request), - fullPath.DatabaseName, - fullPath.Token, - std::move(cb), - ctx); -} - -TActorId RateLimiterAcquireUseSameMailbox( - Ydb::RateLimiter::AcquireResourceRequest&& request, - const TString& database, - const TString& token, - std::function<void(Ydb::RateLimiter::AcquireResourceResponse resp)>&& cb, - const TActorContext& ctx) -{ - return DoLocalRpcSameMailbox<NKikimr::NGRpcService::TEvAcquireRateLimiterResource>( - std::move(request), std::move(cb), database, token, ctx); -} - -TActorId RateLimiterAcquireUseSameMailbox( - const NGRpcService::IRequestCtxBase& reqCtx, - ui64 required, - const TDuration& duration, - std::function<void()>&& onSuccess, - std::function<void()>&& onTimeout, - const TActorContext& ctx) -{ - if (const auto maybeRlPath = reqCtx.GetRlPath()) { - auto cb = [onSuccess{std::move(onSuccess)}, onTimeout{std::move(onTimeout)}] - (const Ydb::RateLimiter::AcquireResourceResponse& resp) - { - switch (resp.operation().status()) { - case Ydb::StatusIds::SUCCESS: - onSuccess(); - break; - case Ydb::StatusIds::TIMEOUT: - onTimeout(); - break; - default: - // Fallback - onSuccess(); - } - }; - - const auto& rlPath = maybeRlPath.GetRef(); - Ydb::RateLimiter::AcquireResourceRequest request; + request.set_coordination_node_path(fullPath.CoordinationNode); + request.set_resource_path(fullPath.ResourcePath); + request.set_required(required); + return RateLimiterAcquireUseSameMailbox( + std::move(request), + fullPath.DatabaseName, + fullPath.Token, + std::move(cb), + ctx); +} + +TActorId RateLimiterAcquireUseSameMailbox( + Ydb::RateLimiter::AcquireResourceRequest&& request, + const TString& database, + const TString& token, + std::function<void(Ydb::RateLimiter::AcquireResourceResponse resp)>&& cb, + const TActorContext& ctx) +{ + return DoLocalRpcSameMailbox<NKikimr::NGRpcService::TEvAcquireRateLimiterResource>( + std::move(request), std::move(cb), database, token, ctx); +} + +TActorId RateLimiterAcquireUseSameMailbox( + const NGRpcService::IRequestCtxBase& reqCtx, + ui64 required, + const TDuration& duration, + std::function<void()>&& onSuccess, + std::function<void()>&& onTimeout, + const TActorContext& ctx) +{ + if (const auto maybeRlPath = reqCtx.GetRlPath()) { + auto cb = [onSuccess{std::move(onSuccess)}, onTimeout{std::move(onTimeout)}] + (const Ydb::RateLimiter::AcquireResourceResponse& resp) + { + switch (resp.operation().status()) { + case Ydb::StatusIds::SUCCESS: + onSuccess(); + break; + case Ydb::StatusIds::TIMEOUT: + onTimeout(); + break; + default: + // Fallback + onSuccess(); + } + }; + + const auto& rlPath = maybeRlPath.GetRef(); + Ydb::RateLimiter::AcquireResourceRequest request; SetDuration(duration, *request.mutable_operation_params()->mutable_operation_timeout()); - request.set_coordination_node_path(rlPath.CoordinationNode); - request.set_resource_path(rlPath.ResourcePath); - request.set_required(required); - return RateLimiterAcquireUseSameMailbox( - std::move(request), - reqCtx.GetDatabaseName().GetOrElse(""), - reqCtx.GetInternalToken(), - std::move(cb), - ctx); - } - return {}; -} - -static void Fill(const TRlConfig::TOnReqAction& action, - const TString& coordinationNodePath, - const TString& resourcePath, - TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>>& res) -{ - Ydb::RateLimiter::AcquireResourceRequest request; - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); - request.set_required(action.Required); - - res.push_back({Actions::OnReq, request}); -} - -static void Fill(const TRlConfig::TOnRespAction&, - const TString& coordinationNodePath, - const TString& resourcePath, - TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>>& res) -{ - Ydb::RateLimiter::AcquireResourceRequest request; - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); - - res.push_back({Actions::OnResp, request}); -} - -TMaybe<TRlPath> Match(const TRlConfig& rlConfig, const THashMap<TString, TString>& attrs) { - const auto coordinationNodeIt = attrs.find(rlConfig.CoordinationNodeKey); - if (coordinationNodeIt == attrs.end()) { - return {}; - } - - const auto rlResourcePathIt = attrs.find(rlConfig.ResourceKey); - if (rlResourcePathIt == attrs.end()) { - return {}; - } - - return TRlPath{coordinationNodeIt->second, rlResourcePathIt->second}; -} - -TVector<std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> MakeRequests( - const TRlConfig& rlConfig, const TRlPath& rlPath) -{ - TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> result; - result.reserve(rlConfig.Actions.size()); - - for (auto& action : rlConfig.Actions) { - auto f = [&](const auto& item) { - Fill(item, rlPath.CoordinationNode, rlPath.ResourcePath, result); - }; + request.set_coordination_node_path(rlPath.CoordinationNode); + request.set_resource_path(rlPath.ResourcePath); + request.set_required(required); + return RateLimiterAcquireUseSameMailbox( + std::move(request), + reqCtx.GetDatabaseName().GetOrElse(""), + reqCtx.GetInternalToken(), + std::move(cb), + ctx); + } + return {}; +} + +static void Fill(const TRlConfig::TOnReqAction& action, + const TString& coordinationNodePath, + const TString& resourcePath, + TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>>& res) +{ + Ydb::RateLimiter::AcquireResourceRequest request; + request.set_coordination_node_path(coordinationNodePath); + request.set_resource_path(resourcePath); + request.set_required(action.Required); + + res.push_back({Actions::OnReq, request}); +} + +static void Fill(const TRlConfig::TOnRespAction&, + const TString& coordinationNodePath, + const TString& resourcePath, + TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>>& res) +{ + Ydb::RateLimiter::AcquireResourceRequest request; + request.set_coordination_node_path(coordinationNodePath); + request.set_resource_path(resourcePath); + + res.push_back({Actions::OnResp, request}); +} + +TMaybe<TRlPath> Match(const TRlConfig& rlConfig, const THashMap<TString, TString>& attrs) { + const auto coordinationNodeIt = attrs.find(rlConfig.CoordinationNodeKey); + if (coordinationNodeIt == attrs.end()) { + return {}; + } + + const auto rlResourcePathIt = attrs.find(rlConfig.ResourceKey); + if (rlResourcePathIt == attrs.end()) { + return {}; + } + + return TRlPath{coordinationNodeIt->second, rlResourcePathIt->second}; +} + +TVector<std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> MakeRequests( + const TRlConfig& rlConfig, const TRlPath& rlPath) +{ + TVector <std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> result; + result.reserve(rlConfig.Actions.size()); + + for (auto& action : rlConfig.Actions) { + auto f = [&](const auto& item) { + Fill(item, rlPath.CoordinationNode, rlPath.ResourcePath, result); + }; std::visit(f, action); - } - - return result; -} - -} // namespace NRpcService -} // namespace NKikimr + } + + return result; +} + +} // namespace NRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/local_rate_limiter.h b/ydb/core/grpc_services/local_rate_limiter.h index 8c483c40b3..cfc04b8d56 100644 --- a/ydb/core/grpc_services/local_rate_limiter.h +++ b/ydb/core/grpc_services/local_rate_limiter.h @@ -1,91 +1,91 @@ -#pragma once -#include "defs.h" - -#include <util/generic/variant.h> +#pragma once +#include "defs.h" + +#include <util/generic/variant.h> #include <ydb/core/grpc_services/base/base.h> - -#include <functional> - -namespace NActors { - struct TActorId; -} - -class TDuration; - -namespace Ydb { -namespace RateLimiter { - class AcquireResourceResponse; - class AcquireResourceRequest; -} -} - -namespace NKikimr { - -namespace NGRpcService { - class IRequestCtxBase; -} - -namespace NRpcService { - -struct TRlFullPath { - TString CoordinationNode; - TString ResourcePath; - TString DatabaseName; - TString Token; -}; - -// Acquire `acquireUnits`. The rpc actor will be registered using same mailbox. -TActorId RateLimiterAcquireUseSameMailbox( - const TRlFullPath& fullPath, - ui64 required, - const TDuration& duration, - std::function<void()>&& onSuccess, - std::function<void()>&& onTimeout, - const TActorContext& ctx); - -TActorId RateLimiterAcquireUseSameMailbox( - Ydb::RateLimiter::AcquireResourceRequest&& request, - const TString& database, - const TString& token, - std::function<void(Ydb::RateLimiter::AcquireResourceResponse resp)>&& cb, const TActorContext &ctx); - -TActorId RateLimiterAcquireUseSameMailbox(const NGRpcService::IRequestCtxBase& reqCtx, - ui64 required, - const TDuration& duration, - std::function<void()>&& onSuccess, - std::function<void()>&& onFail, - const NActors::TActorContext &ctx); - -struct TRlConfig { - struct TOnReqAction { - ui64 Required; - }; - - struct TOnRespAction { - }; - + +#include <functional> + +namespace NActors { + struct TActorId; +} + +class TDuration; + +namespace Ydb { +namespace RateLimiter { + class AcquireResourceResponse; + class AcquireResourceRequest; +} +} + +namespace NKikimr { + +namespace NGRpcService { + class IRequestCtxBase; +} + +namespace NRpcService { + +struct TRlFullPath { + TString CoordinationNode; + TString ResourcePath; + TString DatabaseName; + TString Token; +}; + +// Acquire `acquireUnits`. The rpc actor will be registered using same mailbox. +TActorId RateLimiterAcquireUseSameMailbox( + const TRlFullPath& fullPath, + ui64 required, + const TDuration& duration, + std::function<void()>&& onSuccess, + std::function<void()>&& onTimeout, + const TActorContext& ctx); + +TActorId RateLimiterAcquireUseSameMailbox( + Ydb::RateLimiter::AcquireResourceRequest&& request, + const TString& database, + const TString& token, + std::function<void(Ydb::RateLimiter::AcquireResourceResponse resp)>&& cb, const TActorContext &ctx); + +TActorId RateLimiterAcquireUseSameMailbox(const NGRpcService::IRequestCtxBase& reqCtx, + ui64 required, + const TDuration& duration, + std::function<void()>&& onSuccess, + std::function<void()>&& onFail, + const NActors::TActorContext &ctx); + +struct TRlConfig { + struct TOnReqAction { + ui64 Required; + }; + + struct TOnRespAction { + }; + using TActions = std::variant<TOnReqAction, TOnRespAction>; - - TRlConfig(const TString& coordinationNodeKey, const TString& resourceKey, const TVector<TActions>& actions) - : CoordinationNodeKey(coordinationNodeKey) - , ResourceKey(resourceKey) - , Actions(actions) - {} - - const TString CoordinationNodeKey; - // The key in the database attribute map to find actual 'resource path' - const TString ResourceKey; - const TVector<TActions> Actions; -}; - -enum class Actions { - OnReq, - OnResp -}; - -TMaybe<TRlPath> Match(const TRlConfig& rlConfig, const THashMap<TString, TString>& attrs); -TVector<std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> MakeRequests( - const TRlConfig& rlConfig, const TRlPath& rlPath); - -} // namespace NRpcService -} // namespace NKikimr + + TRlConfig(const TString& coordinationNodeKey, const TString& resourceKey, const TVector<TActions>& actions) + : CoordinationNodeKey(coordinationNodeKey) + , ResourceKey(resourceKey) + , Actions(actions) + {} + + const TString CoordinationNodeKey; + // The key in the database attribute map to find actual 'resource path' + const TString ResourceKey; + const TVector<TActions> Actions; +}; + +enum class Actions { + OnReq, + OnResp +}; + +TMaybe<TRlPath> Match(const TRlConfig& rlConfig, const THashMap<TString, TString>& attrs); +TVector<std::pair<Actions, Ydb::RateLimiter::AcquireResourceRequest>> MakeRequests( + const TRlConfig& rlConfig, const TRlPath& rlPath); + +} // namespace NRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/local_rpc/local_rpc.h b/ydb/core/grpc_services/local_rpc/local_rpc.h index 5c1893536d..23c3b6c803 100644 --- a/ydb/core/grpc_services/local_rpc/local_rpc.h +++ b/ydb/core/grpc_services/local_rpc/local_rpc.h @@ -1,211 +1,211 @@ -#pragma once - +#pragma once + #include <ydb/core/grpc_services/base/base.h> #include <ydb/core/base/appdata.h> - -#include <library/cpp/threading/future/future.h> - -namespace NKikimr { - -namespace NRpcService { - -template<typename TResponse> -class TPromiseWrapper { -public: - TPromiseWrapper(NThreading::TPromise<TResponse> promise) - : Promise(promise) - {} - - void operator()(const TResponse& resp) { - Promise.SetValue(resp); - } - -private: - NThreading::TPromise<TResponse> Promise; -}; - -template<typename TRpc, typename TCbWrapper> -class TLocalRpcCtx : public NGRpcService::IRequestOpCtx { -public: - using TResp = typename TRpc::TResponse; - template<typename TProto, typename TCb> - TLocalRpcCtx(TProto&& req, TCb&& cb, const TString& databaseName, const TString& token) - : Request(std::forward<TProto>(req)) - , CbWrapper(std::forward<TCb>(cb)) - , DatabaseName(databaseName) - , InternalToken(token) - {} - - const TMaybe<TString> GetDatabaseName() const override { - if (DatabaseName.empty()) - return Nothing(); - return DatabaseName; - } - - const TString& GetInternalToken() const override { - return InternalToken; - } - + +#include <library/cpp/threading/future/future.h> + +namespace NKikimr { + +namespace NRpcService { + +template<typename TResponse> +class TPromiseWrapper { +public: + TPromiseWrapper(NThreading::TPromise<TResponse> promise) + : Promise(promise) + {} + + void operator()(const TResponse& resp) { + Promise.SetValue(resp); + } + +private: + NThreading::TPromise<TResponse> Promise; +}; + +template<typename TRpc, typename TCbWrapper> +class TLocalRpcCtx : public NGRpcService::IRequestOpCtx { +public: + using TResp = typename TRpc::TResponse; + template<typename TProto, typename TCb> + TLocalRpcCtx(TProto&& req, TCb&& cb, const TString& databaseName, const TString& token) + : Request(std::forward<TProto>(req)) + , CbWrapper(std::forward<TCb>(cb)) + , DatabaseName(databaseName) + , InternalToken(token) + {} + + const TMaybe<TString> GetDatabaseName() const override { + if (DatabaseName.empty()) + return Nothing(); + return DatabaseName; + } + + const TString& GetInternalToken() const override { + return InternalToken; + } + const TMaybe<TString> GetPeerMetaValues(const TString&) const override { Y_FAIL("Unimplemented"); return TMaybe<TString>{}; } - void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { - TResp resp; - NGRpcService::TCommonResponseFiller<TResp, true>::Fill(resp, IssueManager.GetIssues(), CostInfo.get(), status); - CbWrapper(resp); - } - - TString GetPeerName() const override { - return "localhost"; - } - - const TString& GetRequestName() const override { - return TRpc::TRequest::descriptor()->name(); - } - - void SendResult(const google::protobuf::Message& result, Ydb::StatusIds::StatusCode status) override { - TResp resp; - auto deferred = resp.mutable_operation(); - deferred->set_ready(true); - deferred->set_status(status); - if (CostInfo) { - deferred->mutable_cost_info()->CopyFrom(*CostInfo); - } - NYql::IssuesToMessage(IssueManager.GetIssues(), deferred->mutable_issues()); - auto data = deferred->mutable_result(); - data->PackFrom(result); - CbWrapper(resp); - } - - void SendResult(const google::protobuf::Message& result, - Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<NGRpcService::TYdbIssueMessageType>& message) override - { - TResp resp; - auto deferred = resp.mutable_operation(); - deferred->set_ready(true); - deferred->set_status(status); - deferred->mutable_issues()->MergeFrom(message); - if (CostInfo) { - deferred->mutable_cost_info()->CopyFrom(*CostInfo); - } - auto data = deferred->mutable_result(); - data->PackFrom(result); - CbWrapper(resp); - } - - void SendResult(Ydb::StatusIds::StatusCode status, + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { + TResp resp; + NGRpcService::TCommonResponseFiller<TResp, true>::Fill(resp, IssueManager.GetIssues(), CostInfo.get(), status); + CbWrapper(resp); + } + + TString GetPeerName() const override { + return "localhost"; + } + + const TString& GetRequestName() const override { + return TRpc::TRequest::descriptor()->name(); + } + + void SendResult(const google::protobuf::Message& result, Ydb::StatusIds::StatusCode status) override { + TResp resp; + auto deferred = resp.mutable_operation(); + deferred->set_ready(true); + deferred->set_status(status); + if (CostInfo) { + deferred->mutable_cost_info()->CopyFrom(*CostInfo); + } + NYql::IssuesToMessage(IssueManager.GetIssues(), deferred->mutable_issues()); + auto data = deferred->mutable_result(); + data->PackFrom(result); + CbWrapper(resp); + } + + void SendResult(const google::protobuf::Message& result, + Ydb::StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<NGRpcService::TYdbIssueMessageType>& message) override - { - TResp resp; - auto deferred = resp.mutable_operation(); - deferred->set_ready(true); - deferred->set_status(status); - deferred->mutable_issues()->MergeFrom(message); - if (CostInfo) { - deferred->mutable_cost_info()->CopyFrom(*CostInfo); - } - CbWrapper(resp); - } - + { + TResp resp; + auto deferred = resp.mutable_operation(); + deferred->set_ready(true); + deferred->set_status(status); + deferred->mutable_issues()->MergeFrom(message); + if (CostInfo) { + deferred->mutable_cost_info()->CopyFrom(*CostInfo); + } + auto data = deferred->mutable_result(); + data->PackFrom(result); + CbWrapper(resp); + } + + void SendResult(Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<NGRpcService::TYdbIssueMessageType>& message) override + { + TResp resp; + auto deferred = resp.mutable_operation(); + deferred->set_ready(true); + deferred->set_status(status); + deferred->mutable_issues()->MergeFrom(message); + if (CostInfo) { + deferred->mutable_cost_info()->CopyFrom(*CostInfo); + } + CbWrapper(resp); + } + void SendOperation(const Ydb::Operations::Operation& operation) override { - TResp resp; - resp.mutable_operation()->CopyFrom(operation); - CbWrapper(resp); - } - - void RaiseIssue(const NYql::TIssue& issue) override { - IssueManager.RaiseIssue(issue); - } - - void RaiseIssues(const NYql::TIssues& issues) override { - IssueManager.RaiseIssues(issues); - } - - google::protobuf::Arena* GetArena() override { - return &Arena; - } - - const google::protobuf::Message* GetRequest() const override { - return &Request; - } - - void SetClientLostAction(std::function<void()>&&) override {} - + TResp resp; + resp.mutable_operation()->CopyFrom(operation); + CbWrapper(resp); + } + + void RaiseIssue(const NYql::TIssue& issue) override { + IssueManager.RaiseIssue(issue); + } + + void RaiseIssues(const NYql::TIssues& issues) override { + IssueManager.RaiseIssues(issues); + } + + google::protobuf::Arena* GetArena() override { + return &Arena; + } + + const google::protobuf::Message* GetRequest() const override { + return &Request; + } + + void SetClientLostAction(std::function<void()>&&) override {} + void AddServerHint(const TString&) override {} - void SetRuHeader(ui64) override {} - - // Unimplemented methods - void ReplyWithRpcStatus(grpc::StatusCode, const TString& msg = "") override { - Y_UNUSED(msg); - Y_FAIL("Unimplemented for local rpc"); - } - - TMaybe<TString> GetTraceId() const override { - return Nothing(); - } - - TInstant GetDeadline() const override { - return TInstant::Max(); - } - - const TMaybe<TString> GetRequestType() const override { - return Nothing(); - } - + void SetRuHeader(ui64) override {} + + // Unimplemented methods + void ReplyWithRpcStatus(grpc::StatusCode, const TString& msg = "") override { + Y_UNUSED(msg); + Y_FAIL("Unimplemented for local rpc"); + } + + TMaybe<TString> GetTraceId() const override { + return Nothing(); + } + + TInstant GetDeadline() const override { + return TInstant::Max(); + } + + const TMaybe<TString> GetRequestType() const override { + return Nothing(); + } + void SetCostInfo(float consumed_units) override { - CostInfo = std::make_unique<Ydb::CostInfo>(); - CostInfo->set_consumed_units(consumed_units); - } - - TMaybe<NRpcService::TRlPath> GetRlPath() const override { - return Nothing(); - } - -private: - void Reply(NProtoBuf::Message *r, ui32) override { - TResp* resp = dynamic_cast<TResp*>(r); - Y_VERIFY(resp); - CbWrapper(*resp); - } - -private: - typename TRpc::TRequest Request; - TCbWrapper CbWrapper; - const TString DatabaseName; - const TString InternalToken; - - NYql::TIssueManager IssueManager; - google::protobuf::Arena Arena; - std::unique_ptr<Ydb::CostInfo> CostInfo; -}; - -template<typename TRpc> -NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest&& proto, const TString& database, const TString& token, TActorSystem* actorSystem) { - auto promise = NThreading::NewPromise<typename TRpc::TResponse>(); - - proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); - - using TCbWrapper = TPromiseWrapper<typename TRpc::TResponse>; - auto req = new TLocalRpcCtx<TRpc, TCbWrapper>(std::move(proto), TCbWrapper(promise), database, token); - auto actor = TRpc::CreateRpcActor(req); - actorSystem->Register(actor, TMailboxType::HTSwap, actorSystem->AppData<TAppData>()->UserPoolId); - - return promise.GetFuture(); -} - -template<typename TRpc> -TActorId DoLocalRpcSameMailbox(typename TRpc::TRequest&& proto, std::function<void(typename TRpc::TResponse)>&& cb, const TString& database, const TString& token, const TActorContext& ctx) { - proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); - - auto req = new TLocalRpcCtx<TRpc, std::function<void(typename TRpc::TResponse)>>(std::move(proto), std::move(cb), database, token); - auto actor = TRpc::CreateRpcActor(req); - return ctx.RegisterWithSameMailbox(actor); -} - -} // namespace NRpcService -} // namespace NKikimr + CostInfo = std::make_unique<Ydb::CostInfo>(); + CostInfo->set_consumed_units(consumed_units); + } + + TMaybe<NRpcService::TRlPath> GetRlPath() const override { + return Nothing(); + } + +private: + void Reply(NProtoBuf::Message *r, ui32) override { + TResp* resp = dynamic_cast<TResp*>(r); + Y_VERIFY(resp); + CbWrapper(*resp); + } + +private: + typename TRpc::TRequest Request; + TCbWrapper CbWrapper; + const TString DatabaseName; + const TString InternalToken; + + NYql::TIssueManager IssueManager; + google::protobuf::Arena Arena; + std::unique_ptr<Ydb::CostInfo> CostInfo; +}; + +template<typename TRpc> +NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest&& proto, const TString& database, const TString& token, TActorSystem* actorSystem) { + auto promise = NThreading::NewPromise<typename TRpc::TResponse>(); + + proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); + + using TCbWrapper = TPromiseWrapper<typename TRpc::TResponse>; + auto req = new TLocalRpcCtx<TRpc, TCbWrapper>(std::move(proto), TCbWrapper(promise), database, token); + auto actor = TRpc::CreateRpcActor(req); + actorSystem->Register(actor, TMailboxType::HTSwap, actorSystem->AppData<TAppData>()->UserPoolId); + + return promise.GetFuture(); +} + +template<typename TRpc> +TActorId DoLocalRpcSameMailbox(typename TRpc::TRequest&& proto, std::function<void(typename TRpc::TResponse)>&& cb, const TString& database, const TString& token, const TActorContext& ctx) { + proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); + + auto req = new TLocalRpcCtx<TRpc, std::function<void(typename TRpc::TResponse)>>(std::move(proto), std::move(cb), database, token); + auto actor = TRpc::CreateRpcActor(req); + return ctx.RegisterWithSameMailbox(actor); +} + +} // namespace NRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/operation_helpers.cpp b/ydb/core/grpc_services/operation_helpers.cpp index 2b877a28d1..66e95a319a 100644 --- a/ydb/core/grpc_services/operation_helpers.cpp +++ b/ydb/core/grpc_services/operation_helpers.cpp @@ -1,294 +1,294 @@ -#include "operation_helpers.h" -#include "rpc_calls.h" - -#include "rpc_export_base.h" -#include "rpc_import_base.h" - +#include "operation_helpers.h" +#include "rpc_calls.h" + +#include "rpc_export_base.h" +#include "rpc_import_base.h" + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - + #include <ydb/core/tx/schemeshard/schemeshard_build_index.h> #include <ydb/core/tx/schemeshard/schemeshard_export.h> #include <ydb/core/tx/schemeshard/schemeshard_import.h> - + #include <ydb/core/protos/index_builder.pb.h> - + #include <ydb/public/lib/operation_id/protos/operation_id.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> - -namespace NKikimr { -namespace NGRpcService { - -using std::unique_ptr; - -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) - -IEventBase* CreateNavigateForPath(const TString& path) { - auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - - request->DatabaseName = path; - - auto& entry = request->ResultSet.emplace_back(); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - entry.Path = ::NKikimr::SplitPath(path); + +#include <library/cpp/actors/core/actor_bootstrapped.h> + +namespace NKikimr { +namespace NGRpcService { + +using std::unique_ptr; + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) + +IEventBase* CreateNavigateForPath(const TString& path) { + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + + request->DatabaseName = path; + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = ::NKikimr::SplitPath(path); entry.RedirectRequired = false; - - return new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()); -} - + + return new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()); +} + TActorId CreatePipeClient(ui64 id, const TActorContext& ctx) { - NTabletPipe::TClientConfig clientConfig; + NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = {.RetryLimitCount = 3}; - return ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, id, clientConfig)); -} - -Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) { - Ydb::TOperationId operationId; - operationId.SetKind(Ydb::TOperationId::BUILD_INDEX); - NOperationId::AddOptionalValue(operationId, "id", ToString(build.GetId())); - - return operationId; -} - -void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) { - operation->set_id(NOperationId::ProtoToString(ToOperationId(build))); - operation->mutable_issues()->CopyFrom(build.GetIssues()); - - switch (build.GetState()) { - case Ydb::Table::IndexBuildState::STATE_DONE: - operation->set_ready(true); - operation->set_status(Ydb::StatusIds::SUCCESS); - break; - case Ydb::Table::IndexBuildState::STATE_CANCELLED: - operation->set_ready(true); - operation->set_status(Ydb::StatusIds::CANCELLED); - break; - case Ydb::Table::IndexBuildState::STATE_REJECTED: - operation->set_ready(true); - operation->set_status(Ydb::StatusIds::ABORTED); - break; - default: - operation->set_ready(false); - } - - Ydb::Table::IndexBuildMetadata metadata; - metadata.set_state(build.GetState()); - metadata.set_progress(build.GetProgress()); - auto desc = metadata.mutable_description(); - desc->set_path(build.GetSettings().source_path()); - desc->mutable_index()->CopyFrom(build.GetSettings().index()); - - auto data = operation->mutable_metadata(); - data->PackFrom(metadata); -} - -bool TryGetId(const NOperationId::TOperationId& operationId, ui64& id) { - const auto& ids = operationId.GetValue("id"); - - if (ids.size() != 1) { - return false; - } - - if (!TryFromString(*ids[0], id)) { - return false; - } - - return id; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class TSSOpSubscriber : public TActorBootstrapped<TSSOpSubscriber> { -public: - TSSOpSubscriber(ui64 schemeshardId, ui64 txId, TString dbName, TOpType opType, unique_ptr<IRequestOpCtx>&& op) - : SchemeshardId(schemeshardId) - , TxId(txId) - , DatabaseName(dbName) - , OpType(opType) - , Req(std::move(op)) - { - StateFunc = &TSSOpSubscriber::DoSubscribe; - } - - void Bootstrap(const TActorContext &ctx) { - SSPipeClient = CreatePipeClient(SchemeshardId, ctx); - - LogPrefix = TStringBuilder() << "[SSOpSubscriber " << SelfId() << "] "; - - (this->*StateFunc)(ctx); - - Become(&TSSOpSubscriber::AwaitState); - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr&, const TActorContext&) {} - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&, const TActorContext& ctx) { - if (SSPipeClient) { - NTabletPipe::CloseClient(ctx, SSPipeClient); - SSPipeClient = TActorId(); - } - - LOG_E("Handle TEvTabletPipe::TEvClientDestroyed"); - if (AttemptsCounter > 3u) { - TString error = "Too many attempts to create pipe to SS."; - LOG_E(error); - Req->RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::YDB_DB_NOT_READY, error)); - Req->ReplyWithYdbStatus(Ydb::StatusIds::OVERLOADED); - Die(ctx); - return; - } - - NTabletPipe::TClientConfig config; + return ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, id, clientConfig)); +} + +Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) { + Ydb::TOperationId operationId; + operationId.SetKind(Ydb::TOperationId::BUILD_INDEX); + NOperationId::AddOptionalValue(operationId, "id", ToString(build.GetId())); + + return operationId; +} + +void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) { + operation->set_id(NOperationId::ProtoToString(ToOperationId(build))); + operation->mutable_issues()->CopyFrom(build.GetIssues()); + + switch (build.GetState()) { + case Ydb::Table::IndexBuildState::STATE_DONE: + operation->set_ready(true); + operation->set_status(Ydb::StatusIds::SUCCESS); + break; + case Ydb::Table::IndexBuildState::STATE_CANCELLED: + operation->set_ready(true); + operation->set_status(Ydb::StatusIds::CANCELLED); + break; + case Ydb::Table::IndexBuildState::STATE_REJECTED: + operation->set_ready(true); + operation->set_status(Ydb::StatusIds::ABORTED); + break; + default: + operation->set_ready(false); + } + + Ydb::Table::IndexBuildMetadata metadata; + metadata.set_state(build.GetState()); + metadata.set_progress(build.GetProgress()); + auto desc = metadata.mutable_description(); + desc->set_path(build.GetSettings().source_path()); + desc->mutable_index()->CopyFrom(build.GetSettings().index()); + + auto data = operation->mutable_metadata(); + data->PackFrom(metadata); +} + +bool TryGetId(const NOperationId::TOperationId& operationId, ui64& id) { + const auto& ids = operationId.GetValue("id"); + + if (ids.size() != 1) { + return false; + } + + if (!TryFromString(*ids[0], id)) { + return false; + } + + return id; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class TSSOpSubscriber : public TActorBootstrapped<TSSOpSubscriber> { +public: + TSSOpSubscriber(ui64 schemeshardId, ui64 txId, TString dbName, TOpType opType, unique_ptr<IRequestOpCtx>&& op) + : SchemeshardId(schemeshardId) + , TxId(txId) + , DatabaseName(dbName) + , OpType(opType) + , Req(std::move(op)) + { + StateFunc = &TSSOpSubscriber::DoSubscribe; + } + + void Bootstrap(const TActorContext &ctx) { + SSPipeClient = CreatePipeClient(SchemeshardId, ctx); + + LogPrefix = TStringBuilder() << "[SSOpSubscriber " << SelfId() << "] "; + + (this->*StateFunc)(ctx); + + Become(&TSSOpSubscriber::AwaitState); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr&, const TActorContext&) {} + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&, const TActorContext& ctx) { + if (SSPipeClient) { + NTabletPipe::CloseClient(ctx, SSPipeClient); + SSPipeClient = TActorId(); + } + + LOG_E("Handle TEvTabletPipe::TEvClientDestroyed"); + if (AttemptsCounter > 3u) { + TString error = "Too many attempts to create pipe to SS."; + LOG_E(error); + Req->RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::YDB_DB_NOT_READY, error)); + Req->ReplyWithYdbStatus(Ydb::StatusIds::OVERLOADED); + Die(ctx); + return; + } + + NTabletPipe::TClientConfig config; config.RetryPolicy = { .RetryLimitCount = 5, .MinRetryTime = TDuration::MilliSeconds(50), .MaxRetryTime = TDuration::Seconds(10), .DoFirstRetryInstantly = false }; - SSPipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, SchemeshardId, config)); - - AttemptsCounter++; - (this->*StateFunc)(ctx); - } - + SSPipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, SchemeshardId, config)); + + AttemptsCounter++; + (this->*StateFunc)(ctx); + } + void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr&, const TActorContext&) {} - + void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr&, const TActorContext& ctx) { - //TODO: Change SS API to get operation status just from TEvNotifyTxCompletionResult - switch (OpType) { - case TOpType::Common: - NTabletPipe::CloseClient(ctx, SSPipeClient); - Req->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); - Die(ctx); - return; - case TOpType::BuildIndex: - StateFunc = &TSSOpSubscriber::GetBuildIndexStatus; - break; - case TOpType::Export: - StateFunc = &TSSOpSubscriber::GetExportStatus; - break; - case TOpType::Import: - StateFunc = &TSSOpSubscriber::GetImportStatus; - break; - default: - NTabletPipe::CloseClient(ctx, SSPipeClient); - Req->ReplyWithYdbStatus(Ydb::StatusIds::INTERNAL_ERROR); - Die(ctx); - return; - } - (this->*StateFunc)(ctx); - } - - STFUNC(AwaitState) { - switch (ev->GetTypeRewrite()) { + //TODO: Change SS API to get operation status just from TEvNotifyTxCompletionResult + switch (OpType) { + case TOpType::Common: + NTabletPipe::CloseClient(ctx, SSPipeClient); + Req->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); + Die(ctx); + return; + case TOpType::BuildIndex: + StateFunc = &TSSOpSubscriber::GetBuildIndexStatus; + break; + case TOpType::Export: + StateFunc = &TSSOpSubscriber::GetExportStatus; + break; + case TOpType::Import: + StateFunc = &TSSOpSubscriber::GetImportStatus; + break; + default: + NTabletPipe::CloseClient(ctx, SSPipeClient); + Req->ReplyWithYdbStatus(Ydb::StatusIds::INTERNAL_ERROR); + Die(ctx); + return; + } + (this->*StateFunc)(ctx); + } + + STFUNC(AwaitState) { + switch (ev->GetTypeRewrite()) { HFunc(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult, Handle); HFunc(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(NSchemeShard::TEvExport::TEvGetExportResponse, Handle); HFunc(NSchemeShard::TEvImport::TEvGetImportResponse, Handle); HFunc(NSchemeShard::TEvIndexBuilder::TEvGetResponse, Handle); - default: - { - Req->ReplyWithYdbStatus(Ydb::StatusIds::INTERNAL_ERROR); - Die(ctx); - } - } - } - -private: - void PassAway() override { - if (SSPipeClient) { - NTabletPipe::CloseClient(SelfId(), SSPipeClient); - SSPipeClient = TActorId(); - } - IActor::PassAway(); - } - - void DoSubscribe(const TActorContext& ctx) { + default: + { + Req->ReplyWithYdbStatus(Ydb::StatusIds::INTERNAL_ERROR); + Die(ctx); + } + } + } + +private: + void PassAway() override { + if (SSPipeClient) { + NTabletPipe::CloseClient(SelfId(), SSPipeClient); + SSPipeClient = TActorId(); + } + IActor::PassAway(); + } + + void DoSubscribe(const TActorContext& ctx) { auto request = MakeHolder<NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion>(); - request->Record.SetTxId(TxId); - NTabletPipe::SendData(ctx, SSPipeClient, request.Release()); - } - - void GetBuildIndexStatus(const TActorContext& ctx) { + request->Record.SetTxId(TxId); + NTabletPipe::SendData(ctx, SSPipeClient, request.Release()); + } + + void GetBuildIndexStatus(const TActorContext& ctx) { auto request = new NSchemeShard::TEvIndexBuilder::TEvGetRequest(DatabaseName, TxId); - NTabletPipe::SendData(ctx, SSPipeClient, request); - } - - void GetExportStatus(const TActorContext& ctx) { + NTabletPipe::SendData(ctx, SSPipeClient, request); + } + + void GetExportStatus(const TActorContext& ctx) { auto request = new NSchemeShard::TEvExport::TEvGetExportRequest(DatabaseName, TxId); - NTabletPipe::SendData(ctx, SSPipeClient, request); - } - - void GetImportStatus(const TActorContext& ctx) { + NTabletPipe::SendData(ctx, SSPipeClient, request); + } + + void GetImportStatus(const TActorContext& ctx) { auto request = new NSchemeShard::TEvImport::TEvGetImportRequest(DatabaseName, TxId); - NTabletPipe::SendData(ctx, SSPipeClient, request); - } - + NTabletPipe::SendData(ctx, SSPipeClient, request); + } + void Handle(NSchemeShard::TEvExport::TEvGetExportResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvGetExportResponse" - << ": record# " << record.ShortDebugString()); - - auto op = TExportConv::ToOperation(record.GetEntry()); - Req->SendOperation(op); - Die(ctx); - } - + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvGetExportResponse" + << ": record# " << record.ShortDebugString()); + + auto op = TExportConv::ToOperation(record.GetEntry()); + Req->SendOperation(op); + Die(ctx); + } + void Handle(NSchemeShard::TEvImport::TEvGetImportResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvImport::TEvGetImportResponse" - << ": record# " << record.ShortDebugString()); - - auto op = TImportConv::ToOperation(record.GetEntry()); - Req->SendOperation(op); - Die(ctx); - } - + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvImport::TEvGetImportResponse" + << ": record# " << record.ShortDebugString()); + + auto op = TImportConv::ToOperation(record.GetEntry()); + Req->SendOperation(op); + Die(ctx); + } + void Handle(NSchemeShard::TEvIndexBuilder::TEvGetResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - - LOG_D("Handle TEvIndexBuilder::TEvGetResponse" - << ": record# " << record.ShortDebugString()); - - if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { - Req->ReplyWithYdbStatus(record.GetStatus()); - } else { - Ydb::Operations::Operation op; - ::NKikimr::NGRpcService::ToOperation(record.GetIndexBuild(), &op); - Req->SendOperation(op); - } - Die(ctx); - } - -private: - const ui64 SchemeshardId; - const ui64 TxId; - const TString DatabaseName; - const TOpType OpType; - unique_ptr<IRequestOpCtx> Req; - using TStateFunc = void (TSSOpSubscriber::*)(const TActorContext& ctx); - - TActorId SSPipeClient; - ui64 AttemptsCounter = 0; - TStateFunc StateFunc; - TString LogPrefix; -}; - -void CreateSSOpSubscriber(ui64 schemeshardId, ui64 txId, const TString& dbName, TOpType opType, unique_ptr<IRequestOpCtx>&& op, const TActorContext& ctx) { - ctx.Register(new TSSOpSubscriber(schemeshardId, txId, dbName, opType, std::move(op))); -} - -} // namespace NGRpcService -} // namespace NKikimr + const auto& record = ev->Get()->Record; + + LOG_D("Handle TEvIndexBuilder::TEvGetResponse" + << ": record# " << record.ShortDebugString()); + + if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { + Req->ReplyWithYdbStatus(record.GetStatus()); + } else { + Ydb::Operations::Operation op; + ::NKikimr::NGRpcService::ToOperation(record.GetIndexBuild(), &op); + Req->SendOperation(op); + } + Die(ctx); + } + +private: + const ui64 SchemeshardId; + const ui64 TxId; + const TString DatabaseName; + const TOpType OpType; + unique_ptr<IRequestOpCtx> Req; + using TStateFunc = void (TSSOpSubscriber::*)(const TActorContext& ctx); + + TActorId SSPipeClient; + ui64 AttemptsCounter = 0; + TStateFunc StateFunc; + TString LogPrefix; +}; + +void CreateSSOpSubscriber(ui64 schemeshardId, ui64 txId, const TString& dbName, TOpType opType, unique_ptr<IRequestOpCtx>&& op, const TActorContext& ctx) { + ctx.Register(new TSSOpSubscriber(schemeshardId, txId, dbName, opType, std::move(op))); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/operation_helpers.h b/ydb/core/grpc_services/operation_helpers.h index c78515612b..7e46b65887 100644 --- a/ydb/core/grpc_services/operation_helpers.h +++ b/ydb/core/grpc_services/operation_helpers.h @@ -1,36 +1,36 @@ -#pragma once - -#include "defs.h" +#pragma once + +#include "defs.h" #include <ydb/public/lib/operation_id/operation_id.h> - -namespace NKikimrIndexBuilder { - class TIndexBuild; -} - -namespace Ydb { -namespace Operations { - class Operation; -} -} - -namespace NKikimr { -namespace NGRpcService { - -class IRequestOpCtx; - -IEventBase* CreateNavigateForPath(const TString& path); + +namespace NKikimrIndexBuilder { + class TIndexBuild; +} + +namespace Ydb { +namespace Operations { + class Operation; +} +} + +namespace NKikimr { +namespace NGRpcService { + +class IRequestOpCtx; + +IEventBase* CreateNavigateForPath(const TString& path); TActorId CreatePipeClient(ui64 id, const TActorContext& ctx); -Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build); -void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation); -bool TryGetId(const NOperationId::TOperationId& operationId, ui64& id); - -enum class TOpType { - Common, - BuildIndex, - Export, - Import -}; -void CreateSSOpSubscriber(ui64 schemeshardId, ui64 txId, const TString& dbName, TOpType opType, std::unique_ptr<IRequestOpCtx>&& op, const TActorContext& ctx); - -} // namespace NGRpcService -} // namespace NKikimr +Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build); +void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation); +bool TryGetId(const NOperationId::TOperationId& operationId, ui64& id); + +enum class TOpType { + Common, + BuildIndex, + Export, + Import +}; +void CreateSSOpSubscriber(ui64 schemeshardId, ui64 txId, const TString& dbName, TOpType opType, std::unique_ptr<IRequestOpCtx>&& op, const TActorContext& ctx); + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/operation_helpers_ut.cpp b/ydb/core/grpc_services/operation_helpers_ut.cpp index 8bf0ee2280..604067b330 100644 --- a/ydb/core/grpc_services/operation_helpers_ut.cpp +++ b/ydb/core/grpc_services/operation_helpers_ut.cpp @@ -1,92 +1,92 @@ -#include "operation_helpers.h" - +#include "operation_helpers.h" + #include <google/protobuf/text_format.h> - -#include <library/cpp/testing/unittest/tests_data.h> -#include <library/cpp/testing/unittest/registar.h> - + +#include <library/cpp/testing/unittest/tests_data.h> +#include <library/cpp/testing/unittest/registar.h> + #include <ydb/core/protos/index_builder.pb.h> #include <ydb/public/api/protos/ydb_operation.pb.h> - -namespace NKikimr { - -Y_UNIT_TEST_SUITE(OperationMapping) { - Y_UNIT_TEST(IndexBuildCanceled) { - TString indexBuild = R"___( - State: STATE_CANCELLED - Settings { - source_path: "/MyRoot/Table" - index { - name: "index1" - index_columns: "index" - global_index { - } - } - } - Progress: 0 -)___"; - - NKikimrIndexBuilder::TIndexBuild buildProto; - google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); - - Ydb::Operations::Operation operation; - - NGRpcService::ToOperation(buildProto, &operation); - - UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); - UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::CANCELLED); - } - Y_UNIT_TEST(IndexBuildSuccess) { - TString indexBuild = R"___( - State: STATE_DONE - Settings { - source_path: "/MyRoot/Table" - index { - name: "index1" - index_columns: "index" - global_index { - } - } - } - Progress: 0 -)___"; - - NKikimrIndexBuilder::TIndexBuild buildProto; - google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); - - Ydb::Operations::Operation operation; - - NGRpcService::ToOperation(buildProto, &operation); - - UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); - UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::SUCCESS); - } - Y_UNIT_TEST(IndexBuildRejected) { - TString indexBuild = R"___( - State: STATE_REJECTED - Settings { - source_path: "/MyRoot/Table" - index { - name: "index1" - index_columns: "index" - global_index { - } - } - } - Progress: 0 -)___"; - - NKikimrIndexBuilder::TIndexBuild buildProto; - google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); - - Ydb::Operations::Operation operation; - - NGRpcService::ToOperation(buildProto, &operation); - - UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); - UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::ABORTED); - } -} - -} - + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(OperationMapping) { + Y_UNIT_TEST(IndexBuildCanceled) { + TString indexBuild = R"___( + State: STATE_CANCELLED + Settings { + source_path: "/MyRoot/Table" + index { + name: "index1" + index_columns: "index" + global_index { + } + } + } + Progress: 0 +)___"; + + NKikimrIndexBuilder::TIndexBuild buildProto; + google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); + + Ydb::Operations::Operation operation; + + NGRpcService::ToOperation(buildProto, &operation); + + UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); + UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::CANCELLED); + } + Y_UNIT_TEST(IndexBuildSuccess) { + TString indexBuild = R"___( + State: STATE_DONE + Settings { + source_path: "/MyRoot/Table" + index { + name: "index1" + index_columns: "index" + global_index { + } + } + } + Progress: 0 +)___"; + + NKikimrIndexBuilder::TIndexBuild buildProto; + google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); + + Ydb::Operations::Operation operation; + + NGRpcService::ToOperation(buildProto, &operation); + + UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); + UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::SUCCESS); + } + Y_UNIT_TEST(IndexBuildRejected) { + TString indexBuild = R"___( + State: STATE_REJECTED + Settings { + source_path: "/MyRoot/Table" + index { + name: "index1" + index_columns: "index" + global_index { + } + } + } + Progress: 0 +)___"; + + NKikimrIndexBuilder::TIndexBuild buildProto; + google::protobuf::TextFormat::ParseFromString(indexBuild, &buildProto); + + Ydb::Operations::Operation operation; + + NGRpcService::ToOperation(buildProto, &operation); + + UNIT_ASSERT_VALUES_EQUAL(operation.ready(), true); + UNIT_ASSERT_VALUES_EQUAL(operation.status(), Ydb::StatusIds::ABORTED); + } +} + +} + diff --git a/ydb/core/grpc_services/rpc_alter_coordination_node.cpp b/ydb/core/grpc_services/rpc_alter_coordination_node.cpp index 03e95c6a63..52cfe8c940 100644 --- a/ydb/core/grpc_services/rpc_alter_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_alter_coordination_node.cpp @@ -26,12 +26,12 @@ public: private: void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { pathPair = SplitPath(req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); } @@ -52,7 +52,7 @@ private: } void ReplyWithResult(StatusIds::StatusCode status, const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); + Request_->ReplyWithYdbStatus(status); Die(ctx); } }; diff --git a/ydb/core/grpc_services/rpc_alter_table.cpp b/ydb/core/grpc_services/rpc_alter_table.cpp index f49b94cff0..d1f2755b66 100644 --- a/ydb/core/grpc_services/rpc_alter_table.cpp +++ b/ydb/core/grpc_services/rpc_alter_table.cpp @@ -1,10 +1,10 @@ -#include "grpc_request_proxy.h" - +#include "grpc_request_proxy.h" + #include "rpc_scheme_base.h" -#include "rpc_common.h" -#include "operation_helpers.h" +#include "rpc_common.h" +#include "operation_helpers.h" #include "table_settings.h" - + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/tx/schemeshard/schemeshard_build_index.h> @@ -14,53 +14,53 @@ #include <util/generic/hash_set.h> -#define TXLOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define TXLOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define TXLOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define TXLOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define TXLOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) -#define TXLOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; +#define TXLOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define TXLOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define TXLOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define TXLOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define TXLOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) +#define TXLOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NConsole; using namespace Ydb; - -static bool CheckAccess(const NACLib::TUserToken& userToken, const NSchemeCache::TSchemeCacheNavigate* navigate) { - bool isDatabase = true; // first entry is always database - - using TEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; - - for (const TEntry& entry : navigate->ResultSet) { - if (!entry.SecurityObject) { - continue; - } - - const ui32 access = isDatabase ? NACLib::CreateDirectory | NACLib::CreateTable : NACLib::GenericRead | NACLib::GenericWrite; - if (!entry.SecurityObject->CheckAccess(access, userToken)) { - return false; - } - - isDatabase = false; - } - - return true; -} - + +static bool CheckAccess(const NACLib::TUserToken& userToken, const NSchemeCache::TSchemeCacheNavigate* navigate) { + bool isDatabase = true; // first entry is always database + + using TEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; + + for (const TEntry& entry : navigate->ResultSet) { + if (!entry.SecurityObject) { + continue; + } + + const ui32 access = isDatabase ? NACLib::CreateDirectory | NACLib::CreateTable : NACLib::GenericRead | NACLib::GenericWrite; + if (!entry.SecurityObject->CheckAccess(access, userToken)) { + return false; + } + + isDatabase = false; + } + + return true; +} + static std::pair<StatusIds::StatusCode, TString> CheckAddIndexDesc(const Ydb::Table::TableIndex& desc) { if (!desc.name()) { return {StatusIds::BAD_REQUEST, "Index must have a name"}; } - if (!desc.index_columns_size()) { + if (!desc.index_columns_size()) { return {StatusIds::BAD_REQUEST, "At least one column must be specified"}; - } + } if (!desc.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { return {StatusIds::UNSUPPORTED, "Data column feature is not supported yet"}; - } + } switch (desc.type_case()) { case Table::TableIndex::kGlobalIndex: @@ -75,32 +75,32 @@ static std::pair<StatusIds::StatusCode, TString> CheckAddIndexDesc(const Ydb::Ta } return {StatusIds::SUCCESS, ""}; -} - +} + class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTableRequest> { using TBase = TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTableRequest>; - void PassAway() override { - if (SSPipeClient) { - NTabletPipe::CloseClient(SelfId(), SSPipeClient); + void PassAway() override { + if (SSPipeClient) { + NTabletPipe::CloseClient(SelfId(), SSPipeClient); SSPipeClient = TActorId(); - } - IActor::PassAway(); - } + } + IActor::PassAway(); + } enum class EOp { // columns, column families, storage, ttl Common, - // add indices - AddIndex, - // drop indices - DropIndex, + // add indices + AddIndex, + // drop indices + DropIndex, // add/alter/drop attributes Attribute, }; THashSet<EOp> GetOps() const { - const auto& req = GetProtoRequest(); + const auto& req = GetProtoRequest(); THashSet<EOp> ops; if (req->add_columns_size() || req->drop_columns_size() || req->alter_columns_size() @@ -113,14 +113,14 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab ops.emplace(EOp::Common); } - if (req->add_indexes_size()) { - ops.emplace(EOp::AddIndex); + if (req->add_indexes_size()) { + ops.emplace(EOp::AddIndex); + } + + if (req->drop_indexes_size()) { + ops.emplace(EOp::DropIndex); } - if (req->drop_indexes_size()) { - ops.emplace(EOp::DropIndex); - } - if (req->alter_attributes_size()) { ops.emplace(EOp::Attribute); } @@ -128,23 +128,23 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab return ops; } -public: - TAlterTableRPC(IRequestOpCtx* msg) +public: + TAlterTableRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - const auto& req = GetProtoRequest(); - if (!Request_->GetInternalToken().empty()) { + const auto& req = GetProtoRequest(); + if (!Request_->GetInternalToken().empty()) { UserToken = MakeHolder<NACLib::TUserToken>(Request_->GetInternalToken()); - } + } auto ops = GetOps(); if (!ops) { return Reply(StatusIds::BAD_REQUEST, "Empty alter", NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } + } if (ops.size() != 1) { return Reply(StatusIds::UNSUPPORTED, "Mixed alter is unsupported", NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); @@ -158,48 +158,48 @@ public: Become(&TAlterTableRPC::AlterStateGetConfig); return; - case EOp::AddIndex: - if (req->add_indexes_size() == 1) { - const auto& index = req->add_indexes(0); + case EOp::AddIndex: + if (req->add_indexes_size() == 1) { + const auto& index = req->add_indexes(0); auto [status, issues] = CheckAddIndexDesc(index); if (status == StatusIds::SUCCESS) { - PrepareAlterTableAddIndex(); - } else { + PrepareAlterTableAddIndex(); + } else { return Reply(status, issues, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } + } } else { - return Reply(StatusIds::UNSUPPORTED, "Only one index can be added by one operation", + return Reply(StatusIds::UNSUPPORTED, "Only one index can be added by one operation", + NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); + } + break; + + case EOp::DropIndex: + if (req->drop_indexes_size() == 1) { + DropIndex(ctx); + } else { + return Reply(StatusIds::UNSUPPORTED, "Only one index can be removed by one operation", NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); } break; - case EOp::DropIndex: - if (req->drop_indexes_size() == 1) { - DropIndex(ctx); - } else { - return Reply(StatusIds::UNSUPPORTED, "Only one index can be removed by one operation", - NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } - break; - case EOp::Attribute: AlterUserAttributes(ctx); break; } - Become(&TAlterTableRPC::AlterStateWork); - } - -private: - void AlterStateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); - HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + Become(&TAlterTableRPC::AlterStateWork); + } + +private: + void AlterStateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); HFunc(NSchemeShard::TEvIndexBuilder::TEvCreateResponse, Handle); - default: TBase::StateWork(ev, ctx); - } - } - + default: TBase::StateWork(ev, ctx); + } + } + void AlterStateGetConfig(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { HFunc(TEvConfigsDispatcher::TEvGetConfigResponse, Handle); @@ -246,191 +246,191 @@ private: IEventHandle::FlagTrackDelivery); } - void PrepareAlterTableAddIndex() { - using namespace NTxProxy; - LogPrefix = TStringBuilder() << "[AlterTableAddIndexOp " << SelfId() << "] "; - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); - } - - void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev, const TActorContext& ctx) { - TXLOG_D("Handle TEvTxUserProxy::TEvAllocateTxIdResult"); - - const auto* msg = ev->Get(); - TxId = msg->TxId; - SchemeCache = msg->Services.SchemeCache; - TxProxyMon = msg->TxProxyMon; - LogPrefix = TStringBuilder() << "[AlterTableAddIndex " << SelfId() << " TxId# " << TxId << "] "; - - AlterTableAddIndexOp(ctx); - } - - void AlterTableAddIndexOp(const TActorContext& ctx) { - using namespace NTxProxy; - DatabaseName = Request_->GetDatabaseName() - .GetOrElse(DatabaseFromDomain(AppData())); - - const auto& path = GetProtoRequest()->path(); - - const auto paths = NKikimr::SplitPath(path); - if (paths.empty()) { - TString error = TStringBuilder() << "Failed to split table path " << path; - Request_->RaiseIssue(NYql::TIssue(error)); - return Reply(Ydb::StatusIds::BAD_REQUEST, ctx); - } - - auto ev = CreateNavigateForPath(DatabaseName); - { - auto& entry = static_cast<TEvTxProxySchemeCache::TEvNavigateKeySet*>(ev)->Request->ResultSet.emplace_back(); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.Path = paths; - } - - Send(SchemeCache, ev); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - TXLOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ", errors# " << ev->Get()->Request.Get()->ErrorCount); - - NSchemeCache::TSchemeCacheNavigate* resp = ev->Get()->Request.Get(); - - if (resp->ErrorCount > 0 || resp->ResultSet.empty()) { - TStringBuilder builder; - builder << "Unable to navigate:"; - - for (const auto& entry : resp->ResultSet) { - if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - builder << " " << JoinPath(entry.Path) << " status: " << entry.Status; - } - } - - TString error(builder); - TXLOG_E(error); - Request_->RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, error)); - return Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } - - if (UserToken && !CheckAccess(*UserToken, resp)) { - TXLOG_W("Access check failed"); - return Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); - } - - auto domainInfo = resp->ResultSet.front().DomainInfo; - if (!domainInfo) { - TXLOG_E("Got empty domain info"); - return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); - } - - SchemeshardId = domainInfo->ExtractSchemeShard(); - - SSPipeClient = CreatePipeClient(SchemeshardId, ctx); - SendAddIndexOpToSS(ctx); - } - - void SendAddIndexOpToSS(const TActorContext& ctx) { - const auto& req = *GetProtoRequest(); - + void PrepareAlterTableAddIndex() { + using namespace NTxProxy; + LogPrefix = TStringBuilder() << "[AlterTableAddIndexOp " << SelfId() << "] "; + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); + } + + void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev, const TActorContext& ctx) { + TXLOG_D("Handle TEvTxUserProxy::TEvAllocateTxIdResult"); + + const auto* msg = ev->Get(); + TxId = msg->TxId; + SchemeCache = msg->Services.SchemeCache; + TxProxyMon = msg->TxProxyMon; + LogPrefix = TStringBuilder() << "[AlterTableAddIndex " << SelfId() << " TxId# " << TxId << "] "; + + AlterTableAddIndexOp(ctx); + } + + void AlterTableAddIndexOp(const TActorContext& ctx) { + using namespace NTxProxy; + DatabaseName = Request_->GetDatabaseName() + .GetOrElse(DatabaseFromDomain(AppData())); + + const auto& path = GetProtoRequest()->path(); + + const auto paths = NKikimr::SplitPath(path); + if (paths.empty()) { + TString error = TStringBuilder() << "Failed to split table path " << path; + Request_->RaiseIssue(NYql::TIssue(error)); + return Reply(Ydb::StatusIds::BAD_REQUEST, ctx); + } + + auto ev = CreateNavigateForPath(DatabaseName); + { + auto& entry = static_cast<TEvTxProxySchemeCache::TEvNavigateKeySet*>(ev)->Request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; + entry.Path = paths; + } + + Send(SchemeCache, ev); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + TXLOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ", errors# " << ev->Get()->Request.Get()->ErrorCount); + + NSchemeCache::TSchemeCacheNavigate* resp = ev->Get()->Request.Get(); + + if (resp->ErrorCount > 0 || resp->ResultSet.empty()) { + TStringBuilder builder; + builder << "Unable to navigate:"; + + for (const auto& entry : resp->ResultSet) { + if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { + builder << " " << JoinPath(entry.Path) << " status: " << entry.Status; + } + } + + TString error(builder); + TXLOG_E(error); + Request_->RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, error)); + return Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + } + + if (UserToken && !CheckAccess(*UserToken, resp)) { + TXLOG_W("Access check failed"); + return Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); + } + + auto domainInfo = resp->ResultSet.front().DomainInfo; + if (!domainInfo) { + TXLOG_E("Got empty domain info"); + return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + } + + SchemeshardId = domainInfo->ExtractSchemeShard(); + + SSPipeClient = CreatePipeClient(SchemeshardId, ctx); + SendAddIndexOpToSS(ctx); + } + + void SendAddIndexOpToSS(const TActorContext& ctx) { + const auto& req = *GetProtoRequest(); + NKikimrIndexBuilder::TIndexBuildSettings settings; settings.set_source_path(req.path()); auto tableIndex = settings.mutable_index(); - tableIndex->CopyFrom(req.add_indexes(0)); + tableIndex->CopyFrom(req.add_indexes(0)); auto ev = new NSchemeShard::TEvIndexBuilder::TEvCreateRequest(TxId, DatabaseName, std::move(settings)); - - NTabletPipe::SendData(ctx, SSPipeClient, ev); - } - + + NTabletPipe::SendData(ctx, SSPipeClient, ev); + } + void Handle(NSchemeShard::TEvIndexBuilder::TEvCreateResponse::TPtr& ev, const TActorContext& ctx) { const auto& response = ev->Get()->Record; - const auto status = response.GetStatus(); - auto issuesProto = response.GetIssues(); - - auto getDebugIssues = [issuesProto]() { - NYql::TIssues issues; - NYql::IssuesFromMessage(issuesProto, issues); - return issues.ToString(); - }; - - TXLOG_D("Handle TEvIndexBuilder::TEvCreateResponse" - << ", status# " << status - << ", issues# " << getDebugIssues() - << ", Id# " << response.GetIndexBuild().GetId()); - - if (status == Ydb::StatusIds::SUCCESS) { - if (GetOperationMode() == Ydb::Operations::OperationParams::SYNC) { - CreateSSOpSubscriber(SchemeshardId, TxId, DatabaseName, TOpType::BuildIndex, std::move(Request_), ctx); - } else { - auto op = response.GetIndexBuild(); - Ydb::Operations::Operation operation; - operation.set_id(NOperationId::ProtoToString(ToOperationId(op))); - operation.set_ready(false); - ReplyOperation(operation); - } - } else { - Reply(status, issuesProto, ctx); - } - } - - void DropIndex(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { - pathPair = SplitPath(req->path()); - } catch (const std::exception&) { - return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + const auto status = response.GetStatus(); + auto issuesProto = response.GetIssues(); + + auto getDebugIssues = [issuesProto]() { + NYql::TIssues issues; + NYql::IssuesFromMessage(issuesProto, issues); + return issues.ToString(); + }; + + TXLOG_D("Handle TEvIndexBuilder::TEvCreateResponse" + << ", status# " << status + << ", issues# " << getDebugIssues() + << ", Id# " << response.GetIndexBuild().GetId()); + + if (status == Ydb::StatusIds::SUCCESS) { + if (GetOperationMode() == Ydb::Operations::OperationParams::SYNC) { + CreateSSOpSubscriber(SchemeshardId, TxId, DatabaseName, TOpType::BuildIndex, std::move(Request_), ctx); + } else { + auto op = response.GetIndexBuild(); + Ydb::Operations::Operation operation; + operation.set_id(NOperationId::ProtoToString(ToOperationId(op))); + operation.set_ready(false); + ReplyOperation(operation); + } + } else { + Reply(status, issuesProto, ctx); + } + } + + void DropIndex(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { + pathPair = SplitPath(req->path()); + } catch (const std::exception&) { + return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropIndex); - - for (const auto& drop : req->drop_indexes()) { - auto desc = modifyScheme->MutableDropIndex(); - desc->SetIndexName(drop); - desc->SetTableName(name); - } - - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - + + for (const auto& drop : req->drop_indexes()) { + auto desc = modifyScheme->MutableDropIndex(); + desc->SetIndexName(drop); + desc->SetTableName(name); + } + + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + void AlterTable(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { - pathPair = SplitPath(req->path()); - } catch (const std::exception&) { - return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { + pathPair = SplitPath(req->path()); + } catch (const std::exception&) { + return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterTable); - - auto desc = modifyScheme->MutableAlterTable(); - desc->SetName(name); - - for (const auto& drop : req->drop_columns()) { - desc->AddDropColumns()->SetName(drop); - } - - StatusIds::StatusCode code = StatusIds::SUCCESS; - TString error; - - if (!FillColumnDescription(*desc, req->add_columns(), code, error)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); - } - + + auto desc = modifyScheme->MutableAlterTable(); + desc->SetName(name); + + for (const auto& drop : req->drop_columns()) { + desc->AddDropColumns()->SetName(drop); + } + + StatusIds::StatusCode code = StatusIds::SUCCESS; + TString error; + + if (!FillColumnDescription(*desc, req->add_columns(), code, error)) { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } + for (const auto& alter : req->alter_columns()) { auto column = desc->AddColumns(); column->SetName(alter.name()); @@ -484,11 +484,11 @@ private: return Reply(code, issues, ctx); } - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + void AlterUserAttributes(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { @@ -521,31 +521,31 @@ private: ctx.Send(MakeTxProxyID(), proposeRequest.release()); } - void ReplyWithStatus(StatusIds::StatusCode status, - const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); - Die(ctx); - } + void ReplyWithStatus(StatusIds::StatusCode status, + const TActorContext &ctx) { + Request_->ReplyWithYdbStatus(status); + Die(ctx); + } ui64 TxId = 0; ui64 SchemeshardId = 0; TActorId SchemeCache; - TString DatabaseName; - TIntrusivePtr<NTxProxy::TTxProxyMon> TxProxyMon; - TString LogPrefix; + TString DatabaseName; + TIntrusivePtr<NTxProxy::TTxProxyMon> TxProxyMon; + TString LogPrefix; TActorId SSPipeClient; - THolder<const NACLib::TUserToken> UserToken; + THolder<const NACLib::TUserToken> UserToken; TTableProfiles Profiles; -}; - -void TGRpcRequestProxy::Handle(TEvAlterTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TAlterTableRPC(ev->Release().Release())); -} - -template<> -IActor* TEvAlterTableRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { - return new TAlterTableRPC(msg); -} - - -} // namespace NKikimr -} // namespace NGRpcService +}; + +void TGRpcRequestProxy::Handle(TEvAlterTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TAlterTableRPC(ev->Release().Release())); +} + +template<> +IActor* TEvAlterTableRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { + return new TAlterTableRPC(msg); +} + + +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_analytics_internal.cpp b/ydb/core/grpc_services/rpc_analytics_internal.cpp index b0d48975c5..48227a57fd 100644 --- a/ydb/core/grpc_services/rpc_analytics_internal.cpp +++ b/ydb/core/grpc_services/rpc_analytics_internal.cpp @@ -1,4 +1,4 @@ -#include "service_analytics_internal.h" +#include "service_analytics_internal.h" #include "rpc_common.h" #include "rpc_deferrable.h" @@ -9,10 +9,10 @@ #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/protos/draft/yq_private.pb.h> - + namespace NKikimr { namespace NGRpcService { - + using TEvYqPrivatePingTaskRequest = TGrpcRequestOperationCall<Yq::Private::PingTaskRequest, Yq::Private::PingTaskResponse>; using TEvYqPrivateGetTaskRequest = @@ -21,7 +21,7 @@ using TEvYqPrivateWriteTaskResultRequest = TGrpcRequestOperationCall<Yq::Private::WriteTaskResultRequest, Yq::Private::WriteTaskResultResponse>; using TEvYqPrivateNodesHealthCheckRequest = TGrpcRequestOperationCall<Yq::Private::NodesHealthCheckRequest, Yq::Private::NodesHealthCheckResponse>; - + namespace { template <typename TEv, typename TReq> void SendResponse(const TEv& ev, TReq& req) { diff --git a/ydb/core/grpc_services/rpc_begin_transaction.cpp b/ydb/core/grpc_services/rpc_begin_transaction.cpp index 13b9dd089e..9e77fafdd3 100644 --- a/ydb/core/grpc_services/rpc_begin_transaction.cpp +++ b/ydb/core/grpc_services/rpc_begin_transaction.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -18,7 +18,7 @@ class TBeginTransactionRPC : public TRpcKqpRequestActor<TBeginTransactionRPC, TE using TBase = TRpcKqpRequestActor<TBeginTransactionRPC, TEvBeginTransactionRequest>; public: - using TResult = Ydb::Table::BeginTransactionResult; + using TResult = Ydb::Table::BeginTransactionResult; TBeginTransactionRPC(TEvBeginTransactionRequest* msg) : TBase(msg) {} @@ -39,14 +39,14 @@ public: private: void BeginTransactionImpl(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); TString sessionId; auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); @@ -85,14 +85,14 @@ private: void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetRef(); - SetCost(record.GetConsumedRu()); + SetCost(record.GetConsumedRu()); AddServerHintsIfAny(record); - + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - auto beginTxResult = TEvBeginTransactionRequest::AllocateResult<Ydb::Table::BeginTransactionResult>(Request_); + auto beginTxResult = TEvBeginTransactionRequest::AllocateResult<Ydb::Table::BeginTransactionResult>(Request_); if (kqpResponse.HasTxMeta()) { beginTxResult->mutable_tx_meta()->CopyFrom(kqpResponse.GetTxMeta()); } diff --git a/ydb/core/grpc_services/rpc_calls.cpp b/ydb/core/grpc_services/rpc_calls.cpp index 1fb724e155..b94d168147 100644 --- a/ydb/core/grpc_services/rpc_calls.cpp +++ b/ydb/core/grpc_services/rpc_calls.cpp @@ -1,30 +1,30 @@ -#include "rpc_calls.h" -#include "grpc_request_proxy.h" - +#include "rpc_calls.h" +#include "grpc_request_proxy.h" + #include <ydb/core/base/path.h> -namespace NKikimr { -namespace NGRpcService { - +namespace NKikimr { +namespace NGRpcService { + template <> -void FillYdbStatus(Ydb::PersQueue::V1::StreamingWriteServerMessage& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { - resp.set_status(status); - NYql::IssuesToMessage(issues, resp.mutable_issues()); -} - +void FillYdbStatus(Ydb::PersQueue::V1::StreamingWriteServerMessage& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { + resp.set_status(status); + NYql::IssuesToMessage(issues, resp.mutable_issues()); +} + template <> void FillYdbStatus(Ydb::PersQueue::V1::MigrationStreamingReadServerMessage& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { - resp.set_status(status); - NYql::IssuesToMessage(issues, resp.mutable_issues()); -} - + resp.set_status(status); + NYql::IssuesToMessage(issues, resp.mutable_issues()); +} + template <> -void FillYdbStatus(Draft::Dummy::PingResponse& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { - Y_UNUSED(resp); - Y_UNUSED(issues); - Y_UNUSED(status); -} - +void FillYdbStatus(Draft::Dummy::PingResponse& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { + Y_UNUSED(resp); + Y_UNUSED(issues); + Y_UNUSED(status); +} + template <> void FillYdbStatus(Ydb::Coordination::SessionResponse& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status) { auto* failure = resp.mutable_failure(); @@ -43,30 +43,30 @@ std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TSt return pathPair; } -std::pair<TString, TString> SplitPath(const TString& path) { - auto splitPos = path.find_last_of('/'); - if (splitPos == path.npos || splitPos + 1 == path.size()) { +std::pair<TString, TString> SplitPath(const TString& path) { + auto splitPos = path.find_last_of('/'); + if (splitPos == path.npos || splitPos + 1 == path.size()) { ythrow yexception() << "wrong path format '" << path << "'" ; - } - return {path.substr(0, splitPos), path.substr(splitPos + 1)}; -} - + } + return {path.substr(0, splitPos), path.substr(splitPos + 1)}; +} + void RefreshToken(const TString& token, const TString& database, const TActorContext& ctx, TActorId from) { - ctx.Send(CreateGRpcRequestProxyId(), new TRefreshTokenImpl(token, database, from)); -} - -void TRefreshTokenImpl::ReplyUnauthenticated(const TString&) { + ctx.Send(CreateGRpcRequestProxyId(), new TRefreshTokenImpl(token, database, from)); +} + +void TRefreshTokenImpl::ReplyUnauthenticated(const TString&) { TActivationContext::Send(new IEventHandle(From_, TActorId(), - new TGRpcRequestProxy::TEvRefreshTokenResponse - { false, "", false, IssueManager_.GetIssues()})); -} - -void TRefreshTokenImpl::ReplyUnavaliable() { - const TActorContext& ctx = TActivationContext::AsActorContext(); - ctx.Send(From_, - new TGRpcRequestProxy::TEvRefreshTokenResponse - { false, "", true, IssueManager_.GetIssues()}); -} - -} // namespace NGRpcService -} // namespace NKikimr + new TGRpcRequestProxy::TEvRefreshTokenResponse + { false, "", false, IssueManager_.GetIssues()})); +} + +void TRefreshTokenImpl::ReplyUnavaliable() { + const TActorContext& ctx = TActivationContext::AsActorContext(); + ctx.Send(From_, + new TGRpcRequestProxy::TEvRefreshTokenResponse + { false, "", true, IssueManager_.GetIssues()}); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_calls.h b/ydb/core/grpc_services/rpc_calls.h index 8fb9c1afe9..43136c2f4a 100644 --- a/ydb/core/grpc_services/rpc_calls.h +++ b/ydb/core/grpc_services/rpc_calls.h @@ -1,8 +1,8 @@ -#pragma once +#pragma once #include "defs.h" - -#include "local_rate_limiter.h" - + +#include "local_rate_limiter.h" + #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/protos/ydb_auth.pb.h> @@ -21,134 +21,134 @@ #include <ydb/public/api/protos/ydb_rate_limiter.pb.h> #include <ydb/public/api/protos/ydb_monitoring.pb.h> #include <ydb/public/api/protos/yq.pb.h> - + #include <ydb/public/api/grpc/draft/dummy.pb.h> #include <ydb/public/api/grpc/draft/ydb_long_tx_v1.pb.h> #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.pb.h> - + #include <ydb/public/lib/operation_id/operation_id.h> -#include <util/generic/maybe.h> - -namespace NKikimr { -namespace NGRpcService { - +#include <util/generic/maybe.h> + +namespace NKikimr { +namespace NGRpcService { + template <> void FillYdbStatus(Ydb::PersQueue::V1::StreamingWriteServerMessage& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status); - + template <> void FillYdbStatus(Ydb::PersQueue::V1::MigrationStreamingReadServerMessage& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status); - + template <> void FillYdbStatus(Draft::Dummy::PingResponse& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status); - + template <> void FillYdbStatus(Ydb::Coordination::SessionResponse& resp, const NYql::TIssues& issues, Ydb::StatusIds::StatusCode status); - -using TEvAlterTableRequest = TGRpcRequestValidationWrapper<TRpcServices::EvAlterTable, Ydb::Table::AlterTableRequest, Ydb::Table::AlterTableResponse, true, TRateLimiterMode::Rps>; -using TEvCreateTableRequest = TGRpcRequestValidationWrapper<TRpcServices::EvCreateTable, Ydb::Table::CreateTableRequest, Ydb::Table::CreateTableResponse, true, TRateLimiterMode::Rps>; -using TEvDropTableRequest = TGRpcRequestWrapper<TRpcServices::EvDropTable, Ydb::Table::DropTableRequest, Ydb::Table::DropTableResponse, true, TRateLimiterMode::Rps>; -using TEvCopyTableRequest = TGRpcRequestWrapper<TRpcServices::EvCopyTable, Ydb::Table::CopyTableRequest, Ydb::Table::CopyTableResponse, true, TRateLimiterMode::Rps>; -using TEvCopyTablesRequest = TGRpcRequestWrapper<TRpcServices::EvCopyTables, Ydb::Table::CopyTablesRequest, Ydb::Table::CopyTablesResponse, true, TRateLimiterMode::Rps>; + +using TEvAlterTableRequest = TGRpcRequestValidationWrapper<TRpcServices::EvAlterTable, Ydb::Table::AlterTableRequest, Ydb::Table::AlterTableResponse, true, TRateLimiterMode::Rps>; +using TEvCreateTableRequest = TGRpcRequestValidationWrapper<TRpcServices::EvCreateTable, Ydb::Table::CreateTableRequest, Ydb::Table::CreateTableResponse, true, TRateLimiterMode::Rps>; +using TEvDropTableRequest = TGRpcRequestWrapper<TRpcServices::EvDropTable, Ydb::Table::DropTableRequest, Ydb::Table::DropTableResponse, true, TRateLimiterMode::Rps>; +using TEvCopyTableRequest = TGRpcRequestWrapper<TRpcServices::EvCopyTable, Ydb::Table::CopyTableRequest, Ydb::Table::CopyTableResponse, true, TRateLimiterMode::Rps>; +using TEvCopyTablesRequest = TGRpcRequestWrapper<TRpcServices::EvCopyTables, Ydb::Table::CopyTablesRequest, Ydb::Table::CopyTablesResponse, true, TRateLimiterMode::Rps>; using TEvRenameTablesRequest = TGRpcRequestWrapper<TRpcServices::EvRenameTables, Ydb::Table::RenameTablesRequest, Ydb::Table::RenameTablesResponse, true, TRateLimiterMode::Rps>; -using TEvDescribeTableRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTable, Ydb::Table::DescribeTableRequest, Ydb::Table::DescribeTableResponse, true, TRateLimiterMode::Rps>; -using TEvGetOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvGetOperation, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse, true, TRateLimiterMode::Rps>; -using TEvCancelOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvCancelOperation, Ydb::Operations::CancelOperationRequest, Ydb::Operations::CancelOperationResponse, false, TRateLimiterMode::Rps>; -using TEvForgetOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvForgetOperation, Ydb::Operations::ForgetOperationRequest, Ydb::Operations::ForgetOperationResponse, false, TRateLimiterMode::Rps>; -using TEvListOperationsRequest = TGRpcRequestValidationWrapper<TRpcServices::EvListOperations, Ydb::Operations::ListOperationsRequest, Ydb::Operations::ListOperationsResponse, false, TRateLimiterMode::Rps>; -using TEvCreateSessionRequest = TGRpcRequestWrapper<TRpcServices::EvCreateSession, Ydb::Table::CreateSessionRequest, Ydb::Table::CreateSessionResponse, true, TRateLimiterMode::Rps>; -using TEvDeleteSessionRequest = TGRpcRequestWrapper<TRpcServices::EvDeleteSession, Ydb::Table::DeleteSessionRequest, Ydb::Table::DeleteSessionResponse, true>; -using TEvKeepAliveRequest = TGRpcRequestWrapper<TRpcServices::EvKeepAlive, Ydb::Table::KeepAliveRequest, Ydb::Table::KeepAliveResponse, true, TRateLimiterMode::Rps>; -using TEvReadTableRequest = TGRpcRequestWrapper<TRpcServices::EvReadTable, Ydb::Table::ReadTableRequest, Ydb::Table::ReadTableResponse, false, TRateLimiterMode::RuOnProgress>; -using TEvExplainDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExplainDataQuery, Ydb::Table::ExplainDataQueryRequest, Ydb::Table::ExplainDataQueryResponse, true, TRateLimiterMode::Rps>; -using TEvPrepareDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvPrepareDataQuery, Ydb::Table::PrepareDataQueryRequest, Ydb::Table::PrepareDataQueryResponse, true, TRateLimiterMode::Ru>; -using TEvExecuteDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExecuteDataQuery, Ydb::Table::ExecuteDataQueryRequest, Ydb::Table::ExecuteDataQueryResponse, true, TRateLimiterMode::Ru>; -using TEvExecuteSchemeQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExecuteSchemeQuery, Ydb::Table::ExecuteSchemeQueryRequest, Ydb::Table::ExecuteSchemeQueryResponse, true, TRateLimiterMode::Rps>; -using TEvBeginTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvBeginTransaction, Ydb::Table::BeginTransactionRequest, Ydb::Table::BeginTransactionResponse, true, TRateLimiterMode::Rps>; -using TEvCommitTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvCommitTransaction, Ydb::Table::CommitTransactionRequest, Ydb::Table::CommitTransactionResponse, true>; -using TEvRollbackTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvRollbackTransaction, Ydb::Table::RollbackTransactionRequest, Ydb::Table::RollbackTransactionResponse, true>; -using TEvCreateTenantRequest = TGRpcRequestWrapper<TRpcServices::EvCreateTenant, Ydb::Cms::CreateDatabaseRequest, Ydb::Cms::CreateDatabaseResponse, true>; -using TEvAlterTenantRequest = TGRpcRequestWrapper<TRpcServices::EvAlterTenant, Ydb::Cms::AlterDatabaseRequest, Ydb::Cms::AlterDatabaseResponse, true>; -using TEvGetTenantStatusRequest = TGRpcRequestWrapper<TRpcServices::EvGetTenantStatus, Ydb::Cms::GetDatabaseStatusRequest, Ydb::Cms::GetDatabaseStatusResponse, true>; -using TEvListTenantsRequest = TGRpcRequestWrapper<TRpcServices::EvListTenants, Ydb::Cms::ListDatabasesRequest, Ydb::Cms::ListDatabasesResponse, true>; -using TEvRemoveTenantRequest = TGRpcRequestWrapper<TRpcServices::EvRemoveTenant, Ydb::Cms::RemoveDatabaseRequest, Ydb::Cms::RemoveDatabaseResponse, true>; -using TEvListEndpointsRequest = TGRpcRequestWrapper<TRpcServices::EvListEndpoints, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse, true>; -using TEvDescribeTenantOptionsRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTenantOptions, Ydb::Cms::DescribeDatabaseOptionsRequest, Ydb::Cms::DescribeDatabaseOptionsResponse, true>; -using TEvDescribeTableOptionsRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTableOptions, Ydb::Table::DescribeTableOptionsRequest, Ydb::Table::DescribeTableOptionsResponse, true, TRateLimiterMode::Rps>; -using TEvCreateCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvCreateCoordinationNode, Ydb::Coordination::CreateNodeRequest, Ydb::Coordination::CreateNodeResponse, true, TRateLimiterMode::Rps>; -using TEvAlterCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvAlterCoordinationNode, Ydb::Coordination::AlterNodeRequest, Ydb::Coordination::AlterNodeResponse, true, TRateLimiterMode::Rps>; -using TEvDropCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvDropCoordinationNode, Ydb::Coordination::DropNodeRequest, Ydb::Coordination::DropNodeResponse, true, TRateLimiterMode::Rps>; -using TEvDescribeCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvDescribeCoordinationNode, Ydb::Coordination::DescribeNodeRequest, Ydb::Coordination::DescribeNodeResponse, true, TRateLimiterMode::Rps>; -using TEvReadColumnsRequest = TGRpcRequestWrapper<TRpcServices::EvReadColumns, Ydb::ClickhouseInternal::ScanRequest, Ydb::ClickhouseInternal::ScanResponse, true>; -using TEvGetShardLocationsRequest = TGRpcRequestWrapper<TRpcServices::EvGetShardLocations, Ydb::ClickhouseInternal::GetShardLocationsRequest, Ydb::ClickhouseInternal::GetShardLocationsResponse, true>; -using TEvKikhouseDescribeTableRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseDescribeTable, Ydb::ClickhouseInternal::DescribeTableRequest, Ydb::ClickhouseInternal::DescribeTableResponse, true>; -using TEvS3ListingRequest = TGRpcRequestWrapper<TRpcServices::EvS3Listing, Ydb::S3Internal::S3ListingRequest, Ydb::S3Internal::S3ListingResponse, true>; -using TEvBiStreamPingRequest = TGRpcRequestBiStreamWrapper<TRpcServices::EvBiStreamPing, Draft::Dummy::PingRequest, Draft::Dummy::PingResponse>; +using TEvDescribeTableRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTable, Ydb::Table::DescribeTableRequest, Ydb::Table::DescribeTableResponse, true, TRateLimiterMode::Rps>; +using TEvGetOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvGetOperation, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse, true, TRateLimiterMode::Rps>; +using TEvCancelOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvCancelOperation, Ydb::Operations::CancelOperationRequest, Ydb::Operations::CancelOperationResponse, false, TRateLimiterMode::Rps>; +using TEvForgetOperationRequest = TGRpcRequestValidationWrapper<TRpcServices::EvForgetOperation, Ydb::Operations::ForgetOperationRequest, Ydb::Operations::ForgetOperationResponse, false, TRateLimiterMode::Rps>; +using TEvListOperationsRequest = TGRpcRequestValidationWrapper<TRpcServices::EvListOperations, Ydb::Operations::ListOperationsRequest, Ydb::Operations::ListOperationsResponse, false, TRateLimiterMode::Rps>; +using TEvCreateSessionRequest = TGRpcRequestWrapper<TRpcServices::EvCreateSession, Ydb::Table::CreateSessionRequest, Ydb::Table::CreateSessionResponse, true, TRateLimiterMode::Rps>; +using TEvDeleteSessionRequest = TGRpcRequestWrapper<TRpcServices::EvDeleteSession, Ydb::Table::DeleteSessionRequest, Ydb::Table::DeleteSessionResponse, true>; +using TEvKeepAliveRequest = TGRpcRequestWrapper<TRpcServices::EvKeepAlive, Ydb::Table::KeepAliveRequest, Ydb::Table::KeepAliveResponse, true, TRateLimiterMode::Rps>; +using TEvReadTableRequest = TGRpcRequestWrapper<TRpcServices::EvReadTable, Ydb::Table::ReadTableRequest, Ydb::Table::ReadTableResponse, false, TRateLimiterMode::RuOnProgress>; +using TEvExplainDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExplainDataQuery, Ydb::Table::ExplainDataQueryRequest, Ydb::Table::ExplainDataQueryResponse, true, TRateLimiterMode::Rps>; +using TEvPrepareDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvPrepareDataQuery, Ydb::Table::PrepareDataQueryRequest, Ydb::Table::PrepareDataQueryResponse, true, TRateLimiterMode::Ru>; +using TEvExecuteDataQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExecuteDataQuery, Ydb::Table::ExecuteDataQueryRequest, Ydb::Table::ExecuteDataQueryResponse, true, TRateLimiterMode::Ru>; +using TEvExecuteSchemeQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExecuteSchemeQuery, Ydb::Table::ExecuteSchemeQueryRequest, Ydb::Table::ExecuteSchemeQueryResponse, true, TRateLimiterMode::Rps>; +using TEvBeginTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvBeginTransaction, Ydb::Table::BeginTransactionRequest, Ydb::Table::BeginTransactionResponse, true, TRateLimiterMode::Rps>; +using TEvCommitTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvCommitTransaction, Ydb::Table::CommitTransactionRequest, Ydb::Table::CommitTransactionResponse, true>; +using TEvRollbackTransactionRequest = TGRpcRequestWrapper<TRpcServices::EvRollbackTransaction, Ydb::Table::RollbackTransactionRequest, Ydb::Table::RollbackTransactionResponse, true>; +using TEvCreateTenantRequest = TGRpcRequestWrapper<TRpcServices::EvCreateTenant, Ydb::Cms::CreateDatabaseRequest, Ydb::Cms::CreateDatabaseResponse, true>; +using TEvAlterTenantRequest = TGRpcRequestWrapper<TRpcServices::EvAlterTenant, Ydb::Cms::AlterDatabaseRequest, Ydb::Cms::AlterDatabaseResponse, true>; +using TEvGetTenantStatusRequest = TGRpcRequestWrapper<TRpcServices::EvGetTenantStatus, Ydb::Cms::GetDatabaseStatusRequest, Ydb::Cms::GetDatabaseStatusResponse, true>; +using TEvListTenantsRequest = TGRpcRequestWrapper<TRpcServices::EvListTenants, Ydb::Cms::ListDatabasesRequest, Ydb::Cms::ListDatabasesResponse, true>; +using TEvRemoveTenantRequest = TGRpcRequestWrapper<TRpcServices::EvRemoveTenant, Ydb::Cms::RemoveDatabaseRequest, Ydb::Cms::RemoveDatabaseResponse, true>; +using TEvListEndpointsRequest = TGRpcRequestWrapper<TRpcServices::EvListEndpoints, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse, true>; +using TEvDescribeTenantOptionsRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTenantOptions, Ydb::Cms::DescribeDatabaseOptionsRequest, Ydb::Cms::DescribeDatabaseOptionsResponse, true>; +using TEvDescribeTableOptionsRequest = TGRpcRequestWrapper<TRpcServices::EvDescribeTableOptions, Ydb::Table::DescribeTableOptionsRequest, Ydb::Table::DescribeTableOptionsResponse, true, TRateLimiterMode::Rps>; +using TEvCreateCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvCreateCoordinationNode, Ydb::Coordination::CreateNodeRequest, Ydb::Coordination::CreateNodeResponse, true, TRateLimiterMode::Rps>; +using TEvAlterCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvAlterCoordinationNode, Ydb::Coordination::AlterNodeRequest, Ydb::Coordination::AlterNodeResponse, true, TRateLimiterMode::Rps>; +using TEvDropCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvDropCoordinationNode, Ydb::Coordination::DropNodeRequest, Ydb::Coordination::DropNodeResponse, true, TRateLimiterMode::Rps>; +using TEvDescribeCoordinationNode = TGRpcRequestWrapper<TRpcServices::EvDescribeCoordinationNode, Ydb::Coordination::DescribeNodeRequest, Ydb::Coordination::DescribeNodeResponse, true, TRateLimiterMode::Rps>; +using TEvReadColumnsRequest = TGRpcRequestWrapper<TRpcServices::EvReadColumns, Ydb::ClickhouseInternal::ScanRequest, Ydb::ClickhouseInternal::ScanResponse, true>; +using TEvGetShardLocationsRequest = TGRpcRequestWrapper<TRpcServices::EvGetShardLocations, Ydb::ClickhouseInternal::GetShardLocationsRequest, Ydb::ClickhouseInternal::GetShardLocationsResponse, true>; +using TEvKikhouseDescribeTableRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseDescribeTable, Ydb::ClickhouseInternal::DescribeTableRequest, Ydb::ClickhouseInternal::DescribeTableResponse, true>; +using TEvS3ListingRequest = TGRpcRequestWrapper<TRpcServices::EvS3Listing, Ydb::S3Internal::S3ListingRequest, Ydb::S3Internal::S3ListingResponse, true>; +using TEvBiStreamPingRequest = TGRpcRequestBiStreamWrapper<TRpcServices::EvBiStreamPing, Draft::Dummy::PingRequest, Draft::Dummy::PingResponse>; using TEvExperimentalStreamQueryRequest = TGRpcRequestWrapper<TRpcServices::EvExperimentalStreamQuery, Ydb::Experimental::ExecuteStreamQueryRequest, Ydb::Experimental::ExecuteStreamQueryResponse, false>; using TEvStreamPQWriteRequest = TGRpcRequestBiStreamWrapper<TRpcServices::EvStreamPQWrite, Ydb::PersQueue::V1::StreamingWriteClientMessage, Ydb::PersQueue::V1::StreamingWriteServerMessage>; using TEvStreamPQReadRequest = TGRpcRequestBiStreamWrapper<TRpcServices::EvStreamPQRead, Ydb::PersQueue::V1::MigrationStreamingReadClientMessage, Ydb::PersQueue::V1::MigrationStreamingReadServerMessage>; -using TEvPQReadInfoRequest = TGRpcRequestWrapper<TRpcServices::EvPQReadInfo, Ydb::PersQueue::V1::ReadInfoRequest, Ydb::PersQueue::V1::ReadInfoResponse, true>; -using TEvPQDropTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQDropTopic, Ydb::PersQueue::V1::DropTopicRequest, Ydb::PersQueue::V1::DropTopicResponse, true>; -using TEvPQCreateTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQCreateTopic, Ydb::PersQueue::V1::CreateTopicRequest, Ydb::PersQueue::V1::CreateTopicResponse, true>; -using TEvPQAlterTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQAlterTopic, Ydb::PersQueue::V1::AlterTopicRequest, Ydb::PersQueue::V1::AlterTopicResponse, true>; -using TEvPQDescribeTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQDescribeTopic, Ydb::PersQueue::V1::DescribeTopicRequest, Ydb::PersQueue::V1::DescribeTopicResponse, true>; -using TEvPQAddReadRuleRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQAddReadRule, Ydb::PersQueue::V1::AddReadRuleRequest, Ydb::PersQueue::V1::AddReadRuleResponse, true>; -using TEvPQRemoveReadRuleRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQRemoveReadRule, Ydb::PersQueue::V1::RemoveReadRuleRequest, Ydb::PersQueue::V1::RemoveReadRuleResponse, true>; -using TEvExportToYtRequest = TGRpcRequestValidationWrapper<TRpcServices::EvExportToYt, Ydb::Export::ExportToYtRequest, Ydb::Export::ExportToYtResponse, true>; -using TEvExportToS3Request = TGRpcRequestValidationWrapper<TRpcServices::EvExportToS3, Ydb::Export::ExportToS3Request, Ydb::Export::ExportToS3Response, true>; -using TEvImportFromS3Request = TGRpcRequestValidationWrapper<TRpcServices::EvImportFromS3, Ydb::Import::ImportFromS3Request, Ydb::Import::ImportFromS3Response, true>; -using TEvImportDataRequest = TGRpcRequestValidationWrapper<TRpcServices::EvImportData, Ydb::Import::ImportDataRequest, Ydb::Import::ImportDataResponse, true>; -using TEvDiscoverPQClustersRequest = TGRpcRequestWrapper<TRpcServices::EvDiscoverPQClusters, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse, true>; -using TEvBulkUpsertRequest = TGRpcRequestWrapper<TRpcServices::EvBulkUpsert, Ydb::Table::BulkUpsertRequest, Ydb::Table::BulkUpsertResponse, true, TRateLimiterMode::Ru>; -using TEvWhoAmIRequest = TGRpcRequestWrapper<TRpcServices::EvWhoAmI, Ydb::Discovery::WhoAmIRequest, Ydb::Discovery::WhoAmIResponse, true, TRateLimiterMode::Rps>; -using TEvCreateRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvCreateRateLimiterResource, Ydb::RateLimiter::CreateResourceRequest, Ydb::RateLimiter::CreateResourceResponse, true, TRateLimiterMode::Rps>; -using TEvAlterRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvAlterRateLimiterResource, Ydb::RateLimiter::AlterResourceRequest, Ydb::RateLimiter::AlterResourceResponse, true, TRateLimiterMode::Rps>; -using TEvDropRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvDropRateLimiterResource, Ydb::RateLimiter::DropResourceRequest, Ydb::RateLimiter::DropResourceResponse, true, TRateLimiterMode::Rps>; -using TEvListRateLimiterResources = TGRpcRequestWrapper<TRpcServices::EvListRateLimiterResources, Ydb::RateLimiter::ListResourcesRequest, Ydb::RateLimiter::ListResourcesResponse, true, TRateLimiterMode::Rps>; -using TEvDescribeRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvDescribeRateLimiterResource, Ydb::RateLimiter::DescribeResourceRequest, Ydb::RateLimiter::DescribeResourceResponse, true, TRateLimiterMode::Rps>; -using TEvAcquireRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvAcquireRateLimiterResource, Ydb::RateLimiter::AcquireResourceRequest, Ydb::RateLimiter::AcquireResourceResponse, true>; -using TEvKikhouseCreateSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseCreateSnapshot, Ydb::ClickhouseInternal::CreateSnapshotRequest, Ydb::ClickhouseInternal::CreateSnapshotResponse, true>; -using TEvKikhouseRefreshSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseRefreshSnapshot, Ydb::ClickhouseInternal::RefreshSnapshotRequest, Ydb::ClickhouseInternal::RefreshSnapshotResponse, true>; -using TEvKikhouseDiscardSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseDiscardSnapshot, Ydb::ClickhouseInternal::DiscardSnapshotRequest, Ydb::ClickhouseInternal::DiscardSnapshotResponse, true>; -using TEvSelfCheckRequest = TGRpcRequestWrapper<TRpcServices::EvSelfCheck, Ydb::Monitoring::SelfCheckRequest, Ydb::Monitoring::SelfCheckResponse, true>; +using TEvPQReadInfoRequest = TGRpcRequestWrapper<TRpcServices::EvPQReadInfo, Ydb::PersQueue::V1::ReadInfoRequest, Ydb::PersQueue::V1::ReadInfoResponse, true>; +using TEvPQDropTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQDropTopic, Ydb::PersQueue::V1::DropTopicRequest, Ydb::PersQueue::V1::DropTopicResponse, true>; +using TEvPQCreateTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQCreateTopic, Ydb::PersQueue::V1::CreateTopicRequest, Ydb::PersQueue::V1::CreateTopicResponse, true>; +using TEvPQAlterTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQAlterTopic, Ydb::PersQueue::V1::AlterTopicRequest, Ydb::PersQueue::V1::AlterTopicResponse, true>; +using TEvPQDescribeTopicRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQDescribeTopic, Ydb::PersQueue::V1::DescribeTopicRequest, Ydb::PersQueue::V1::DescribeTopicResponse, true>; +using TEvPQAddReadRuleRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQAddReadRule, Ydb::PersQueue::V1::AddReadRuleRequest, Ydb::PersQueue::V1::AddReadRuleResponse, true>; +using TEvPQRemoveReadRuleRequest = TGRpcRequestValidationWrapper<TRpcServices::EvPQRemoveReadRule, Ydb::PersQueue::V1::RemoveReadRuleRequest, Ydb::PersQueue::V1::RemoveReadRuleResponse, true>; +using TEvExportToYtRequest = TGRpcRequestValidationWrapper<TRpcServices::EvExportToYt, Ydb::Export::ExportToYtRequest, Ydb::Export::ExportToYtResponse, true>; +using TEvExportToS3Request = TGRpcRequestValidationWrapper<TRpcServices::EvExportToS3, Ydb::Export::ExportToS3Request, Ydb::Export::ExportToS3Response, true>; +using TEvImportFromS3Request = TGRpcRequestValidationWrapper<TRpcServices::EvImportFromS3, Ydb::Import::ImportFromS3Request, Ydb::Import::ImportFromS3Response, true>; +using TEvImportDataRequest = TGRpcRequestValidationWrapper<TRpcServices::EvImportData, Ydb::Import::ImportDataRequest, Ydb::Import::ImportDataResponse, true>; +using TEvDiscoverPQClustersRequest = TGRpcRequestWrapper<TRpcServices::EvDiscoverPQClusters, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse, true>; +using TEvBulkUpsertRequest = TGRpcRequestWrapper<TRpcServices::EvBulkUpsert, Ydb::Table::BulkUpsertRequest, Ydb::Table::BulkUpsertResponse, true, TRateLimiterMode::Ru>; +using TEvWhoAmIRequest = TGRpcRequestWrapper<TRpcServices::EvWhoAmI, Ydb::Discovery::WhoAmIRequest, Ydb::Discovery::WhoAmIResponse, true, TRateLimiterMode::Rps>; +using TEvCreateRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvCreateRateLimiterResource, Ydb::RateLimiter::CreateResourceRequest, Ydb::RateLimiter::CreateResourceResponse, true, TRateLimiterMode::Rps>; +using TEvAlterRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvAlterRateLimiterResource, Ydb::RateLimiter::AlterResourceRequest, Ydb::RateLimiter::AlterResourceResponse, true, TRateLimiterMode::Rps>; +using TEvDropRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvDropRateLimiterResource, Ydb::RateLimiter::DropResourceRequest, Ydb::RateLimiter::DropResourceResponse, true, TRateLimiterMode::Rps>; +using TEvListRateLimiterResources = TGRpcRequestWrapper<TRpcServices::EvListRateLimiterResources, Ydb::RateLimiter::ListResourcesRequest, Ydb::RateLimiter::ListResourcesResponse, true, TRateLimiterMode::Rps>; +using TEvDescribeRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvDescribeRateLimiterResource, Ydb::RateLimiter::DescribeResourceRequest, Ydb::RateLimiter::DescribeResourceResponse, true, TRateLimiterMode::Rps>; +using TEvAcquireRateLimiterResource = TGRpcRequestWrapper<TRpcServices::EvAcquireRateLimiterResource, Ydb::RateLimiter::AcquireResourceRequest, Ydb::RateLimiter::AcquireResourceResponse, true>; +using TEvKikhouseCreateSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseCreateSnapshot, Ydb::ClickhouseInternal::CreateSnapshotRequest, Ydb::ClickhouseInternal::CreateSnapshotResponse, true>; +using TEvKikhouseRefreshSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseRefreshSnapshot, Ydb::ClickhouseInternal::RefreshSnapshotRequest, Ydb::ClickhouseInternal::RefreshSnapshotResponse, true>; +using TEvKikhouseDiscardSnapshotRequest = TGRpcRequestWrapper<TRpcServices::EvKikhouseDiscardSnapshot, Ydb::ClickhouseInternal::DiscardSnapshotRequest, Ydb::ClickhouseInternal::DiscardSnapshotResponse, true>; +using TEvSelfCheckRequest = TGRpcRequestWrapper<TRpcServices::EvSelfCheck, Ydb::Monitoring::SelfCheckRequest, Ydb::Monitoring::SelfCheckResponse, true>; using TEvLoginRequest = TGRpcRequestWrapperNoAuth<TRpcServices::EvLogin, Ydb::Auth::LoginRequest, Ydb::Auth::LoginResponse>; -using TEvStreamExecuteScanQueryRequest = TGRpcRequestWrapper<TRpcServices::EvStreamExecuteScanQuery, Ydb::Table::ExecuteScanQueryRequest, Ydb::Table::ExecuteScanQueryPartialResponse, false, TRateLimiterMode::RuOnProgress>; +using TEvStreamExecuteScanQueryRequest = TGRpcRequestWrapper<TRpcServices::EvStreamExecuteScanQuery, Ydb::Table::ExecuteScanQueryRequest, Ydb::Table::ExecuteScanQueryPartialResponse, false, TRateLimiterMode::RuOnProgress>; using TEvCoordinationSessionRequest = TGRpcRequestBiStreamWrapper<TRpcServices::EvCoordinationSession, Ydb::Coordination::SessionRequest, Ydb::Coordination::SessionResponse>; -using TEvLongTxBeginRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxBegin, Ydb::LongTx::BeginTransactionRequest, Ydb::LongTx::BeginTransactionResponse, true>; -using TEvLongTxCommitRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxCommit, Ydb::LongTx::CommitTransactionRequest, Ydb::LongTx::CommitTransactionResponse, true>; -using TEvLongTxRollbackRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxRollback, Ydb::LongTx::RollbackTransactionRequest, Ydb::LongTx::RollbackTransactionResponse, true>; -using TEvLongTxWriteRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxWrite, Ydb::LongTx::WriteRequest, Ydb::LongTx::WriteResponse, true>; -using TEvLongTxReadRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxRead, Ydb::LongTx::ReadRequest, Ydb::LongTx::ReadResponse, true>; -using TEvDataStreamsCreateStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsCreateStream, Ydb::DataStreams::V1::CreateStreamRequest, Ydb::DataStreams::V1::CreateStreamResponse, true>; -using TEvDataStreamsDeleteStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDeleteStream, Ydb::DataStreams::V1::DeleteStreamRequest, Ydb::DataStreams::V1::DeleteStreamResponse, true>; -using TEvDataStreamsDescribeStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStream, Ydb::DataStreams::V1::DescribeStreamRequest, Ydb::DataStreams::V1::DescribeStreamResponse, true>; -using TEvDataStreamsRegisterStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsRegisterStreamConsumer, Ydb::DataStreams::V1::RegisterStreamConsumerRequest, Ydb::DataStreams::V1::RegisterStreamConsumerResponse, true>; -using TEvDataStreamsDeregisterStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDeregisterStreamConsumer, Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, true>; -using TEvDataStreamsDescribeStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStreamConsumer, Ydb::DataStreams::V1::DescribeStreamConsumerRequest, Ydb::DataStreams::V1::DescribeStreamConsumerResponse, true>; -using TEvDataStreamsPutRecordRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsPutRecord, Ydb::DataStreams::V1::PutRecordRequest, Ydb::DataStreams::V1::PutRecordResponse, true>; -using TEvDataStreamsListStreamsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListStreams, Ydb::DataStreams::V1::ListStreamsRequest, Ydb::DataStreams::V1::ListStreamsResponse, true>; -using TEvDataStreamsListShardsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListShards, Ydb::DataStreams::V1::ListShardsRequest, Ydb::DataStreams::V1::ListShardsResponse, true>; -using TEvDataStreamsPutRecordsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsPutRecords, Ydb::DataStreams::V1::PutRecordsRequest, Ydb::DataStreams::V1::PutRecordsResponse, true>; -using TEvDataStreamsGetRecordsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsGetRecords, Ydb::DataStreams::V1::GetRecordsRequest, Ydb::DataStreams::V1::GetRecordsResponse, true>; -using TEvDataStreamsGetShardIteratorRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsGetShardIterator, Ydb::DataStreams::V1::GetShardIteratorRequest, Ydb::DataStreams::V1::GetShardIteratorResponse, true>; -using TEvDataStreamsSubscribeToShardRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSubscribeToShard, Ydb::DataStreams::V1::SubscribeToShardRequest, Ydb::DataStreams::V1::SubscribeToShardResponse, true>; -using TEvDataStreamsDescribeLimitsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeLimits, Ydb::DataStreams::V1::DescribeLimitsRequest, Ydb::DataStreams::V1::DescribeLimitsResponse, true>; -using TEvDataStreamsDescribeStreamSummaryRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStreamSummary, Ydb::DataStreams::V1::DescribeStreamSummaryRequest, Ydb::DataStreams::V1::DescribeStreamSummaryResponse, true>; -using TEvDataStreamsDecreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDecreaseStreamRetentionPeriod, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, true>; -using TEvDataStreamsIncreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsIncreaseStreamRetentionPeriod, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, true>; -using TEvDataStreamsUpdateShardCountRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateShardCount, Ydb::DataStreams::V1::UpdateShardCountRequest, Ydb::DataStreams::V1::UpdateShardCountResponse, true>; +using TEvLongTxBeginRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxBegin, Ydb::LongTx::BeginTransactionRequest, Ydb::LongTx::BeginTransactionResponse, true>; +using TEvLongTxCommitRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxCommit, Ydb::LongTx::CommitTransactionRequest, Ydb::LongTx::CommitTransactionResponse, true>; +using TEvLongTxRollbackRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxRollback, Ydb::LongTx::RollbackTransactionRequest, Ydb::LongTx::RollbackTransactionResponse, true>; +using TEvLongTxWriteRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxWrite, Ydb::LongTx::WriteRequest, Ydb::LongTx::WriteResponse, true>; +using TEvLongTxReadRequest = TGRpcRequestWrapper<TRpcServices::EvLongTxRead, Ydb::LongTx::ReadRequest, Ydb::LongTx::ReadResponse, true>; +using TEvDataStreamsCreateStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsCreateStream, Ydb::DataStreams::V1::CreateStreamRequest, Ydb::DataStreams::V1::CreateStreamResponse, true>; +using TEvDataStreamsDeleteStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDeleteStream, Ydb::DataStreams::V1::DeleteStreamRequest, Ydb::DataStreams::V1::DeleteStreamResponse, true>; +using TEvDataStreamsDescribeStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStream, Ydb::DataStreams::V1::DescribeStreamRequest, Ydb::DataStreams::V1::DescribeStreamResponse, true>; +using TEvDataStreamsRegisterStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsRegisterStreamConsumer, Ydb::DataStreams::V1::RegisterStreamConsumerRequest, Ydb::DataStreams::V1::RegisterStreamConsumerResponse, true>; +using TEvDataStreamsDeregisterStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDeregisterStreamConsumer, Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, true>; +using TEvDataStreamsDescribeStreamConsumerRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStreamConsumer, Ydb::DataStreams::V1::DescribeStreamConsumerRequest, Ydb::DataStreams::V1::DescribeStreamConsumerResponse, true>; +using TEvDataStreamsPutRecordRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsPutRecord, Ydb::DataStreams::V1::PutRecordRequest, Ydb::DataStreams::V1::PutRecordResponse, true>; +using TEvDataStreamsListStreamsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListStreams, Ydb::DataStreams::V1::ListStreamsRequest, Ydb::DataStreams::V1::ListStreamsResponse, true>; +using TEvDataStreamsListShardsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListShards, Ydb::DataStreams::V1::ListShardsRequest, Ydb::DataStreams::V1::ListShardsResponse, true>; +using TEvDataStreamsPutRecordsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsPutRecords, Ydb::DataStreams::V1::PutRecordsRequest, Ydb::DataStreams::V1::PutRecordsResponse, true>; +using TEvDataStreamsGetRecordsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsGetRecords, Ydb::DataStreams::V1::GetRecordsRequest, Ydb::DataStreams::V1::GetRecordsResponse, true>; +using TEvDataStreamsGetShardIteratorRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsGetShardIterator, Ydb::DataStreams::V1::GetShardIteratorRequest, Ydb::DataStreams::V1::GetShardIteratorResponse, true>; +using TEvDataStreamsSubscribeToShardRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSubscribeToShard, Ydb::DataStreams::V1::SubscribeToShardRequest, Ydb::DataStreams::V1::SubscribeToShardResponse, true>; +using TEvDataStreamsDescribeLimitsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeLimits, Ydb::DataStreams::V1::DescribeLimitsRequest, Ydb::DataStreams::V1::DescribeLimitsResponse, true>; +using TEvDataStreamsDescribeStreamSummaryRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDescribeStreamSummary, Ydb::DataStreams::V1::DescribeStreamSummaryRequest, Ydb::DataStreams::V1::DescribeStreamSummaryResponse, true>; +using TEvDataStreamsDecreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDecreaseStreamRetentionPeriod, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, true>; +using TEvDataStreamsIncreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsIncreaseStreamRetentionPeriod, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, true>; +using TEvDataStreamsUpdateShardCountRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateShardCount, Ydb::DataStreams::V1::UpdateShardCountRequest, Ydb::DataStreams::V1::UpdateShardCountResponse, true>; using TEvDataStreamsUpdateStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateStream, Ydb::DataStreams::V1::UpdateStreamRequest, Ydb::DataStreams::V1::UpdateStreamResponse, true>; using TEvDataStreamsSetWriteQuotaRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSetWriteQuota, Ydb::DataStreams::V1::SetWriteQuotaRequest, Ydb::DataStreams::V1::SetWriteQuotaResponse, true>; -using TEvDataStreamsListStreamConsumersRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListStreamConsumers, Ydb::DataStreams::V1::ListStreamConsumersRequest, Ydb::DataStreams::V1::ListStreamConsumersResponse, true>; -using TEvDataStreamsAddTagsToStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsAddTagsToStream, Ydb::DataStreams::V1::AddTagsToStreamRequest, Ydb::DataStreams::V1::AddTagsToStreamResponse, true>; -using TEvDataStreamsDisableEnhancedMonitoringRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDisableEnhancedMonitoring, Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, true>; -using TEvDataStreamsEnableEnhancedMonitoringRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsEnableEnhancedMonitoring, Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, true>; -using TEvDataStreamsListTagsForStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListTagsForStream, Ydb::DataStreams::V1::ListTagsForStreamRequest, Ydb::DataStreams::V1::ListTagsForStreamResponse, true>; -using TEvDataStreamsMergeShardsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsMergeShards, Ydb::DataStreams::V1::MergeShardsRequest, Ydb::DataStreams::V1::MergeShardsResponse, true>; -using TEvDataStreamsRemoveTagsFromStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsRemoveTagsFromStream, Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, true>; -using TEvDataStreamsSplitShardRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSplitShard, Ydb::DataStreams::V1::SplitShardRequest, Ydb::DataStreams::V1::SplitShardResponse, true>; -using TEvDataStreamsStartStreamEncryptionRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsStartStreamEncryption, Ydb::DataStreams::V1::StartStreamEncryptionRequest, Ydb::DataStreams::V1::StartStreamEncryptionResponse, true>; -using TEvDataStreamsStopStreamEncryptionRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsStopStreamEncryption, Ydb::DataStreams::V1::StopStreamEncryptionRequest, Ydb::DataStreams::V1::StopStreamEncryptionResponse, true>; - -} // namespace NGRpcService -} // namespace NKikimr +using TEvDataStreamsListStreamConsumersRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListStreamConsumers, Ydb::DataStreams::V1::ListStreamConsumersRequest, Ydb::DataStreams::V1::ListStreamConsumersResponse, true>; +using TEvDataStreamsAddTagsToStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsAddTagsToStream, Ydb::DataStreams::V1::AddTagsToStreamRequest, Ydb::DataStreams::V1::AddTagsToStreamResponse, true>; +using TEvDataStreamsDisableEnhancedMonitoringRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDisableEnhancedMonitoring, Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, true>; +using TEvDataStreamsEnableEnhancedMonitoringRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsEnableEnhancedMonitoring, Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, true>; +using TEvDataStreamsListTagsForStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListTagsForStream, Ydb::DataStreams::V1::ListTagsForStreamRequest, Ydb::DataStreams::V1::ListTagsForStreamResponse, true>; +using TEvDataStreamsMergeShardsRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsMergeShards, Ydb::DataStreams::V1::MergeShardsRequest, Ydb::DataStreams::V1::MergeShardsResponse, true>; +using TEvDataStreamsRemoveTagsFromStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsRemoveTagsFromStream, Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, true>; +using TEvDataStreamsSplitShardRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSplitShard, Ydb::DataStreams::V1::SplitShardRequest, Ydb::DataStreams::V1::SplitShardResponse, true>; +using TEvDataStreamsStartStreamEncryptionRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsStartStreamEncryption, Ydb::DataStreams::V1::StartStreamEncryptionRequest, Ydb::DataStreams::V1::StartStreamEncryptionResponse, true>; +using TEvDataStreamsStopStreamEncryptionRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsStopStreamEncryption, Ydb::DataStreams::V1::StopStreamEncryptionRequest, Ydb::DataStreams::V1::StopStreamEncryptionResponse, true>; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_calls_ut.cpp b/ydb/core/grpc_services/rpc_calls_ut.cpp index 877b98973c..514be6ffee 100644 --- a/ydb/core/grpc_services/rpc_calls_ut.cpp +++ b/ydb/core/grpc_services/rpc_calls_ut.cpp @@ -1,22 +1,22 @@ -#include "rpc_calls.h" +#include "rpc_calls.h" #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - + +namespace NKikimr { + Y_UNIT_TEST_SUITE(SplitPathTests) { Y_UNIT_TEST(WithoutDatabaseShouldSuccess) { - { + { auto pair = NGRpcService::SplitPath("/root/table"); UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root"); - UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); - } + UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); + } { auto pair = NGRpcService::SplitPath("/root/dir/table"); UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root/dir"); UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); } - } + } Y_UNIT_TEST(WithDatabaseShouldSuccess) { for (auto db : TVector<TString>{"/root/db", "/root/db/"}) { @@ -33,6 +33,6 @@ Y_UNIT_TEST_SUITE(SplitPathTests) { UNIT_ASSERT_EXCEPTION(NGRpcService::SplitPath("/root/db2", path), yexception); } } -} - -} // namespace NKikimr +} + +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_cancel_operation.cpp b/ydb/core/grpc_services/rpc_cancel_operation.cpp index 8c4692566f..579f927c80 100644 --- a/ydb/core/grpc_services/rpc_cancel_operation.cpp +++ b/ydb/core/grpc_services/rpc_cancel_operation.cpp @@ -57,22 +57,22 @@ class TCancelOperationRPC: public TRpcOperationRequestActor<TCancelOperationRPC, void Handle(TEvImport::TEvCancelImportResponse::TPtr& ev) { const auto& record = ev->Get()->Record.GetResponse(); - + LOG_D("Handle TEvImport::TEvCancelImportResponse" << ": record# " << record.ShortDebugString()); - + Reply(record.GetStatus(), record.GetIssues()); - } - + } + void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev) { const auto& record = ev->Get()->Record; - + LOG_D("Handle TEvIndexBuilder::TEvCancelResponse" << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); - } - + + Reply(record.GetStatus(), record.GetIssues()); + } + public: using TRpcOperationRequestActor::TRpcOperationRequestActor; @@ -80,9 +80,9 @@ public: const TString& id = Request->GetProtoRequest()->id(); try { - OperationId = TOperationId(id); + OperationId = TOperationId(id); - switch (OperationId.GetKind()) { + switch (OperationId.GetKind()) { case TOperationId::EXPORT: case TOperationId::IMPORT: case TOperationId::BUILD_INDEX: @@ -90,7 +90,7 @@ public: return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unable to extract operation id"); } break; - + default: return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); } @@ -113,8 +113,8 @@ public: } } -private: - TOperationId OperationId; +private: + TOperationId OperationId; ui64 RawOperationId = 0; }; // TCancelOperationRPC diff --git a/ydb/core/grpc_services/rpc_cms.cpp b/ydb/core/grpc_services/rpc_cms.cpp index cd3e22c49c..7845023306 100644 --- a/ydb/core/grpc_services/rpc_cms.cpp +++ b/ydb/core/grpc_services/rpc_cms.cpp @@ -38,7 +38,7 @@ public: NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 10}; auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeConsoleID(group), pipeConfig); - CmsPipe = ctx.RegisterWithSameMailbox(pipe); + CmsPipe = ctx.RegisterWithSameMailbox(pipe); SendRequest(ctx); @@ -52,56 +52,56 @@ private: TBase::Die(ctx); } - template<typename T> - void HandleWithOperationParams(T& ev, const TActorContext& ctx) + template<typename T> + void HandleWithOperationParams(T& ev, const TActorContext& ctx) + { + const auto& response = ev->Get()->Record.GetResponse(); + if (response.operation().ready() == false + && this->GetProtoRequest()->operation_params().operation_mode() == Ydb::Operations::OperationParams::SYNC) { + auto request = MakeHolder<TEvConsole::TEvNotifyOperationCompletionRequest>(); + request->Record.MutableRequest()->set_id(response.operation().id()); + request->Record.SetUserToken(this->Request_->GetInternalToken()); + + NTabletPipe::SendData(ctx, CmsPipe, request.Release()); + } else { + TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), this->Request_); + Die(ctx); + } + } + + void Handle(TEvConsole::TEvCreateTenantResponse::TPtr& ev, const TActorContext& ctx) { + HandleWithOperationParams(ev, ctx); + } + + void Handle(TEvConsole::TEvRemoveTenantResponse::TPtr& ev, const TActorContext& ctx) { + HandleWithOperationParams(ev, ctx); + } + + template<typename T> + void Handle(T& ev, const TActorContext& ctx) { - const auto& response = ev->Get()->Record.GetResponse(); - if (response.operation().ready() == false - && this->GetProtoRequest()->operation_params().operation_mode() == Ydb::Operations::OperationParams::SYNC) { - auto request = MakeHolder<TEvConsole::TEvNotifyOperationCompletionRequest>(); - request->Record.MutableRequest()->set_id(response.operation().id()); - request->Record.SetUserToken(this->Request_->GetInternalToken()); - - NTabletPipe::SendData(ctx, CmsPipe, request.Release()); - } else { - TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), this->Request_); - Die(ctx); - } - } - - void Handle(TEvConsole::TEvCreateTenantResponse::TPtr& ev, const TActorContext& ctx) { - HandleWithOperationParams(ev, ctx); - } - - void Handle(TEvConsole::TEvRemoveTenantResponse::TPtr& ev, const TActorContext& ctx) { - HandleWithOperationParams(ev, ctx); - } - - template<typename T> - void Handle(T& ev, const TActorContext& ctx) - { auto& response = ev->Get()->Record.GetResponse(); - TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), this->Request_); + TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), this->Request_); Die(ctx); } - void Handle(TEvConsole::TEvOperationCompletionNotification::TPtr& ev, const TActorContext& ctx) - { - this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); - Die(ctx); - } - - void Handle(TEvConsole::TEvNotifyOperationCompletionResponse::TPtr& ev, const TActorContext& ctx) - { - if (ev->Get()->Record.GetResponse().operation().ready() == true) { - this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); - Die(ctx); - } - } - + void Handle(TEvConsole::TEvOperationCompletionNotification::TPtr& ev, const TActorContext& ctx) + { + this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); + Die(ctx); + } + + void Handle(TEvConsole::TEvNotifyOperationCompletionResponse::TPtr& ev, const TActorContext& ctx) + { + if (ev->Get()->Record.GetResponse().operation().ready() == true) { + this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); + Die(ctx); + } + } + void Undelivered(const TActorContext &ctx) { - this->Request_->RaiseIssue(NYql::TIssue("CMS is unavailable")); - this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); + this->Request_->RaiseIssue(NYql::TIssue("CMS is unavailable")); + this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); Die(ctx); } @@ -116,8 +116,8 @@ private: HFunc(TCmsResponse, Handle); CFunc(TEvTabletPipe::EvClientDestroyed, Undelivered); HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvConsole::TEvOperationCompletionNotification, Handle); - HFunc(TEvConsole::TEvNotifyOperationCompletionResponse, Handle); + HFunc(TEvConsole::TEvOperationCompletionNotification, Handle); + HFunc(TEvConsole::TEvNotifyOperationCompletionResponse, Handle); default: TBase::StateFuncBase(ev, ctx); } } @@ -125,7 +125,7 @@ private: void SendRequest(const TActorContext &ctx) { auto request = MakeHolder<TCmsRequest>(); - request->Record.MutableRequest()->CopyFrom(*this->GetProtoRequest()); + request->Record.MutableRequest()->CopyFrom(*this->GetProtoRequest()); request->Record.SetUserToken(this->Request_->GetInternalToken()); NTabletPipe::SendData(ctx, CmsPipe, request.Release()); } diff --git a/ydb/core/grpc_services/rpc_commit_transaction.cpp b/ydb/core/grpc_services/rpc_commit_transaction.cpp index 9d0cd1fd9c..3b85b5ef62 100644 --- a/ydb/core/grpc_services/rpc_commit_transaction.cpp +++ b/ydb/core/grpc_services/rpc_commit_transaction.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -37,14 +37,14 @@ public: private: void CommitTransactionImpl(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); TString sessionId; auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); @@ -72,14 +72,14 @@ private: void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetRef(); - SetCost(record.GetConsumedRu()); + SetCost(record.GetConsumedRu()); AddServerHintsIfAny(record); - + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - auto commitResult = TEvCommitTransactionRequest::AllocateResult<Ydb::Table::CommitTransactionResult>(Request_); + auto commitResult = TEvCommitTransactionRequest::AllocateResult<Ydb::Table::CommitTransactionResult>(Request_); if (kqpResponse.HasQueryStats()) { FillQueryStats(*commitResult->mutable_query_stats(), kqpResponse); diff --git a/ydb/core/grpc_services/rpc_common.h b/ydb/core/grpc_services/rpc_common.h index 36164be3ef..075318037c 100644 --- a/ydb/core/grpc_services/rpc_common.h +++ b/ydb/core/grpc_services/rpc_common.h @@ -1,49 +1,49 @@ -#pragma once - -#include "grpc_request_proxy.h" - +#pragma once + +#include "grpc_request_proxy.h" + #include <ydb/core/base/path.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/util/proto_duration.h> -namespace NKikimr { -namespace NGRpcService { - -template<typename TEv> -inline void SetRlPath(TEv& ev, const IRequestCtx& ctx) { - if (const auto& path = ctx.GetRlPath()) { - auto rl = ev->Record.MutableRlPath(); - rl->SetCoordinationNode(path->CoordinationNode); - rl->SetResourcePath(path->ResourcePath); - } -} - -template<typename TEv> -inline void SetAuthToken(TEv& ev, const IRequestCtx& ctx) { - if (ctx.GetInternalToken()) { - ev->Record.SetUserToken(ctx.GetInternalToken()); - } -} - -template<typename TEv> -inline void SetDatabase(TEv& ev, const IRequestCtx& ctx) { - // Empty database in case of absent header +namespace NKikimr { +namespace NGRpcService { + +template<typename TEv> +inline void SetRlPath(TEv& ev, const IRequestCtx& ctx) { + if (const auto& path = ctx.GetRlPath()) { + auto rl = ev->Record.MutableRlPath(); + rl->SetCoordinationNode(path->CoordinationNode); + rl->SetResourcePath(path->ResourcePath); + } +} + +template<typename TEv> +inline void SetAuthToken(TEv& ev, const IRequestCtx& ctx) { + if (ctx.GetInternalToken()) { + ev->Record.SetUserToken(ctx.GetInternalToken()); + } +} + +template<typename TEv> +inline void SetDatabase(TEv& ev, const IRequestCtx& ctx) { + // Empty database in case of absent header ev->Record.MutableRequest()->SetDatabase(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); -} - -inline void SetDatabase(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { +} + +inline void SetDatabase(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { // Empty database in case of absent header ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); } -inline void SetDatabase(TEvTxUserProxy::TEvNavigate* ev, const IRequestCtx& ctx) { +inline void SetDatabase(TEvTxUserProxy::TEvNavigate* ev, const IRequestCtx& ctx) { // Empty database in case of absent header ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); } -inline void SetRequestType(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { +inline void SetRequestType(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { ev->Record.SetRequestType(ctx.GetRequestType().GetOrElse("")); } -} // namespace NGRpcService -} // namespace NKikimr +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_copy_table.cpp b/ydb/core/grpc_services/rpc_copy_table.cpp index f821de46a4..74a0c9d9a8 100644 --- a/ydb/core/grpc_services/rpc_copy_table.cpp +++ b/ydb/core/grpc_services/rpc_copy_table.cpp @@ -1,58 +1,58 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" #include "rpc_scheme_base.h" -#include "rpc_common.h" - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; +#include "rpc_common.h" + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - + class TCopyTableRPC : public TRpcSchemeRequestActor<TCopyTableRPC, TEvCopyTableRequest> { using TBase = TRpcSchemeRequestActor<TCopyTableRPC, TEvCopyTableRequest>; -public: - TCopyTableRPC(TEvCopyTableRequest* msg) +public: + TCopyTableRPC(TEvCopyTableRequest* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); - Become(&TCopyTableRPC::StateWork); - } - -private: - void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> destinationPathPair; - try { - destinationPathPair = SplitPath(req->destination_path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return Reply(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = destinationPathPair.first; - const auto& name = destinationPathPair.second; - + SendProposeRequest(ctx); + Become(&TCopyTableRPC::StateWork); + } + +private: + void SendProposeRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + std::pair<TString, TString> destinationPathPair; + try { + destinationPathPair = SplitPath(req->destination_path()); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return Reply(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = destinationPathPair.first; + const auto& name = destinationPathPair.second; + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); - auto create = modifyScheme->MutableCreateTable(); - create->SetName(name); - create->SetCopyFromTable(req->source_path()); - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } -}; - -void TGRpcRequestProxy::Handle(TEvCopyTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TCopyTableRPC(ev->Release().Release())); -} - -} // namespace NKikimr -} // namespace NGRpcService + auto create = modifyScheme->MutableCreateTable(); + create->SetName(name); + create->SetCopyFromTable(req->source_path()); + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } +}; + +void TGRpcRequestProxy::Handle(TEvCopyTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TCopyTableRPC(ev->Release().Release())); +} + +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_copy_tables.cpp b/ydb/core/grpc_services/rpc_copy_tables.cpp index 343d1fe9c5..1f5fd281a5 100644 --- a/ydb/core/grpc_services/rpc_copy_tables.cpp +++ b/ydb/core/grpc_services/rpc_copy_tables.cpp @@ -32,7 +32,7 @@ private: modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables); auto copy = modifyScheme->MutableCreateConsistentCopyTables(); - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); for (const auto& item: req->tables()) { auto description = copy->AddCopyTableDescriptions(); description->SetSrcPath(item.source_path()); diff --git a/ydb/core/grpc_services/rpc_create_coordination_node.cpp b/ydb/core/grpc_services/rpc_create_coordination_node.cpp index 75f7541b2b..11c838592c 100644 --- a/ydb/core/grpc_services/rpc_create_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_create_coordination_node.cpp @@ -26,12 +26,12 @@ public: private: void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); } @@ -50,7 +50,7 @@ private: } void ReplyWithResult(StatusIds::StatusCode status, const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); + Request_->ReplyWithYdbStatus(status); Die(ctx); } }; diff --git a/ydb/core/grpc_services/rpc_create_session.cpp b/ydb/core/grpc_services/rpc_create_session.cpp index e061e2bf0d..2f6e9fae26 100644 --- a/ydb/core/grpc_services/rpc_create_session.cpp +++ b/ydb/core/grpc_services/rpc_create_session.cpp @@ -1,112 +1,112 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" -#include "rpc_common.h" -#include "rpc_kqp_base.h" - +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" +#include "rpc_common.h" +#include "rpc_kqp_base.h" + #include <ydb/core/grpc_services/local_rpc/local_rpc.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; -using namespace NKqp; - +using namespace NKqp; + class TCreateSessionRPC : public TRpcKqpRequestActor<TCreateSessionRPC, TEvCreateSessionRequest> { using TBase = TRpcKqpRequestActor<TCreateSessionRPC, TEvCreateSessionRequest>; -public: +public: TCreateSessionRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext& ctx) { + + void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - Become(&TCreateSessionRPC::StateWork); - - auto now = TInstant::Now(); - - if (Request().GetDeadline() <= now) { - LOG_WARN_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, - SelfId() << " Request deadline has expired for " << now - Request().GetDeadline() << " seconds"); - - Reply(Ydb::StatusIds::TIMEOUT, ctx); - return; - } - - auto selfId = this->SelfId(); - auto as = TActivationContext::ActorSystem(); - - Request_->SetClientLostAction([selfId, as]() { - as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::WakeupTagClientLost)); - }); - - CreateSessionImpl(); - } - -private: - void CreateSessionImpl() { + Become(&TCreateSessionRPC::StateWork); + + auto now = TInstant::Now(); + + if (Request().GetDeadline() <= now) { + LOG_WARN_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, + SelfId() << " Request deadline has expired for " << now - Request().GetDeadline() << " seconds"); + + Reply(Ydb::StatusIds::TIMEOUT, ctx); + return; + } + + auto selfId = this->SelfId(); + auto as = TActivationContext::ActorSystem(); + + Request_->SetClientLostAction([selfId, as]() { + as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::WakeupTagClientLost)); + }); + + CreateSessionImpl(); + } + +private: + void CreateSessionImpl() { const auto traceId = Request().GetTraceId(); auto ev = MakeHolder<NKqp::TEvKqp::TEvCreateSessionRequest>(); - - ev->Record.SetDeadlineUs(Request().GetDeadline().MicroSeconds()); - + + ev->Record.SetDeadlineUs(Request().GetDeadline().MicroSeconds()); + if (traceId) { ev->Record.SetTraceId(traceId.GetRef()); } - + SetDatabase(ev, Request()); - Send(NKqp::MakeKqpProxyID(SelfId().NodeId()), ev.Release()); + Send(NKqp::MakeKqpProxyID(SelfId().NodeId()), ev.Release()); } void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle); - HFunc(TEvents::TEvWakeup, Handle); - // Overide default forget action which terminate this actor on client disconnect - hFunc(TRpcServices::TEvForgetOperation, HandleForget); + switch (ev->GetTypeRewrite()) { + HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle); + HFunc(TEvents::TEvWakeup, Handle); + // Overide default forget action which terminate this actor on client disconnect + hFunc(TRpcServices::TEvForgetOperation, HandleForget); default: TBase::StateWork(ev, ctx); - } - } - - void HandleForget(TRpcServices::TEvForgetOperation::TPtr &ev) { - Y_UNUSED(ev); - } - - void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) { - switch ((EWakeupTag) ev->Get()->Tag) { - case EWakeupTag::WakeupTagClientLost: - return HandleClientLost(); - default: TBase::HandleWakeup(ev, ctx); - } - } - - void HandleClientLost() { - ClientLost = true; - } - - void DoCloseSession(const TActorContext& ctx, const TString& sessionId) { - Ydb::Table::DeleteSessionRequest request; - request.set_session_id(sessionId); - - auto cb = [](const Ydb::Table::DeleteSessionResponse&){}; - - auto database = Request_->GetDatabaseName().GetOrElse(""); - - auto actorId = NRpcService::DoLocalRpcSameMailbox<NKikimr::NGRpcService::TEvDeleteSessionRequest>( - std::move(request), std::move(cb), database, Request_->GetInternalToken(), ctx); - - LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, - SelfId() << " Client lost, session " << sessionId << " will be closed by " << actorId); - } - - void Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; + } + } + + void HandleForget(TRpcServices::TEvForgetOperation::TPtr &ev) { + Y_UNUSED(ev); + } + + void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) { + switch ((EWakeupTag) ev->Get()->Tag) { + case EWakeupTag::WakeupTagClientLost: + return HandleClientLost(); + default: TBase::HandleWakeup(ev, ctx); + } + } + + void HandleClientLost() { + ClientLost = true; + } + + void DoCloseSession(const TActorContext& ctx, const TString& sessionId) { + Ydb::Table::DeleteSessionRequest request; + request.set_session_id(sessionId); + + auto cb = [](const Ydb::Table::DeleteSessionResponse&){}; + + auto database = Request_->GetDatabaseName().GetOrElse(""); + + auto actorId = NRpcService::DoLocalRpcSameMailbox<NKikimr::NGRpcService::TEvDeleteSessionRequest>( + std::move(request), std::move(cb), database, Request_->GetInternalToken(), ctx); + + LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, + SelfId() << " Client lost, session " << sessionId << " will be closed by " << actorId); + } + + void Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record; if (record.GetResourceExhausted()) { Request().ReplyWithRpcStatus(grpc::StatusCode::RESOURCE_EXHAUSTED, record.GetError()); Die(ctx); @@ -114,34 +114,34 @@ private: } if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { - const auto& kqpResponse = record.GetResponse(); - Ydb::Table::CreateSessionResult result; - if (ClientLost) { - DoCloseSession(ctx, kqpResponse.GetSessionId()); - // We already lost the client, so the client should not see this status - Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); - } else { - result.set_session_id(kqpResponse.GetSessionId()); - return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); - } - } else { + const auto& kqpResponse = record.GetResponse(); + Ydb::Table::CreateSessionResult result; + if (ClientLost) { + DoCloseSession(ctx, kqpResponse.GetSessionId()); + // We already lost the client, so the client should not see this status + Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + } else { + result.set_session_id(kqpResponse.GetSessionId()); + return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); + } + } else { return OnQueryResponseError(record, ctx); - } - } -private: - - bool ClientLost = false; - -}; - -void TGRpcRequestProxy::Handle(TEvCreateSessionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TCreateSessionRPC(ev->Release().Release())); -} - + } + } +private: + + bool ClientLost = false; + +}; + +void TGRpcRequestProxy::Handle(TEvCreateSessionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TCreateSessionRPC(ev->Release().Release())); +} + template<> IActor* TEvCreateSessionRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { return new TCreateSessionRPC(msg); } -} // namespace NGRpcService -} // namespace NKikimr +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_create_table.cpp b/ydb/core/grpc_services/rpc_create_table.cpp index 1d8495f490..302dfde57a 100644 --- a/ydb/core/grpc_services/rpc_create_table.cpp +++ b/ydb/core/grpc_services/rpc_create_table.cpp @@ -1,34 +1,34 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include "table_profiles.h" #include "table_settings.h" - + #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/console_config.pb.h> #include <ydb/core/ydb_convert/column_families.h> #include <ydb/core/ydb_convert/table_description.h> -namespace NKikimr { -namespace NGRpcService { - +namespace NKikimr { +namespace NGRpcService { + using namespace NSchemeShard; -using namespace NActors; +using namespace NActors; using namespace NConsole; using namespace Ydb; -using namespace Ydb::Table; - +using namespace Ydb::Table; + class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreateTableRequest> { using TBase = TRpcSchemeRequestActor<TCreateTableRPC, TEvCreateTableRequest>; -public: - TCreateTableRPC(TEvCreateTableRequest* msg) +public: + TCreateTableRPC(TEvCreateTableRequest* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); SendConfigRequest(ctx); @@ -57,10 +57,10 @@ private: LOG_CRIT_S(ctx, NKikimrServices::GRPC_PROXY, "TCreateTableRPC: cannot deliver config request to Configs Dispatcher" " (empty default profile is available only)"); - SendProposeRequest(ctx); - Become(&TCreateTableRPC::StateWork); - } - + SendProposeRequest(ctx); + Become(&TCreateTableRPC::StateWork); + } + void Handle(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr &ev, const TActorContext &ctx) { auto &config = ev->Get()->Config->GetTableProfilesConfig(); Profiles.Load(config); @@ -98,62 +98,62 @@ private: ); } - void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { + void SendProposeRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return Reply(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - if (!req->columnsSize()) { - auto issue = NYql::TIssue("At least one column shoult be in table"); - Request_->RaiseIssue(issue); - return Reply(StatusIds::BAD_REQUEST, ctx); - } - - if (!req->primary_keySize()) { - auto issue = NYql::TIssue("At least one primary key should be specified"); - Request_->RaiseIssue(issue); - return Reply(StatusIds::BAD_REQUEST, ctx); - } - + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return Reply(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + if (!req->columnsSize()) { + auto issue = NYql::TIssue("At least one column shoult be in table"); + Request_->RaiseIssue(issue); + return Reply(StatusIds::BAD_REQUEST, ctx); + } + + if (!req->primary_keySize()) { + auto issue = NYql::TIssue("At least one primary key should be specified"); + Request_->RaiseIssue(issue); + return Reply(StatusIds::BAD_REQUEST, ctx); + } + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); NKikimrSchemeOp::TTableDescription* tableDesc = nullptr; - if (req->indexesSize()) { + if (req->indexesSize()) { modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable); - tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription(); - } else { + tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription(); + } else { modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); - tableDesc = modifyScheme->MutableCreateTable(); - } - - tableDesc->SetName(name); + tableDesc = modifyScheme->MutableCreateTable(); + } + + tableDesc->SetName(name); StatusIds::StatusCode code = StatusIds::SUCCESS; TString error; - if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) { + if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); return Reply(code, issues, ctx); - } - + } + tableDesc->MutableKeyColumnNames()->CopyFrom(req->primary_key()); - + if (!FillIndexDescription(*modifyScheme->MutableCreateIndexedTable(), *req, code, error)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); return Reply(code, issues, ctx); - } - + } + bool tableProfileSet = false; if (req->has_profile()) { const auto& profile = req->profile(); @@ -162,11 +162,11 @@ private: || profile.has_caching_policy(); } - if (!Profiles.ApplyTableProfile(req->profile(), *tableDesc, code, error)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); - } + if (!Profiles.ApplyTableProfile(req->profile(), *tableDesc, code, error)) { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } TColumnFamilyManager families(tableDesc->MutablePartitionConfig()); @@ -219,16 +219,16 @@ private: ); } - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } private: TTableProfiles Profiles; -}; - -void TGRpcRequestProxy::Handle(TEvCreateTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TCreateTableRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr +}; + +void TGRpcRequestProxy::Handle(TEvCreateTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TCreateTableRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_deferrable.h b/ydb/core/grpc_services/rpc_deferrable.h index 5c04985b02..644e7b84c4 100644 --- a/ydb/core/grpc_services/rpc_deferrable.h +++ b/ydb/core/grpc_services/rpc_deferrable.h @@ -1,9 +1,9 @@ -#pragma once +#pragma once #include "defs.h" #include "grpc_request_proxy.h" #include "rpc_common.h" - + #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/base/tablet_pipe.h> @@ -15,10 +15,10 @@ #include <ydb/core/actorlib_impl/long_timer.h> #include <library/cpp/actors/core/actor_bootstrapped.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + template <typename TDerived, typename TRequest, bool IsOperation> class TRpcRequestWithOperationParamsActor : public TActorBootstrapped<TDerived> { private: @@ -30,17 +30,17 @@ public: WakeupTagTimeout = 10, WakeupTagCancel = 11, WakeupTagGetConfig = 21, - WakeupTagClientLost = 22, + WakeupTagClientLost = 22, }; public: TRpcRequestWithOperationParamsActor(TRequestBase* request) : Request_(request) { - auto& operationParams = GetProtoRequest()->operation_params(); + auto& operationParams = GetProtoRequest()->operation_params(); OperationTimeout_ = GetDuration(operationParams.operation_timeout()); CancelAfter_ = GetDuration(operationParams.cancel_after()); - ReportCostInfo_ = operationParams.report_cost_info() == Ydb::FeatureFlag::ENABLED; + ReportCostInfo_ = operationParams.report_cost_info() == Ydb::FeatureFlag::ENABLED; } const typename TRequest::TRequest* GetProtoRequest() const { @@ -64,8 +64,8 @@ public: CancelAfterTimer = CreateLongTimer(ctx, CancelAfter_, new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup(WakeupTagCancel)), AppData(ctx)->UserPoolId); - } - + } + auto selfId = ctx.SelfID; auto* actorSystem = ctx.ExecutorThread.ActorSystem; auto clientLostCb = [selfId, actorSystem]() { @@ -86,8 +86,8 @@ public: protected: TDuration GetOperationTimeout() { return OperationTimeout_; - } - + } + TDuration GetCancelAfter() { return CancelAfter_; } @@ -113,13 +113,13 @@ protected: protected: std::unique_ptr<TRequestBase> Request_; - - TActorId OperationTimeoutTimer; - TActorId CancelAfterTimer; + + TActorId OperationTimeoutTimer; + TActorId CancelAfterTimer; TDuration OperationTimeout_; TDuration CancelAfter_; bool HasCancel_ = false; - bool ReportCostInfo_ = false; + bool ReportCostInfo_ = false; }; template <typename TDerived, typename TRequest> @@ -139,8 +139,8 @@ public: void OnCancelOperation(const TActorContext& ctx) { Y_UNUSED(ctx); - } - + } + void OnForgetOperation(const TActorContext& ctx) { // No client is waiting for the reply, but we have to issue fake reply // anyway before dying to make Grpc happy. @@ -155,8 +155,8 @@ public: issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Operation timeout.")); Reply(Ydb::StatusIds::TIMEOUT, issues, ctx); - } - + } + protected: void StateFuncBase(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { @@ -169,25 +169,25 @@ protected: << ev->GetTypeRewrite())); return this->Reply(Ydb::StatusIds::INTERNAL_ERROR, issues, ctx); } - } - } - + } + } + protected: using TBase::Request_; void Reply(Ydb::StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx) { - Request_->SendResult(status, message); - this->Die(ctx); - } - + Request_->SendResult(status, message); + this->Die(ctx); + } + void Reply(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { - Request_->RaiseIssues(issues); - Request_->ReplyWithYdbStatus(status); - this->Die(ctx); - } - + Request_->RaiseIssues(issues); + Request_->ReplyWithYdbStatus(status); + this->Die(ctx); + } + void Reply(Ydb::StatusIds::StatusCode status, const TString& message, NKikimrIssues::TIssuesIds::EIssueCode issueCode, const TActorContext& ctx) { NYql::TIssues issues; issues.AddIssue(MakeIssue(issueCode, message)); @@ -202,41 +202,41 @@ protected: void ReplyWithResult(Ydb::StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext &ctx) { - Request_->SendResult(status, message); - this->Die(ctx); - } - - template<typename TResult> - void ReplyWithResult(Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, - const TResult& result, - const TActorContext& ctx) - { - Request_->SendResult(result, status, message); - this->Die(ctx); - } - - template<typename TResult> - void ReplyWithResult(Ydb::StatusIds::StatusCode status, - const TResult& result, - const TActorContext& ctx) { - Request_->SendResult(result, status); - this->Die(ctx); - } - - void ReplyOperation(Ydb::Operations::Operation& operation) - { - Request_->SendOperation(operation); - this->PassAway(); - } - - void SetCost(ui64 ru) { - Request_->SetRuHeader(ru); - if (TBase::ReportCostInfo_) { - Request_->SetCostInfo(ru); - } - } - + Request_->SendResult(status, message); + this->Die(ctx); + } + + template<typename TResult> + void ReplyWithResult(Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, + const TResult& result, + const TActorContext& ctx) + { + Request_->SendResult(result, status, message); + this->Die(ctx); + } + + template<typename TResult> + void ReplyWithResult(Ydb::StatusIds::StatusCode status, + const TResult& result, + const TActorContext& ctx) { + Request_->SendResult(result, status); + this->Die(ctx); + } + + void ReplyOperation(Ydb::Operations::Operation& operation) + { + Request_->SendOperation(operation); + this->PassAway(); + } + + void SetCost(ui64 ru) { + Request_->SetRuHeader(ru); + if (TBase::ReportCostInfo_) { + Request_->SetCostInfo(ru); + } + } + protected: void HandleWakeup(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { switch (ev->Get()->Tag) { @@ -255,7 +255,7 @@ protected: Y_UNUSED(ev); static_cast<TDerived*>(this)->OnForgetOperation(ctx); } -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_delete_session.cpp b/ydb/core/grpc_services/rpc_delete_session.cpp index bb7b3b0dd6..a2245af236 100644 --- a/ydb/core/grpc_services/rpc_delete_session.cpp +++ b/ydb/core/grpc_services/rpc_delete_session.cpp @@ -1,35 +1,35 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" -#include "rpc_kqp_base.h" - +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" +#include "rpc_kqp_base.h" + #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; -using namespace NKqp; - +using namespace NKqp; + class TDeleteSessionRPC : public TRpcKqpRequestActor<TDeleteSessionRPC, TEvDeleteSessionRequest> { using TBase = TRpcKqpRequestActor<TDeleteSessionRPC, TEvDeleteSessionRequest>; -public: - TDeleteSessionRPC(IRequestOpCtx* msg) +public: + TDeleteSessionRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext& ctx) { + + void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - DeleteSessionImpl(ctx); + DeleteSessionImpl(ctx); Become(&TDeleteSessionRPC::StateWork); - } + } -private: - void DeleteSessionImpl(const TActorContext& ctx) { - const auto req = GetProtoRequest(); +private: + void DeleteSessionImpl(const TActorContext& ctx) { + const auto req = GetProtoRequest(); auto ev = MakeHolder<NKqp::TEvKqp::TEvCloseSessionRequest>(); @@ -38,22 +38,22 @@ private: ev->Record.MutableRequest()->SetSessionId(req->session_id()); } else { return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); - } - + } + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); //no respose will be sended, so don't wait for anything - Request_->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); - this->Die(ctx); - } -}; - -void TGRpcRequestProxy::Handle(TEvDeleteSessionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDeleteSessionRPC(ev->Release().Release())); -} - -template<> -IActor* TEvDeleteSessionRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { - return new TDeleteSessionRPC(msg); -} - -} // namespace NGRpcService -} // namespace NKikimr + Request_->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); + this->Die(ctx); + } +}; + +void TGRpcRequestProxy::Handle(TEvDeleteSessionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TDeleteSessionRPC(ev->Release().Release())); +} + +template<> +IActor* TEvDeleteSessionRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { + return new TDeleteSessionRPC(msg); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_describe_coordination_node.cpp b/ydb/core/grpc_services/rpc_describe_coordination_node.cpp index 9d18c180b0..04c16bf32e 100644 --- a/ydb/core/grpc_services/rpc_describe_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_describe_coordination_node.cpp @@ -81,7 +81,7 @@ private: } void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); SetAuthToken(navigateRequest, *Request_); @@ -92,7 +92,7 @@ private: ctx.Send(MakeTxProxyID(), navigateRequest.release()); } - + }; void TGRpcRequestProxy::Handle(TEvDescribeCoordinationNode::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/grpc_services/rpc_describe_path.cpp b/ydb/core/grpc_services/rpc_describe_path.cpp index fbc75717e5..7960f50779 100644 --- a/ydb/core/grpc_services/rpc_describe_path.cpp +++ b/ydb/core/grpc_services/rpc_describe_path.cpp @@ -1,40 +1,40 @@ -#include "service_scheme.h" +#include "service_scheme.h" #include <ydb/core/grpc_services/base/base.h> - + #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/ydb_convert/ydb_convert.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - -using TEvListDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::ListDirectoryRequest, - Ydb::Scheme::ListDirectoryResponse>; - -using TEvDescribePathRequest = TGrpcRequestOperationCall<Ydb::Scheme::DescribePathRequest, - Ydb::Scheme::DescribePathResponse>; - + +using TEvListDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::ListDirectoryRequest, + Ydb::Scheme::ListDirectoryResponse>; + +using TEvDescribePathRequest = TGrpcRequestOperationCall<Ydb::Scheme::DescribePathRequest, + Ydb::Scheme::DescribePathResponse>; + template <typename TDerived, typename TRequest, typename TResult, bool ListChildren = false> class TBaseDescribe : public TRpcSchemeRequestActor<TDerived, TRequest> { using TBase = TRpcSchemeRequestActor<TDerived, TRequest>; -public: - TBaseDescribe(IRequestOpCtx* msg) +public: + TBaseDescribe(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); + SendProposeRequest(ctx); this->Become(&TDerived::StateWork); - } - -private: + } + +private: void SendProposeRequest(const TActorContext &ctx) { const auto req = this->GetProtoRequest(); @@ -47,26 +47,26 @@ private: ctx.Send(MakeTxProxyID(), navigateRequest.release()); } - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); default: TBase::StateWork(ev, ctx); - } - } - + } + } + void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->GetRecord(); - const auto status = record.GetStatus(); + const auto status = record.GetStatus(); - if (record.HasReason()) { - auto issue = NYql::TIssue(record.GetReason()); + if (record.HasReason()) { + auto issue = NYql::TIssue(record.GetReason()); this->Request_->RaiseIssue(issue); - } + } TResult result; - switch (status) { + switch (status) { case NKikimrScheme::StatusSuccess: { - const auto& pathDescription = record.GetPathDescription(); + const auto& pathDescription = record.GetPathDescription(); ConvertDirectoryEntry(pathDescription, result.mutable_self(), true); if constexpr (ListChildren) { for (const auto& child : pathDescription.GetChildren()) { @@ -74,41 +74,41 @@ private: } } return this->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); - } + } case NKikimrScheme::StatusPathDoesNotExist: case NKikimrScheme::StatusSchemeError: { return this->Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } + } case NKikimrScheme::StatusAccessDenied: { return this->Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); - } + } case NKikimrScheme::StatusNotAvailable: { return this->Reply(Ydb::StatusIds::UNAVAILABLE, ctx); } - default: { + default: { return this->Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); - } - } - } + } + } + } }; - + class TListDirectoryRPC : public TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true> { public: using TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true>::TBaseDescribe; }; - + class TDescribePathRPC : public TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult> { public: using TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult>::TBaseDescribe; -}; - -void DoListDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TListDirectoryRPC(p.release())); +}; + +void DoListDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TListDirectoryRPC(p.release())); +} + +void DoDescribePathRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TDescribePathRPC(p.release())); } -void DoDescribePathRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TDescribePathRPC(p.release())); -} - -} // namespace NKikimr -} // namespace NGRpcService +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp index 416dcbb12b..261f10d467 100644 --- a/ydb/core/grpc_services/rpc_describe_table.cpp +++ b/ydb/core/grpc_services/rpc_describe_table.cpp @@ -1,85 +1,85 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/ydb_convert/table_description.h> #include <ydb/core/ydb_convert/ydb_convert.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - + class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDescribeTableRequest> { using TBase = TRpcSchemeRequestActor<TDescribeTableRPC, TEvDescribeTableRequest>; -public: - TDescribeTableRPC(TEvDescribeTableRequest* msg) +public: + TDescribeTableRPC(TEvDescribeTableRequest* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); - Become(&TDescribeTableRPC::StateWork); - } - -private: - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { + SendProposeRequest(ctx); + Become(&TDescribeTableRPC::StateWork); + } + +private: + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); default: TBase::StateWork(ev, ctx); - } - } - + } + } + void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->GetRecord(); - const auto status = record.GetStatus(); - if (record.HasReason()) { - auto issue = NYql::TIssue(record.GetReason()); - Request_->RaiseIssue(issue); - } - Ydb::Table::DescribeTableResult describeTableResult; - switch (status) { + const auto status = record.GetStatus(); + if (record.HasReason()) { + auto issue = NYql::TIssue(record.GetReason()); + Request_->RaiseIssue(issue); + } + Ydb::Table::DescribeTableResult describeTableResult; + switch (status) { case NKikimrScheme::StatusSuccess: { - const auto& pathDescription = record.GetPathDescription(); - Ydb::Scheme::Entry* selfEntry = describeTableResult.mutable_self(); - selfEntry->set_name(pathDescription.GetSelf().GetName()); - selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(pathDescription.GetSelf().GetPathType())); - ConvertDirectoryEntry(pathDescription.GetSelf(), selfEntry, true); - - const auto& tableDescription = pathDescription.GetTable(); - NKikimrMiniKQL::TType splitKeyType; - - try { - FillColumnDescription(describeTableResult, splitKeyType, tableDescription); - } catch (const std::exception& ex) { - LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill column description: %s", ex.what()); - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); - } - - describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames()); - - try { - FillTableBoundary(describeTableResult, tableDescription, splitKeyType); - } catch (const std::exception& ex) { - LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill table boundary: %s", ex.what()); - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); - } - - FillIndexDescription(describeTableResult, tableDescription); - - if (GetProtoRequest()->include_table_stats()) { - FillTableStats(describeTableResult, pathDescription, - GetProtoRequest()->include_partition_stats()); - } - + const auto& pathDescription = record.GetPathDescription(); + Ydb::Scheme::Entry* selfEntry = describeTableResult.mutable_self(); + selfEntry->set_name(pathDescription.GetSelf().GetName()); + selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(pathDescription.GetSelf().GetPathType())); + ConvertDirectoryEntry(pathDescription.GetSelf(), selfEntry, true); + + const auto& tableDescription = pathDescription.GetTable(); + NKikimrMiniKQL::TType splitKeyType; + + try { + FillColumnDescription(describeTableResult, splitKeyType, tableDescription); + } catch (const std::exception& ex) { + LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill column description: %s", ex.what()); + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + } + + describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames()); + + try { + FillTableBoundary(describeTableResult, tableDescription, splitKeyType); + } catch (const std::exception& ex) { + LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill table boundary: %s", ex.what()); + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + } + + FillIndexDescription(describeTableResult, tableDescription); + + if (GetProtoRequest()->include_table_stats()) { + FillTableStats(describeTableResult, pathDescription, + GetProtoRequest()->include_partition_stats()); + } + FillStorageSettings(describeTableResult, tableDescription); FillColumnFamilies(describeTableResult, tableDescription); FillAttributes(describeTableResult, pathDescription); @@ -88,54 +88,54 @@ private: FillReadReplicasSettings(describeTableResult, tableDescription); return ReplyWithResult(Ydb::StatusIds::SUCCESS, describeTableResult, ctx); - } - + } + case NKikimrScheme::StatusPathDoesNotExist: case NKikimrScheme::StatusSchemeError: { return Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } - + } + case NKikimrScheme::StatusAccessDenied: { - return Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); - } - + return Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); + } + case NKikimrScheme::StatusNotAvailable: { return Reply(Ydb::StatusIds::UNAVAILABLE, ctx); } - default: { + default: { return Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); - } - } - } - - void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - - std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); - SetAuthToken(navigateRequest, *Request_); + } + } + } + + void SendProposeRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + + std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); + SetAuthToken(navigateRequest, *Request_); SetDatabase(navigateRequest.get(), *Request_); NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath(); - record->SetPath(req->path()); - if (req->include_shard_key_bounds()) { - record->MutableOptions()->SetReturnBoundaries(true); - } - - if (req->include_partition_stats() && req->include_table_stats()) { - record->MutableOptions()->SetReturnPartitionStats(true); - } - - if (AppData(ctx)->AllowPrivateTableDescribeForTest) { - record->MutableOptions()->SetShowPrivateTable(true); - } - - ctx.Send(MakeTxProxyID(), navigateRequest.release()); - } -}; - -void TGRpcRequestProxy::Handle(TEvDescribeTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDescribeTableRPC(ev->Release().Release())); -} - -} // namespace NKikimr -} // namespace NGRpcService + record->SetPath(req->path()); + if (req->include_shard_key_bounds()) { + record->MutableOptions()->SetReturnBoundaries(true); + } + + if (req->include_partition_stats() && req->include_table_stats()) { + record->MutableOptions()->SetReturnPartitionStats(true); + } + + if (AppData(ctx)->AllowPrivateTableDescribeForTest) { + record->MutableOptions()->SetShowPrivateTable(true); + } + + ctx.Send(MakeTxProxyID(), navigateRequest.release()); + } +}; + +void TGRpcRequestProxy::Handle(TEvDescribeTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TDescribeTableRPC(ev->Release().Release())); +} + +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_describe_table_options.cpp b/ydb/core/grpc_services/rpc_describe_table_options.cpp index 1b5b9d2732..6d77e3cff7 100644 --- a/ydb/core/grpc_services/rpc_describe_table_options.cpp +++ b/ydb/core/grpc_services/rpc_describe_table_options.cpp @@ -172,7 +172,7 @@ private: description.add_allowed_caching_policies(pr.first); } - this->Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + this->Request_->SendResult(result, Ydb::StatusIds::SUCCESS); Die(ctx); } diff --git a/ydb/core/grpc_services/rpc_discovery.cpp b/ydb/core/grpc_services/rpc_discovery.cpp index 9dd0e8394c..75550f92ac 100644 --- a/ydb/core/grpc_services/rpc_discovery.cpp +++ b/ydb/core/grpc_services/rpc_discovery.cpp @@ -318,8 +318,8 @@ public: entries.emplace_back(&xpair.second.Payload); Shuffle(entries.begin(), entries.end()); - auto *result = TEvListEndpointsRequest::AllocateResult<Ydb::Discovery::ListEndpointsResult>(Request); - result->mutable_endpoints()->Reserve(LookupResponse->InfoEntries.size()); + auto *result = TEvListEndpointsRequest::AllocateResult<Ydb::Discovery::ListEndpointsResult>(Request); + result->mutable_endpoints()->Reserve(LookupResponse->InfoEntries.size()); const TSet<TString> services(Request->GetProtoRequest()->Getservice().begin(), Request->GetProtoRequest()->Getservice().end()); const bool sslServer = Request->SslServer(); @@ -398,11 +398,11 @@ public: if (nodeInfo && nodeInfo->Location.GetDataCenterId()) { const auto &location = nodeInfo->Location.GetDataCenterId(); if (IsSafeLocationMarker(location)) - result->set_self_location(location); + result->set_self_location(location); } - Request->SendResult(*result, Ydb::StatusIds::SUCCESS); + Request->SendResult(*result, Ydb::StatusIds::SUCCESS); PassAway(); } @@ -476,11 +476,11 @@ public: } }; -void TGRpcRequestProxy::Handle(TEvListEndpointsRequest::TPtr& ev, const TActorContext& ctx) { +void TGRpcRequestProxy::Handle(TEvListEndpointsRequest::TPtr& ev, const TActorContext& ctx) { if (!DiscoveryCacheActorID) - DiscoveryCacheActorID = ctx.Register(new NDiscoveryPrivate::TDiscoveryCache()); + DiscoveryCacheActorID = ctx.Register(new NDiscoveryPrivate::TDiscoveryCache()); - ctx.Register(new TListEndpointsRPC(ev, DiscoveryCacheActorID)); + ctx.Register(new TListEndpointsRPC(ev, DiscoveryCacheActorID)); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_drop_coordination_node.cpp b/ydb/core/grpc_services/rpc_drop_coordination_node.cpp index d81d106630..3632ec5872 100644 --- a/ydb/core/grpc_services/rpc_drop_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_drop_coordination_node.cpp @@ -26,12 +26,12 @@ public: private: void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { pathPair = SplitPath(req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); } @@ -49,7 +49,7 @@ private: } void ReplyWithResult(StatusIds::StatusCode status, const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); + Request_->ReplyWithYdbStatus(status); Die(ctx); } }; diff --git a/ydb/core/grpc_services/rpc_drop_table.cpp b/ydb/core/grpc_services/rpc_drop_table.cpp index e4a3335fc5..27899a9d77 100644 --- a/ydb/core/grpc_services/rpc_drop_table.cpp +++ b/ydb/core/grpc_services/rpc_drop_table.cpp @@ -1,67 +1,67 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" #include "rpc_scheme_base.h" -#include "rpc_common.h" - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; +#include "rpc_common.h" + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - + class TDropTableRPC : public TRpcSchemeRequestActor<TDropTableRPC, TEvDropTableRequest> { using TBase = TRpcSchemeRequestActor<TDropTableRPC, TEvDropTableRequest>; -public: - TDropTableRPC(IRequestOpCtx* msg) +public: + TDropTableRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); - Become(&TDropTableRPC::StateWork); - } - -private: - void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { - pathPair = SplitPath(req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - + SendProposeRequest(ctx); + Become(&TDropTableRPC::StateWork); + } + +private: + void SendProposeRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { + pathPair = SplitPath(req->path()); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); - auto drop = modifyScheme->MutableDrop(); - drop->SetName(name); - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - - void ReplyWithResult(StatusIds::StatusCode status, const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); - Die(ctx); - } -}; - -void TGRpcRequestProxy::Handle(TEvDropTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDropTableRPC(ev->Release().Release())); -} - -template<> -IActor* TEvDropTableRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { - return new TDropTableRPC(msg); -} - -} // namespace NKikimr -} // namespace NGRpcService + auto drop = modifyScheme->MutableDrop(); + drop->SetName(name); + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + + void ReplyWithResult(StatusIds::StatusCode status, const TActorContext &ctx) { + Request_->ReplyWithYdbStatus(status); + Die(ctx); + } +}; + +void TGRpcRequestProxy::Handle(TEvDropTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TDropTableRPC(ev->Release().Release())); +} + +template<> +IActor* TEvDropTableRequest::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { + return new TDropTableRPC(msg); +} + +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp index 8a9421d813..317a3488ce 100644 --- a/ydb/core/grpc_services/rpc_execute_data_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/base/counters.h> #include <ydb/core/protos/console_config.pb.h> @@ -17,14 +17,14 @@ namespace NGRpcService { using namespace NActors; using namespace NOperationId; using namespace Ydb; -using namespace Ydb::Table; +using namespace Ydb::Table; using namespace NKqp; class TExecuteDataQueryRPC : public TRpcKqpRequestActor<TExecuteDataQueryRPC, TEvExecuteDataQueryRequest> { using TBase = TRpcKqpRequestActor<TExecuteDataQueryRPC, TEvExecuteDataQueryRequest>; public: - using TResult = Ydb::Table::ExecuteQueryResult; + using TResult = Ydb::Table::ExecuteQueryResult; TExecuteDataQueryRPC(TEvExecuteDataQueryRequest* msg) : TBase(msg) {} @@ -44,14 +44,14 @@ public: } void Proceed(const TActorContext& ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); const auto requestType = Request_->GetRequestType(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); @@ -208,14 +208,14 @@ public: void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetRef(); - SetCost(record.GetConsumedRu()); + SetCost(record.GetConsumedRu()); AddServerHintsIfAny(record); - + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - auto queryResult = TEvExecuteDataQueryRequest::AllocateResult<Ydb::Table::ExecuteQueryResult>(Request_); + auto queryResult = TEvExecuteDataQueryRequest::AllocateResult<Ydb::Table::ExecuteQueryResult>(Request_); ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); ConvertQueryStats(kqpResponse, queryResult); if (kqpResponse.HasTxMeta()) { diff --git a/ydb/core/grpc_services/rpc_execute_scheme_query.cpp b/ydb/core/grpc_services/rpc_execute_scheme_query.cpp index 45cac04bd0..6239bd7798 100644 --- a/ydb/core/grpc_services/rpc_execute_scheme_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_scheme_query.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/protos/console_config.pb.h> @@ -38,14 +38,14 @@ public: } void Proceed(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); const auto requestType = Request_->GetRequestType(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); diff --git a/ydb/core/grpc_services/rpc_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_execute_yql_script.cpp index cf655740fa..2650d32df5 100644 --- a/ydb/core/grpc_services/rpc_execute_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_execute_yql_script.cpp @@ -1,16 +1,16 @@ -#include "service_yql_scripting.h" +#include "service_yql_scripting.h" #include "rpc_kqp_base.h" #include "rpc_common.h" #include <ydb/public/api/protos/ydb_scripting.pb.h> - + namespace NKikimr { namespace NGRpcService { -using Ydb::Scripting::ExecuteYqlRequest; -using Ydb::Scripting::ExecuteYqlResponse; -using TEvExecuteYqlScriptRequest = TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse>; - +using Ydb::Scripting::ExecuteYqlRequest; +using Ydb::Scripting::ExecuteYqlResponse; +using TEvExecuteYqlScriptRequest = TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse>; + using namespace Ydb; class TExecuteYqlScriptRPC : public TRpcKqpRequestActor<TExecuteYqlScriptRPC, TEvExecuteYqlScriptRequest> { @@ -19,7 +19,7 @@ class TExecuteYqlScriptRPC : public TRpcKqpRequestActor<TExecuteYqlScriptRPC, TE public: using TResult = Ydb::Scripting::ExecuteYqlResult; - TExecuteYqlScriptRPC(IRequestOpCtx* msg) + TExecuteYqlScriptRPC(IRequestOpCtx* msg) : TBase(msg) {} void Bootstrap(const TActorContext &ctx) { @@ -37,7 +37,7 @@ public: } void Proceed(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); @@ -86,7 +86,7 @@ public: void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetRef(); - SetCost(record.GetConsumedRu()); + SetCost(record.GetConsumedRu()); AddServerHintsIfAny(record); if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { @@ -96,7 +96,7 @@ public: const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - auto queryResult = TEvExecuteYqlScriptRequest::AllocateResult<TResult>(Request_); + auto queryResult = TEvExecuteYqlScriptRequest::AllocateResult<TResult>(Request_); ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); if (kqpResponse.HasQueryStats()) { @@ -109,8 +109,8 @@ public: } }; -void DoExecuteYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TExecuteYqlScriptRPC(p.release())); +void DoExecuteYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TExecuteYqlScriptRPC(p.release())); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_explain_data_query.cpp b/ydb/core/grpc_services/rpc_explain_data_query.cpp index c442aa85aa..cc42fb9235 100644 --- a/ydb/core/grpc_services/rpc_explain_data_query.cpp +++ b/ydb/core/grpc_services/rpc_explain_data_query.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/protos/console_config.pb.h> @@ -40,12 +40,12 @@ public: } void Proceed(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + if (traceId) { ev->Record.SetTraceId(traceId.GetRef()); } @@ -76,7 +76,7 @@ public: const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - Ydb::Table::ExplainQueryResult queryResult; + Ydb::Table::ExplainQueryResult queryResult; queryResult.set_query_ast(kqpResponse.GetQueryAst()); queryResult.set_query_plan(kqpResponse.GetQueryPlan()); diff --git a/ydb/core/grpc_services/rpc_explain_yql_script.cpp b/ydb/core/grpc_services/rpc_explain_yql_script.cpp index d9591df363..1d4667feec 100644 --- a/ydb/core/grpc_services/rpc_explain_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_explain_yql_script.cpp @@ -1,15 +1,15 @@ -#include "service_yql_scripting.h" +#include "service_yql_scripting.h" #include "rpc_kqp_base.h" #include "rpc_common.h" #include <ydb/public/api/protos/ydb_scripting.pb.h> - + namespace NKikimr { namespace NGRpcService { -using TEvExplainYqlScriptRequest = - TGrpcRequestOperationCall<Ydb::Scripting::ExplainYqlRequest, Ydb::Scripting::ExplainYqlResponse>; - +using TEvExplainYqlScriptRequest = + TGrpcRequestOperationCall<Ydb::Scripting::ExplainYqlRequest, Ydb::Scripting::ExplainYqlResponse>; + using namespace Ydb; class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest> { @@ -18,7 +18,7 @@ class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TE public: using TResult = Ydb::Scripting::ExplainYqlResult; - TExplainYqlScriptRPC(IRequestOpCtx* msg) + TExplainYqlScriptRPC(IRequestOpCtx* msg) : TBase(msg) {} void Bootstrap(const TActorContext &ctx) { @@ -36,7 +36,7 @@ public: } void Proceed(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); @@ -104,8 +104,8 @@ public: } }; -void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TExplainYqlScriptRPC(p.release())); +void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TExplainYqlScriptRPC(p.release())); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_forget_operation.cpp b/ydb/core/grpc_services/rpc_forget_operation.cpp index 8d369e1821..8b250125ca 100644 --- a/ydb/core/grpc_services/rpc_forget_operation.cpp +++ b/ydb/core/grpc_services/rpc_forget_operation.cpp @@ -80,9 +80,9 @@ public: const TString& id = Request->GetProtoRequest()->id(); try { - OperationId = TOperationId(id); + OperationId = TOperationId(id); - switch (OperationId.GetKind()) { + switch (OperationId.GetKind()) { case TOperationId::EXPORT: case TOperationId::IMPORT: case TOperationId::BUILD_INDEX: @@ -110,10 +110,10 @@ public: hFunc(TEvIndexBuilder::TEvForgetResponse, Handle); default: return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -private: + } + } + +private: TOperationId OperationId; ui64 RawOperationId = 0; diff --git a/ydb/core/grpc_services/rpc_get_operation.cpp b/ydb/core/grpc_services/rpc_get_operation.cpp index b78245f6f1..6f7a140686 100644 --- a/ydb/core/grpc_services/rpc_get_operation.cpp +++ b/ydb/core/grpc_services/rpc_get_operation.cpp @@ -1,11 +1,11 @@ -#include "grpc_request_proxy.h" +#include "grpc_request_proxy.h" #include "operation_helpers.h" #include "rpc_export_base.h" #include "rpc_import_base.h" #include "rpc_operation_request_base.h" - + #include <google/protobuf/text_format.h> - + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/console.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> @@ -15,21 +15,21 @@ #include <ydb/core/tx/schemeshard/schemeshard_import.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <library/cpp/actors/core/hfunc.h> -#include <util/string/cast.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; +#include <util/string/cast.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NOperationId; using namespace Ydb; - + class TGetOperationRPC : public TRpcOperationRequestActor<TGetOperationRPC, TEvGetOperationRequest, true>, public TExportConv { - + TStringBuf GetLogPrefix() const override { switch (OperationId_.GetKind()) { case TOperationId::EXPORT: @@ -40,9 +40,9 @@ class TGetOperationRPC : public TRpcOperationRequestActor<TGetOperationRPC, TEvG return "[GetIndexBuild]"; default: return "[Untagged]"; - } + } } - + IEventBase* MakeRequest() override { switch (OperationId_.GetKind()) { case TOperationId::EXPORT: @@ -57,24 +57,24 @@ class TGetOperationRPC : public TRpcOperationRequestActor<TGetOperationRPC, TEvG } void PassAway() override { - if (PipeActorId_) { - NTabletPipe::CloseClient(SelfId(), PipeActorId_); - PipeActorId_ = TActorId(); - } - + if (PipeActorId_) { + NTabletPipe::CloseClient(SelfId(), PipeActorId_); + PipeActorId_ = TActorId(); + } + TRpcOperationRequestActor::PassAway(); - } - -public: + } + +public: using TRpcOperationRequestActor::TRpcOperationRequestActor; - void Bootstrap(const TActorContext &ctx) { + void Bootstrap(const TActorContext &ctx) { const auto req = Request->GetProtoRequest(); try { - OperationId_ = TOperationId(req->id()); + OperationId_ = TOperationId(req->id()); - switch (OperationId_.GetKind()) { + switch (OperationId_.GetKind()) { case TOperationId::CMS_REQUEST: SendCheckCmsOperation(ctx); break; @@ -91,61 +91,61 @@ public: break; } } catch (const yexception& ex) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); + return ReplyWithStatus(StatusIds::BAD_REQUEST); } - Become(&TGetOperationRPC::AwaitState); - } - STFUNC(AwaitState) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleResponse); + Become(&TGetOperationRPC::AwaitState); + } + STFUNC(AwaitState) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleResponse); HFunc(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult, Handle); HFunc(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered, Handle); HFunc(NConsole::TEvConsole::TEvGetOperationResponse, Handle); HFunc(NSchemeShard::TEvExport::TEvGetExportResponse, Handle); HFunc(NSchemeShard::TEvImport::TEvGetImportResponse, Handle); HFunc(NSchemeShard::TEvIndexBuilder::TEvGetResponse, Handle); - - default: + + default: return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } -private: + } + } +private: void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr&, const TActorContext& ctx) { - ReplyGetOperationResponse(false, ctx); - } - + ReplyGetOperationResponse(false, ctx); + } + void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr&, const TActorContext& ctx) { - ReplyGetOperationResponse(true, ctx); - } - + ReplyGetOperationResponse(true, ctx); + } + void Handle(NConsole::TEvConsole::TEvGetOperationResponse::TPtr &ev, const TActorContext& ctx) { auto &rec = ev->Get()->Record.GetResponse(); - if (rec.operation().ready()) + if (rec.operation().ready()) ReplyWithError(rec.operation().status(), rec.operation().issues(), ctx); else ReplyGetOperationResponse(false, ctx); } - void HandleResponse(typename TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Y_UNUSED(ctx); - } - + void HandleResponse(typename TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + Y_UNUSED(ctx); + } + void SendCheckCmsOperation(const TActorContext& ctx) { ui64 tid; try { - const auto& cmsIds = OperationId_.GetValue("cmstid"); + const auto& cmsIds = OperationId_.GetValue("cmstid"); if (cmsIds.size() != 1) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); + return ReplyWithStatus(StatusIds::BAD_REQUEST); } if (!TryFromString(*cmsIds[0], tid)) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); + return ReplyWithStatus(StatusIds::BAD_REQUEST); } } catch (const yexception& ex) { Request->RaiseIssue(NYql::ExceptionToIssue(ex)); - return ReplyWithStatus(StatusIds::BAD_REQUEST); + return ReplyWithStatus(StatusIds::BAD_REQUEST); } IActor* pipeActor = NTabletPipe::CreateClient(ctx.SelfID, tid); @@ -158,32 +158,32 @@ private: NTabletPipe::SendData(ctx, PipeActorId_, request.Release()); } - void SendNotifyTxCompletion(const TActorContext& ctx) { - ui64 txId; - ui64 schemeShardTabletId; - try { - const auto& txIds = OperationId_.GetValue("txid"); - const auto& sstIds = OperationId_.GetValue("sstid"); - if (txIds.size() != 1 || sstIds.size() != 1) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); - } - - if (!TryFromString(*txIds[0], txId) || !TryFromString(*sstIds[0], schemeShardTabletId)) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); - } - } catch (const yexception& ex) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); - } - - IActor* pipeActor = NTabletPipe::CreateClient(ctx.SelfID, schemeShardTabletId); - Y_VERIFY(pipeActor); + void SendNotifyTxCompletion(const TActorContext& ctx) { + ui64 txId; + ui64 schemeShardTabletId; + try { + const auto& txIds = OperationId_.GetValue("txid"); + const auto& sstIds = OperationId_.GetValue("sstid"); + if (txIds.size() != 1 || sstIds.size() != 1) { + return ReplyWithStatus(StatusIds::BAD_REQUEST); + } + + if (!TryFromString(*txIds[0], txId) || !TryFromString(*sstIds[0], schemeShardTabletId)) { + return ReplyWithStatus(StatusIds::BAD_REQUEST); + } + } catch (const yexception& ex) { + return ReplyWithStatus(StatusIds::BAD_REQUEST); + } + + IActor* pipeActor = NTabletPipe::CreateClient(ctx.SelfID, schemeShardTabletId); + Y_VERIFY(pipeActor); PipeActorId_ = ctx.ExecutorThread.RegisterActor(pipeActor); - + auto request = MakeHolder<NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion>(); - request->Record.SetTxId(txId); + request->Record.SetTxId(txId); NTabletPipe::SendData(ctx, PipeActorId_, request.Release()); - } - + } + void Handle(NSchemeShard::TEvExport::TEvGetExportResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetResponse(); @@ -197,90 +197,90 @@ private: void Handle(NSchemeShard::TEvImport::TEvGetImportResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetResponse(); - + LOG_D("Handle TEvImport::TEvGetImportResponse" << ": record# " << record.ShortDebugString()); - + TEvGetOperationRequest::TResponse resp; *resp.mutable_operation() = TImportConv::ToOperation(record.GetEntry()); Reply(resp, ctx); - } - + } + void Handle(NSchemeShard::TEvIndexBuilder::TEvGetResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; - + LOG_D("Handle TEvIndexBuilder::TEvGetResponse" << ": record# " << record.ShortDebugString()); - if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { - ReplyGetOperationResponse(true, ctx, record.GetStatus()); - } else { - TEvGetOperationRequest::TResponse resp; - - ::NKikimr::NGRpcService::ToOperation(record.GetIndexBuild(), resp.mutable_operation()); - Reply(resp, ctx); - } - } - - void ReplyWithError(const StatusIds::StatusCode status, + if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { + ReplyGetOperationResponse(true, ctx, record.GetStatus()); + } else { + TEvGetOperationRequest::TResponse resp; + + ::NKikimr::NGRpcService::ToOperation(record.GetIndexBuild(), resp.mutable_operation()); + Reply(resp, ctx); + } + } + + void ReplyWithError(const StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> &issues, const TActorContext &ctx) { - TEvGetOperationRequest::TResponse resp; - auto deferred = resp.mutable_operation(); + TEvGetOperationRequest::TResponse resp; + auto deferred = resp.mutable_operation(); deferred->set_id(Request->GetProtoRequest()->id()); - deferred->set_ready(true); - deferred->set_status(status); + deferred->set_ready(true); + deferred->set_status(status); if (issues.size()) deferred->mutable_issues()->CopyFrom(issues); Reply(resp, ctx); - } - - void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, StatusIds::StatusCode status = StatusIds::SUCCESS) { - TEvGetOperationRequest::TResponse resp; - auto deferred = resp.mutable_operation(); + } + + void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, StatusIds::StatusCode status = StatusIds::SUCCESS) { + TEvGetOperationRequest::TResponse resp; + auto deferred = resp.mutable_operation(); deferred->set_id(Request->GetProtoRequest()->id()); - deferred->set_ready(ready); - if (ready) { - deferred->set_status(status); - } + deferred->set_ready(ready); + if (ready) { + deferred->set_status(status); + } Reply(resp, ctx); } - template<typename TMessage> - void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, - const TMessage& metadata, StatusIds::StatusCode status = StatusIds::SUCCESS) - { - TEvGetOperationRequest::TResponse resp; - auto deferred = resp.mutable_operation(); + template<typename TMessage> + void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, + const TMessage& metadata, StatusIds::StatusCode status = StatusIds::SUCCESS) + { + TEvGetOperationRequest::TResponse resp; + auto deferred = resp.mutable_operation(); deferred->set_id(Request->GetProtoRequest()->id()); - deferred->set_ready(ready); - if (ready) { - deferred->set_status(status); - } - auto data = deferred->mutable_metadata(); - data->PackFrom(metadata); - Reply(resp, ctx); - } - - + deferred->set_ready(ready); + if (ready) { + deferred->set_status(status); + } + auto data = deferred->mutable_metadata(); + data->PackFrom(metadata); + Reply(resp, ctx); + } + + void Reply(const TEvGetOperationRequest::TResponse& response, const TActorContext& ctx) { - TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), Request); - this->Die(ctx); - } - - void ReplyWithStatus(StatusIds::StatusCode status) { - Request->ReplyWithYdbStatus(status); - this->PassAway(); - } - - TOperationId OperationId_; + TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), Request); + this->Die(ctx); + } + + void ReplyWithStatus(StatusIds::StatusCode status) { + Request->ReplyWithYdbStatus(status); + this->PassAway(); + } + + TOperationId OperationId_; ui64 RawOperationId_ = 0; TActorId PipeActorId_; -}; - -void TGRpcRequestProxy::Handle(TEvGetOperationRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TGetOperationRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr +}; + +void TGRpcRequestProxy::Handle(TEvGetOperationRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TGetOperationRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_get_shard_locations.cpp b/ydb/core/grpc_services/rpc_get_shard_locations.cpp index fb46988bb4..f7088c8eb6 100644 --- a/ydb/core/grpc_services/rpc_get_shard_locations.cpp +++ b/ydb/core/grpc_services/rpc_get_shard_locations.cpp @@ -173,7 +173,7 @@ private: void ReplyWithError(StatusIds::StatusCode status, const TString& message, const TActorContext& ctx) { Request->RaiseIssue(NYql::TIssue(message)); - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); Die(ctx); } diff --git a/ydb/core/grpc_services/rpc_keep_alive.cpp b/ydb/core/grpc_services/rpc_keep_alive.cpp index dbb35f122b..e4188c8ab9 100644 --- a/ydb/core/grpc_services/rpc_keep_alive.cpp +++ b/ydb/core/grpc_services/rpc_keep_alive.cpp @@ -1,72 +1,72 @@ -#include "grpc_request_proxy.h" - -#include "rpc_calls.h" -#include "rpc_kqp_base.h" -#include "rpc_common.h" - +#include "grpc_request_proxy.h" + +#include "rpc_calls.h" +#include "rpc_kqp_base.h" +#include "rpc_common.h" + #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; -using namespace NKqp; - +using namespace NKqp; + class TKeepAliveRPC : public TRpcKqpRequestActor<TKeepAliveRPC, TEvKeepAliveRequest> { using TBase = TRpcKqpRequestActor<TKeepAliveRPC, TEvKeepAliveRequest>; -public: - TKeepAliveRPC(TEvKeepAliveRequest* msg) +public: + TKeepAliveRPC(TEvKeepAliveRequest* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext& ctx) { + + void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - KeepAliveImpl(ctx); - Become(&TKeepAliveRPC::StateWork); - } -private: + KeepAliveImpl(ctx); + Become(&TKeepAliveRPC::StateWork); + } +private: void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); HFunc(NKqp::TEvKqp::TEvPingSessionResponse, Handle); default: TBase::StateWork(ev, ctx); - } - } - - void KeepAliveImpl(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - const auto traceId = Request_->GetTraceId(); - + } + } + + void KeepAliveImpl(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + const auto traceId = Request_->GetTraceId(); + auto ev = MakeHolder<NKqp::TEvKqp::TEvPingSessionRequest>(); - + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); - } else { + } else { return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); - } - - if (traceId) { - ev->Record.SetTraceId(traceId.GetRef()); - } - + } + + if (traceId) { + ev->Record.SetTraceId(traceId.GetRef()); + } + ev->Record.MutableRequest()->SetTimeoutMs(GetOperationTimeout().MilliSeconds()); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); - } - + } + void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { Ydb::Table::KeepAliveResult result; ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); - } else { + } else { return OnProcessError(record, ctx); - } - } - + } + } + void Handle(NKqp::TEvKqp::TEvPingSessionResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; @@ -83,13 +83,13 @@ private: const TActorContext& ctx) { Request().SendResult(result, status); - Die(ctx); - } -}; - -void TGRpcRequestProxy::Handle(TEvKeepAliveRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TKeepAliveRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + Die(ctx); + } +}; + +void TGRpcRequestProxy::Handle(TEvKeepAliveRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TKeepAliveRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_kh_describe.cpp b/ydb/core/grpc_services/rpc_kh_describe.cpp index f530be43c1..e2c48213ce 100644 --- a/ydb/core/grpc_services/rpc_kh_describe.cpp +++ b/ydb/core/grpc_services/rpc_kh_describe.cpp @@ -316,7 +316,7 @@ private: void ReplyWithError(StatusIds::StatusCode status, const TString& message, const TActorContext& ctx) { Finished = true; Request->RaiseIssue(NYql::TIssue(message)); - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); // We cannot Die() while scheme cache request is in flight because that request has pointer to // KeyRange member so we must not destroy it before we get the response diff --git a/ydb/core/grpc_services/rpc_kh_snapshots.cpp b/ydb/core/grpc_services/rpc_kh_snapshots.cpp index 5fdd94a324..c390c81fae 100644 --- a/ydb/core/grpc_services/rpc_kh_snapshots.cpp +++ b/ydb/core/grpc_services/rpc_kh_snapshots.cpp @@ -71,7 +71,7 @@ public: void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - const auto* proto = GetProtoRequest(); + const auto* proto = GetProtoRequest(); if (proto->path_size() <= 0) { NYql::TIssues issues; @@ -96,7 +96,7 @@ public: auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); req->Record.SetExecTimeoutPeriod(reqTimeout.MilliSeconds()); - auto token = Request_->GetInternalToken(); + auto token = Request_->GetInternalToken(); if (!token.empty()) { req->Record.SetUserToken(token); } @@ -200,7 +200,7 @@ public: void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - const auto* proto = GetProtoRequest(); + const auto* proto = GetProtoRequest(); if (!SnapshotId.Parse(proto->snapshot_id())) { NYql::TIssues issues; @@ -232,7 +232,7 @@ public: auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); req->Record.SetExecTimeoutPeriod(reqTimeout.MilliSeconds()); - auto token = Request_->GetInternalToken(); + auto token = Request_->GetInternalToken(); if (!token.empty()) { req->Record.SetUserToken(token); } @@ -341,7 +341,7 @@ public: void Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); - const auto* proto = GetProtoRequest(); + const auto* proto = GetProtoRequest(); if (!SnapshotId.Parse(proto->snapshot_id())) { NYql::TIssues issues; @@ -373,7 +373,7 @@ public: auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); req->Record.SetExecTimeoutPeriod(reqTimeout.MilliSeconds()); - auto token = Request_->GetInternalToken(); + auto token = Request_->GetInternalToken(); if (!token.empty()) { req->Record.SetUserToken(token); } diff --git a/ydb/core/grpc_services/rpc_kqp_base.cpp b/ydb/core/grpc_services/rpc_kqp_base.cpp index 40d0b22165..414543f574 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.cpp +++ b/ydb/core/grpc_services/rpc_kqp_base.cpp @@ -1,4 +1,4 @@ -#include "rpc_kqp_base.h" +#include "rpc_kqp_base.h" #include <ydb/core/kqp/prepare/kqp_query_plan.h> @@ -43,11 +43,11 @@ void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::T toTable.set_partitions_count(table.GetAffectedPartitions()); } - std::sort(toPhase.mutable_table_access()->begin(), toPhase.mutable_table_access()->end(), - [](const Ydb::TableStats::TableAccessStats& a, const Ydb::TableStats::TableAccessStats& b) { - return a.name() < b.name(); - }); - + std::sort(toPhase.mutable_table_access()->begin(), toPhase.mutable_table_access()->end(), + [](const Ydb::TableStats::TableAccessStats& a, const Ydb::TableStats::TableAccessStats& b) { + return a.name() < b.name(); + }); + NKqpProto::TKqpExecutionExtraStats executionExtraStats; if (exec.HasExtra() && exec.GetExtra().UnpackTo(&executionExtraStats)) { toPhase.set_affected_shards(executionExtraStats.GetAffectedShards()); diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h index e56f9b4b3c..2811a49e78 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.h +++ b/ydb/core/grpc_services/rpc_kqp_base.h @@ -1,33 +1,33 @@ -#pragma once +#pragma once #include "defs.h" - -#include "rpc_deferrable.h" - + +#include "rpc_deferrable.h" + #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/kqp/kqp.h> #include <ydb/core/ydb_convert/ydb_convert.h> #include <ydb/public/lib/operation_id/operation_id.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - -namespace NKikimr { -namespace NGRpcService { - - -inline TString DecodePreparedQueryId(const TString& in) { - if (in.empty()) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "got empty preparedQueryId message"; - } + +namespace NKikimr { +namespace NGRpcService { + + +inline TString DecodePreparedQueryId(const TString& in) { + if (in.empty()) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "got empty preparedQueryId message"; + } NOperationId::TOperationId opId(in); - const auto& ids = opId.GetValue("id"); - if (ids.size() != 1) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "expected exactly one preparedQueryId identifier"; - } - return *ids[0]; -} - + const auto& ids = opId.GetValue("id"); + if (ids.size() != 1) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "expected exactly one preparedQueryId identifier"; + } + return *ids[0]; +} + inline TString GetTransactionModeName(const Ydb::Table::TransactionSettings& settings) { switch (settings.tx_mode_case()) { case Ydb::Table::TransactionSettings::kSerializableReadWrite: @@ -83,8 +83,8 @@ inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, columnMeta->set_name(column.GetName()); columnTypes.push_back(column.GetType()); ConvertMiniKQLTypeToYdbType(column.GetType(), *columnMeta->mutable_type()); - } - } + } + } } for (const auto& responseStruct : from.GetValue().GetStruct()) { for (const auto& row : responseStruct.GetList()) { @@ -94,14 +94,14 @@ inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, for (ui32 i = 0; i < columnCount; i++) { const auto& column = row.GetStruct(i); ConvertMiniKQLValueToYdbValue(columnTypes[i], column, *newRow->add_items()); - } - } + } + } if (responseStruct.Getvalue_valueCase() == NKikimrMiniKQL::TValue::kBool) { to->set_truncated(responseStruct.GetBool()); } - } -} - + } +} + template<typename TFrom, typename TTo> inline void ConvertKqpQueryResultsToDbResult(const TFrom& from, TTo* to) { const auto& results = from.GetResults(); @@ -110,19 +110,19 @@ inline void ConvertKqpQueryResultsToDbResult(const TFrom& from, TTo* to) { } } -template <typename TDerived, typename TRequest> +template <typename TDerived, typename TRequest> class TRpcKqpRequestActor : public TRpcOperationRequestActor<TDerived, TRequest> { using TBase = TRpcOperationRequestActor<TDerived, TRequest>; - + public: - TRpcKqpRequestActor(IRequestOpCtx* request) + TRpcKqpRequestActor(IRequestOpCtx* request) : TBase(request) {} void OnOperationTimeout(const TActorContext& ctx) { Y_UNUSED(ctx); } -protected: +protected: void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); @@ -130,7 +130,7 @@ protected: } } - template<typename TKqpResponse> + template<typename TKqpResponse> void AddServerHintsIfAny(const TKqpResponse& kqpResponse) { if (kqpResponse.GetWorkerIsClosing()) { this->Request_->AddServerHint(TString(NYdb::YDB_SESSION_CLOSE)); @@ -139,42 +139,42 @@ protected: template<typename TKqpResponse> void OnGenericQueryResponseError(const TKqpResponse& kqpResponse, const TActorContext& ctx) { - RaiseIssuesFromKqp(kqpResponse); + RaiseIssuesFromKqp(kqpResponse); - this->Request_->ReplyWithYdbStatus(kqpResponse.GetYdbStatus()); - this->Die(ctx); - } - - template<typename TKqpResponse> + this->Request_->ReplyWithYdbStatus(kqpResponse.GetYdbStatus()); + this->Die(ctx); + } + + template<typename TKqpResponse> void OnQueryResponseErrorWithTxMeta(const TKqpResponse& kqpResponse, const TActorContext& ctx) { - RaiseIssuesFromKqp(kqpResponse); - - auto queryResult = TRequest::template AllocateResult<typename TDerived::TResult>(this->Request_); + RaiseIssuesFromKqp(kqpResponse); + + auto queryResult = TRequest::template AllocateResult<typename TDerived::TResult>(this->Request_); if (kqpResponse.GetResponse().HasTxMeta()) { queryResult->mutable_tx_meta()->CopyFrom(kqpResponse.GetResponse().GetTxMeta()); } - this->Request_->SendResult(*queryResult, kqpResponse.GetYdbStatus()); + this->Request_->SendResult(*queryResult, kqpResponse.GetYdbStatus()); this->Die(ctx); - } - + } + void OnQueryResponseError(const NKikimrKqp::TEvCreateSessionResponse& kqpResponse, const TActorContext& ctx) { if (kqpResponse.HasError()) { - NYql::TIssues issues; + NYql::TIssues issues; issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, kqpResponse.GetError())); return this->Reply(kqpResponse.GetYdbStatus(), issues, ctx); - } else { + } else { return this->Reply(kqpResponse.GetYdbStatus(), ctx); - } - } - + } + } + template<typename TKqpResponse> void OnKqpError(const TKqpResponse& response, const TActorContext& ctx) { NYql::TIssues issues; NYql::IssuesFromMessage(response.GetIssues(), issues); this->Request_->RaiseIssues(issues); - this->Request_->ReplyWithYdbStatus(response.GetStatus()); + this->Request_->ReplyWithYdbStatus(response.GetStatus()); this->Die(ctx); } @@ -189,24 +189,24 @@ protected: } private: - void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - NYql::TIssues issues; + void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + NYql::TIssues issues; if (record.HasError()) { issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, record.GetError())); - } - return this->Reply(record.GetYdbStatus(), issues, ctx); - } + } + return this->Reply(record.GetYdbStatus(), issues, ctx); + } -private: - template<typename TKqpResponse> - void RaiseIssuesFromKqp(const TKqpResponse& kqpResponse) { +private: + template<typename TKqpResponse> + void RaiseIssuesFromKqp(const TKqpResponse& kqpResponse) { NYql::TIssues issues; const auto& issueMessage = kqpResponse.GetResponse().GetQueryIssues(); NYql::IssuesFromMessage(issueMessage, issues); this->Request_->RaiseIssues(issues); - } -}; - -} // namespace NGRpcService -} // namespace NKikimr + } +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_list_operations.cpp b/ydb/core/grpc_services/rpc_list_operations.cpp index 7a3bce3f48..a905efd8d6 100644 --- a/ydb/core/grpc_services/rpc_list_operations.cpp +++ b/ydb/core/grpc_services/rpc_list_operations.cpp @@ -1,5 +1,5 @@ #include "grpc_request_proxy.h" -#include "operation_helpers.h" +#include "operation_helpers.h" #include "rpc_export_base.h" #include "rpc_import_base.h" #include "rpc_calls.h" @@ -34,9 +34,9 @@ class TListOperationsRPC: public TRpcOperationRequestActor<TListOperationsRPC, T return "[ListIndexBuilds]"; default: return "[Untagged]"; - } - } - + } + } + IEventBase* MakeRequest() override { const auto& request = *Request->GetProtoRequest(); @@ -54,40 +54,40 @@ class TListOperationsRPC: public TRpcOperationRequestActor<TListOperationsRPC, T void Handle(TEvExport::TEvListExportsResponse::TPtr& ev) { const auto& record = ev->Get()->Record.GetResponse(); - + LOG_D("Handle TEvExport::TEvListExportsResponse" << ": record# " << record.ShortDebugString()); - + TResponse response; response.set_status(record.GetStatus()); if (record.GetIssues().size()) { response.mutable_issues()->CopyFrom(record.GetIssues()); - } + } for (const auto& entry : record.GetEntries()) { *response.add_operations() = TExportConv::ToOperation(entry); - } + } response.set_next_page_token(record.GetNextPageToken()); Reply(response); } - + void Handle(TEvImport::TEvListImportsResponse::TPtr& ev) { const auto& record = ev->Get()->Record.GetResponse(); - + LOG_D("Handle TEvImport::TEvListImportsResponse" << ": record# " << record.ShortDebugString()); - - TResponse response; - response.set_status(record.GetStatus()); - if (record.GetIssues().size()) { - response.mutable_issues()->CopyFrom(record.GetIssues()); - } - for (const auto& entry : record.GetEntries()) { + + TResponse response; + response.set_status(record.GetStatus()); + if (record.GetIssues().size()) { + response.mutable_issues()->CopyFrom(record.GetIssues()); + } + for (const auto& entry : record.GetEntries()) { *response.add_operations() = TImportConv::ToOperation(entry); - } - response.set_next_page_token(record.GetNextPageToken()); - Reply(response); - } - + } + response.set_next_page_token(record.GetNextPageToken()); + Reply(response); + } + void Handle(TEvIndexBuilder::TEvListResponse::TPtr& ev) { const auto& record = ev->Get()->Record; diff --git a/ydb/core/grpc_services/rpc_load_rows.cpp b/ydb/core/grpc_services/rpc_load_rows.cpp index 851b7306b0..127347e067 100644 --- a/ydb/core/grpc_services/rpc_load_rows.cpp +++ b/ydb/core/grpc_services/rpc_load_rows.cpp @@ -9,12 +9,12 @@ #include <ydb/library/yql/minikql/dom/json.h> #include <ydb/library/yql/utils/utf8.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> - + #include <ydb/library/binary_json/write.h> #include <ydb/library/dynumber/dynumber.h> #include <util/string/vector.h> -#include <util/generic/size_literals.h> +#include <util/generic/size_literals.h> namespace NKikimr { namespace NGRpcService { @@ -22,8 +22,8 @@ namespace NGRpcService { using namespace NActors; using namespace Ydb; -namespace { - +namespace { + bool CheckValueData(NScheme::TTypeId type, const TCell& cell, TString& err) { bool ok = true; switch (type) { @@ -169,8 +169,8 @@ bool ConvertArrowToYdbPrimitive(const arrow::DataType& type, Ydb::Type& toType) break; } return false; -} - +} + } class TUploadRowsRPCPublic : public NTxProxy::TUploadRowsBase<NKikimrServices::TActivity::GRPC_REQ> { @@ -180,7 +180,7 @@ public: : TBase(GetDuration(request->GetProtoRequest()->operation_params().operation_timeout())) , Request(request) {} - + private: static bool CellFromProtoVal(NScheme::TTypeId type, const Ydb::Value* vp, TCell& c, TString& err, TMemoryPool& valueDataPool) @@ -292,91 +292,91 @@ private: return true; } -private: - bool ReportCostInfoEnabled() const { - return Request->GetProtoRequest()->operation_params().report_cost_info() == Ydb::FeatureFlag::ENABLED; - } - +private: + bool ReportCostInfoEnabled() const { + return Request->GetProtoRequest()->operation_params().report_cost_info() == Ydb::FeatureFlag::ENABLED; + } + TString GetDatabase()override { return Request->GetDatabaseName().GetOrElse(DatabaseFromDomain(AppData())); } - const TString& GetTable() override { - return Request->GetProtoRequest()->table(); - } - - const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const override { - return AllRows; - } - - void RaiseIssue(const NYql::TIssue& issue) override { - return Request->RaiseIssue(issue); - } - - void SendResult(const NActors::TActorContext&, const StatusIds::StatusCode& status) override { + const TString& GetTable() override { + return Request->GetProtoRequest()->table(); + } + + const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const override { + return AllRows; + } + + void RaiseIssue(const NYql::TIssue& issue) override { + return Request->RaiseIssue(issue); + } + + void SendResult(const NActors::TActorContext&, const StatusIds::StatusCode& status) override { const Ydb::Table::BulkUpsertResult result; - if (status == StatusIds::SUCCESS) { - ui64 cost = std::ceil(RuCost); - Request->SetRuHeader(cost); - if (ReportCostInfoEnabled()) { - Request->SetCostInfo(cost); - } - } - return Request->SendResult(result, status); - } - - bool CheckAccess(TString& errorMessage) override { - if (Request->GetInternalToken().empty()) - return true; - - NACLib::TUserToken userToken(Request->GetInternalToken()); - const ui32 access = NACLib::EAccessRights::UpdateRow; + if (status == StatusIds::SUCCESS) { + ui64 cost = std::ceil(RuCost); + Request->SetRuHeader(cost); + if (ReportCostInfoEnabled()) { + Request->SetCostInfo(cost); + } + } + return Request->SendResult(result, status); + } + + bool CheckAccess(TString& errorMessage) override { + if (Request->GetInternalToken().empty()) + return true; + + NACLib::TUserToken userToken(Request->GetInternalToken()); + const ui32 access = NACLib::EAccessRights::UpdateRow; auto resolveResult = GetResolveNameResult(); if (!resolveResult) { - TStringStream explanation; - explanation << "Access denied for " << userToken.GetUserSID() + TStringStream explanation; + explanation << "Access denied for " << userToken.GetUserSID() << " table '" << Request->GetProtoRequest()->table() << "' has not been resolved yet"; - - errorMessage = explanation.Str(); - return false; - } + + errorMessage = explanation.Str(); + return false; + } for (const NSchemeCache::TSchemeCacheNavigate::TEntry& entry : resolveResult->ResultSet) { if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok && entry.SecurityObject != nullptr && !entry.SecurityObject->CheckAccess(access, userToken)) - { - TStringStream explanation; - explanation << "Access denied for " << userToken.GetUserSID() - << " with access " << NACLib::AccessRightsToString(access) + { + TStringStream explanation; + explanation << "Access denied for " << userToken.GetUserSID() + << " with access " << NACLib::AccessRightsToString(access) << " to table '" << Request->GetProtoRequest()->table() << "'"; - - errorMessage = explanation.Str(); - return false; - } - } - return true; - } - + + errorMessage = explanation.Str(); + return false; + } + } + return true; + } + TVector<std::pair<TString, Ydb::Type>> GetRequestColumns(TString& errorMessage) const override { Y_UNUSED(errorMessage); - const auto& type = Request->GetProtoRequest()->Getrows().Gettype(); - const auto& rowType = type.Getlist_type(); - const auto& rowFields = rowType.Getitem().Getstruct_type().Getmembers(); - - TVector<std::pair<TString, Ydb::Type>> result; - - for (i32 pos = 0; pos < rowFields.size(); ++pos) { - const auto& name = rowFields[pos].Getname(); - const auto& typeInProto = rowFields[pos].type().has_optional_type() ? - rowFields[pos].type().optional_type().item() : rowFields[pos].type(); - - result.emplace_back(name, typeInProto); - } - return result; - } - + const auto& type = Request->GetProtoRequest()->Getrows().Gettype(); + const auto& rowType = type.Getlist_type(); + const auto& rowFields = rowType.Getitem().Getstruct_type().Getmembers(); + + TVector<std::pair<TString, Ydb::Type>> result; + + for (i32 pos = 0; pos < rowFields.size(); ++pos) { + const auto& name = rowFields[pos].Getname(); + const auto& typeInProto = rowFields[pos].type().has_optional_type() ? + rowFields[pos].type().optional_type().item() : rowFields[pos].type(); + + result.emplace_back(name, typeInProto); + } + return result; + } + bool ExtractRows(TString& errorMessage) override { // Parse type field // Check that it is a list of stuct @@ -395,7 +395,7 @@ private: for (const auto& r : rows) { valueDataPool.Clear(); - ui64 sz = 0; + ui64 sz = 0; // Take members corresponding to key columns if (!FillCellsFromProto(keyCells, KeyColumnPositions, r, errorMessage, valueDataPool)) { return false; @@ -406,16 +406,16 @@ private: return false; } - for (const auto& cell : keyCells) { - sz += cell.Size(); - } - - for (const auto& cell : valueCells) { - sz += cell.Size(); - } - + for (const auto& cell : keyCells) { + sz += cell.Size(); + } + + for (const auto& cell : valueCells) { + sz += cell.Size(); + } + cost += TUpsertCost::OneRowCost(sz); - + // Save serialized key and value TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(keyCells)); TString serializedValue = TSerializedCellVec::Serialize(valueCells); @@ -431,9 +431,9 @@ private: return Batch.get(); } -private: +private: TAutoPtr<TEvBulkUpsertRequest> Request; - TVector<std::pair<TSerializedCellVec, TString>> AllRows; + TVector<std::pair<TSerializedCellVec, TString>> AllRows; }; class TUploadColumnsRPCPublic : public NTxProxy::TUploadRowsBase<NKikimrServices::TActivity::GRPC_REQ> { diff --git a/ydb/core/grpc_services/rpc_log_store.cpp b/ydb/core/grpc_services/rpc_log_store.cpp index 293714fc31..d77f7c7cc0 100644 --- a/ydb/core/grpc_services/rpc_log_store.cpp +++ b/ydb/core/grpc_services/rpc_log_store.cpp @@ -1,4 +1,4 @@ -#include "service_logstore.h" +#include "service_logstore.h" #include "rpc_common.h" #include "rpc_scheme_base.h" @@ -10,28 +10,28 @@ #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/grpc/draft/ydb_logstore_v1.pb.h> - + namespace NKikimr { namespace NGRpcService { using namespace NActors; using namespace Ydb; -using TEvCreateLogStoreRequest = - TGrpcRequestOperationCall<Ydb::LogStore::CreateLogStoreRequest, Ydb::LogStore::CreateLogStoreResponse>; -using TEvDescribeLogStoreRequest = - TGrpcRequestOperationCall<Ydb::LogStore::DescribeLogStoreRequest, Ydb::LogStore::DescribeLogStoreResponse>; -using TEvDropLogStoreRequest = - TGrpcRequestOperationCall<Ydb::LogStore::DropLogStoreRequest, Ydb::LogStore::DropLogStoreResponse>; -using TEvCreateLogTableRequest = - TGrpcRequestOperationCall<Ydb::LogStore::CreateLogTableRequest, Ydb::LogStore::CreateLogTableResponse>; -using TEvDescribeLogTableRequest = - TGrpcRequestOperationCall<Ydb::LogStore::DescribeLogTableRequest, Ydb::LogStore::DescribeLogTableResponse>; -using TEvDropLogTableRequest = - TGrpcRequestOperationCall<Ydb::LogStore::DropLogTableRequest, Ydb::LogStore::DropLogTableResponse>; -using TEvAlterLogTableRequest = - TGrpcRequestOperationCall<Ydb::LogStore::AlterLogTableRequest, Ydb::LogStore::AlterLogTableResponse>; - +using TEvCreateLogStoreRequest = + TGrpcRequestOperationCall<Ydb::LogStore::CreateLogStoreRequest, Ydb::LogStore::CreateLogStoreResponse>; +using TEvDescribeLogStoreRequest = + TGrpcRequestOperationCall<Ydb::LogStore::DescribeLogStoreRequest, Ydb::LogStore::DescribeLogStoreResponse>; +using TEvDropLogStoreRequest = + TGrpcRequestOperationCall<Ydb::LogStore::DropLogStoreRequest, Ydb::LogStore::DropLogStoreResponse>; +using TEvCreateLogTableRequest = + TGrpcRequestOperationCall<Ydb::LogStore::CreateLogTableRequest, Ydb::LogStore::CreateLogTableResponse>; +using TEvDescribeLogTableRequest = + TGrpcRequestOperationCall<Ydb::LogStore::DescribeLogTableRequest, Ydb::LogStore::DescribeLogTableResponse>; +using TEvDropLogTableRequest = + TGrpcRequestOperationCall<Ydb::LogStore::DropLogTableRequest, Ydb::LogStore::DropLogTableResponse>; +using TEvAlterLogTableRequest = + TGrpcRequestOperationCall<Ydb::LogStore::AlterLogTableRequest, Ydb::LogStore::AlterLogTableResponse>; + bool ConvertSchemaFromPublicToInternal(const Ydb::LogStore::Schema& from, NKikimrSchemeOp::TColumnTableSchema& to, Ydb::StatusIds::StatusCode& status, TString& error) { @@ -90,8 +90,8 @@ public: return NKikimrServices::TActivity::GRPC_REQ; } - explicit TCreateLogStoreRPC(IRequestOpCtx* request) - : TBase(request) + explicit TCreateLogStoreRPC(IRequestOpCtx* request) + : TBase(request) {} void Bootstrap(const TActorContext &ctx) { @@ -142,8 +142,8 @@ class TDescribeLogStoreRPC : public TRpcSchemeRequestActor<TDescribeLogStoreRPC, using TBase = TRpcSchemeRequestActor<TDescribeLogStoreRPC, TEvDescribeLogStoreRequest>; public: - TDescribeLogStoreRPC(IRequestOpCtx* request) - : TBase(request) {} + TDescribeLogStoreRPC(IRequestOpCtx* request) + : TBase(request) {} void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); @@ -279,8 +279,8 @@ public: return NKikimrServices::TActivity::GRPC_REQ; } - explicit TCreateLogTableRPC(IRequestOpCtx* request) - : TBase(request) + explicit TCreateLogTableRPC(IRequestOpCtx* request) + : TBase(request) {} void Bootstrap(const TActorContext &ctx) { @@ -342,8 +342,8 @@ class TDescribeLogTableRPC : public TRpcSchemeRequestActor<TDescribeLogTableRPC, using TBase = TRpcSchemeRequestActor<TDescribeLogTableRPC, TEvDescribeLogTableRequest>; public: - TDescribeLogTableRPC(IRequestOpCtx* request) - : TBase(request) {} + TDescribeLogTableRPC(IRequestOpCtx* request) + : TBase(request) {} void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); @@ -460,8 +460,8 @@ public: return NKikimrServices::TActivity::GRPC_REQ; } - explicit TAlterLogTableRPC(IRequestOpCtx* request) - : TBase(request) + explicit TAlterLogTableRPC(IRequestOpCtx* request) + : TBase(request) {} void Bootstrap(const TActorContext &ctx) { @@ -511,33 +511,33 @@ private: using TDropLogStoreRPC = TDropLogRPC<TEvDropLogStoreRequest, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnStore>; using TDropLogTableRPC = TDropLogRPC<TEvDropLogTableRequest, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnTable>; -void DoCreateLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TCreateLogStoreRPC(p.release())); +void DoCreateLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TCreateLogStoreRPC(p.release())); } -void DoDescribeLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TDescribeLogStoreRPC(p.release())); +void DoDescribeLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TDescribeLogStoreRPC(p.release())); } -void DoDropLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TDropLogStoreRPC(p.release())); +void DoDropLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TDropLogStoreRPC(p.release())); } -void DoCreateLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TCreateLogTableRPC(p.release())); +void DoCreateLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TCreateLogTableRPC(p.release())); } -void DoDescribeLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TDescribeLogTableRPC(p.release())); +void DoDescribeLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TDescribeLogTableRPC(p.release())); } -void DoDropLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TDropLogTableRPC(p.release())); +void DoDropLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TDropLogTableRPC(p.release())); } -void DoAlterLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TAlterLogTableRPC(p.release())); +void DoAlterLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TAlterLogTableRPC(p.release())); } } // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_long_tx.cpp b/ydb/core/grpc_services/rpc_long_tx.cpp index 45a6fb8cfd..442972f993 100644 --- a/ydb/core/grpc_services/rpc_long_tx.cpp +++ b/ydb/core/grpc_services/rpc_long_tx.cpp @@ -224,7 +224,7 @@ private: if (!message.empty()) { Request->RaiseIssue(NYql::TIssue(message)); } - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); PassAway(); } @@ -296,7 +296,7 @@ private: if (!message.empty()) { Request->RaiseIssue(NYql::TIssue(message)); } - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); PassAway(); } @@ -630,7 +630,7 @@ protected: if (!message.empty()) { Request->RaiseIssue(NYql::TIssue(message)); } - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); PassAway(); } @@ -927,7 +927,7 @@ private: } Ydb::LongTx::ReadResult* MakeResult(ui64 outChunk, bool finished) const { - auto result = TEvLongTxReadRequest::AllocateResult<Ydb::LongTx::ReadResult>(Request); + auto result = TEvLongTxReadRequest::AllocateResult<Ydb::LongTx::ReadResult>(Request); const auto* req = Request->GetProtoRequest(); result->set_tx_id(req->tx_id()); diff --git a/ydb/core/grpc_services/rpc_make_directory.cpp b/ydb/core/grpc_services/rpc_make_directory.cpp index ba5faf5031..f6e6b2d0cd 100644 --- a/ydb/core/grpc_services/rpc_make_directory.cpp +++ b/ydb/core/grpc_services/rpc_make_directory.cpp @@ -1,68 +1,68 @@ -#include "service_scheme.h" - +#include "service_scheme.h" + #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/grpc_services/base/base.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/public/api/protos/ydb_scheme.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - -using TEvMakeDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::MakeDirectoryRequest, - Ydb::Scheme::MakeDirectoryResponse>; - + +using TEvMakeDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::MakeDirectoryRequest, + Ydb::Scheme::MakeDirectoryResponse>; + class TMakeDirectoryRPC : public TRpcSchemeRequestActor<TMakeDirectoryRPC, TEvMakeDirectoryRequest> { using TBase = TRpcSchemeRequestActor<TMakeDirectoryRPC, TEvMakeDirectoryRequest>; -public: - TMakeDirectoryRPC(IRequestOpCtx* msg) +public: + TMakeDirectoryRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); - Become(&TMakeDirectoryRPC::StateWork); - } -private: - void SendProposeRequest(const TActorContext &ctx) { - - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { + SendProposeRequest(ctx); + Become(&TMakeDirectoryRPC::StateWork); + } +private: + void SendProposeRequest(const TActorContext &ctx) { + + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpMkDir); - modifyScheme->MutableMkDir()->SetName(name); - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - - void ReplyWithResult(const StatusIds::StatusCode status, - const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); - Die(ctx); - } -}; - -void DoMakeDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TMakeDirectoryRPC(p.release())); -} - -} // namespace NGRpcService -} // namespace NKikimr - + modifyScheme->MutableMkDir()->SetName(name); + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + + void ReplyWithResult(const StatusIds::StatusCode status, + const TActorContext &ctx) { + Request_->ReplyWithYdbStatus(status); + Die(ctx); + } +}; + +void DoMakeDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TMakeDirectoryRPC(p.release())); +} + +} // namespace NGRpcService +} // namespace NKikimr + diff --git a/ydb/core/grpc_services/rpc_modify_permissions.cpp b/ydb/core/grpc_services/rpc_modify_permissions.cpp index 2439d41448..d9b01887d2 100644 --- a/ydb/core/grpc_services/rpc_modify_permissions.cpp +++ b/ydb/core/grpc_services/rpc_modify_permissions.cpp @@ -1,104 +1,104 @@ -#include "service_scheme.h" - +#include "service_scheme.h" + #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/ydb_convert/ydb_convert.h> #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/protos/ydb_scheme.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - -using TEvModifyPermissionsRequest = TGrpcRequestOperationCall<Ydb::Scheme::ModifyPermissionsRequest, - Ydb::Scheme::ModifyPermissionsResponse>; - + +using TEvModifyPermissionsRequest = TGrpcRequestOperationCall<Ydb::Scheme::ModifyPermissionsRequest, + Ydb::Scheme::ModifyPermissionsResponse>; + class TModifyPermissionsRPC : public TRpcSchemeRequestActor<TModifyPermissionsRPC, TEvModifyPermissionsRequest> { using TBase = TRpcSchemeRequestActor<TModifyPermissionsRPC, TEvModifyPermissionsRequest>; -public: - TModifyPermissionsRPC(IRequestOpCtx* msg) +public: + TModifyPermissionsRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - try { - SendRequest(ctx); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return Reply(StatusIds::BAD_REQUEST, ctx); - } - - Become(&TThis::StateWork); - } - -private: - void SendRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - - const std::pair<TString, TString> pathPair = SplitPath(req->path()); - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - + try { + SendRequest(ctx); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return Reply(StatusIds::BAD_REQUEST, ctx); + } + + Become(&TThis::StateWork); + } + +private: + void SendRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + + const std::pair<TString, TString> pathPair = SplitPath(req->path()); + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpModifyACL); - modifyScheme->SetWorkingDir(workingDir); - modifyScheme->MutableModifyACL()->SetName(name); - - NACLib::TDiffACL acl; - if (req->clear_permissions()) { - acl.ClearAccess(); - } - for (const auto& perm : req->actions()) { - switch (perm.action_case()) { - case Ydb::Scheme::PermissionsAction::kSet: { - acl.ClearAccessForSid(perm.set().subject()); - for (const auto& n : perm.set().permission_names()) { + modifyScheme->SetWorkingDir(workingDir); + modifyScheme->MutableModifyACL()->SetName(name); + + NACLib::TDiffACL acl; + if (req->clear_permissions()) { + acl.ClearAccess(); + } + for (const auto& perm : req->actions()) { + switch (perm.action_case()) { + case Ydb::Scheme::PermissionsAction::kSet: { + acl.ClearAccessForSid(perm.set().subject()); + for (const auto& n : perm.set().permission_names()) { auto aclAttrs = ConvertYdbPermissionNameToACLAttrs(n); acl.AddAccess(NACLib::EAccessType::Allow, aclAttrs.AccessMask, perm.set().subject(), aclAttrs.InheritanceType); - } - } - break; - case Ydb::Scheme::PermissionsAction::kGrant: { - for (const auto& n : perm.grant().permission_names()) { + } + } + break; + case Ydb::Scheme::PermissionsAction::kGrant: { + for (const auto& n : perm.grant().permission_names()) { auto aclAttrs = ConvertYdbPermissionNameToACLAttrs(n); acl.AddAccess(NACLib::EAccessType::Allow, aclAttrs.AccessMask, perm.grant().subject(), aclAttrs.InheritanceType); - } - } - break; - case Ydb::Scheme::PermissionsAction::kRevoke: { - for (const auto& n : perm.revoke().permission_names()) { + } + } + break; + case Ydb::Scheme::PermissionsAction::kRevoke: { + for (const auto& n : perm.revoke().permission_names()) { auto aclAttrs = ConvertYdbPermissionNameToACLAttrs(n); acl.RemoveAccess(NACLib::EAccessType::Allow, aclAttrs.AccessMask, perm.revoke().subject(), aclAttrs.InheritanceType); - } - } - break; - case Ydb::Scheme::PermissionsAction::kChangeOwner: { - modifyScheme->MutableModifyACL()->SetNewOwner(perm.change_owner()); - } - break; - default: { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Unknown permission action"; - } - } - } - modifyScheme->MutableModifyACL()->SetDiffACL(acl.SerializeAsString()); - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } -}; - -void DoModifyPermissionsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TModifyPermissionsRPC(p.release())); -} - -} // namespace NKikimr -} // namespace NGRpcService + } + } + break; + case Ydb::Scheme::PermissionsAction::kChangeOwner: { + modifyScheme->MutableModifyACL()->SetNewOwner(perm.change_owner()); + } + break; + default: { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Unknown permission action"; + } + } + } + modifyScheme->MutableModifyACL()->SetDiffACL(acl.SerializeAsString()); + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } +}; + +void DoModifyPermissionsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TModifyPermissionsRPC(p.release())); +} + +} // namespace NKikimr +} // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_operation_request_base.h b/ydb/core/grpc_services/rpc_operation_request_base.h index bc6575b10d..697d2109cd 100644 --- a/ydb/core/grpc_services/rpc_operation_request_base.h +++ b/ydb/core/grpc_services/rpc_operation_request_base.h @@ -106,7 +106,7 @@ protected: if (!PipeClient) { NTabletPipe::TClientConfig config; config.RetryPolicy = {.RetryLimitCount = 3}; - PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), schemeShardId, config)); + PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), schemeShardId, config)); } NTabletPipe::SendData(this->SelfId(), PipeClient, MakeRequest()); diff --git a/ydb/core/grpc_services/rpc_prepare_data_query.cpp b/ydb/core/grpc_services/rpc_prepare_data_query.cpp index b69009ae3f..eb4c571877 100644 --- a/ydb/core/grpc_services/rpc_prepare_data_query.cpp +++ b/ydb/core/grpc_services/rpc_prepare_data_query.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/protos/console_config.pb.h> #include <ydb/core/ydb_convert/ydb_convert.h> @@ -41,21 +41,21 @@ public: } void Proceed(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); - const auto requestType = Request_->GetRequestType(); + const auto requestType = Request_->GetRequestType(); auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + if (traceId) { ev->Record.SetTraceId(traceId.GetRef()); } - if (requestType) { - ev->Record.SetRequestType(requestType.GetRef()); - } - + if (requestType) { + ev->Record.SetRequestType(requestType.GetRef()); + } + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); @@ -76,9 +76,9 @@ public: void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record.GetRef(); - SetCost(record.GetConsumedRu()); + SetCost(record.GetConsumedRu()); AddServerHintsIfAny(record); - + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { const auto& kqpResponse = record.GetResponse(); const auto& queryId = kqpResponse.GetPreparedQuery(); @@ -90,7 +90,7 @@ public: AddOptionalValue(opId, "id", queryId); - Ydb::Table::PrepareQueryResult queryResult; + Ydb::Table::PrepareQueryResult queryResult; queryResult.set_query_id(ProtoToString(opId)); for (const auto& queryParameter: queryParameters) { Ydb::Type parameterType; diff --git a/ydb/core/grpc_services/rpc_rate_limiter_api.cpp b/ydb/core/grpc_services/rpc_rate_limiter_api.cpp index 4133081d42..5579be8bdb 100644 --- a/ydb/core/grpc_services/rpc_rate_limiter_api.cpp +++ b/ydb/core/grpc_services/rpc_rate_limiter_api.cpp @@ -20,11 +20,11 @@ class TRateLimiterRequest : public TRpcOperationRequestActor<TDerived, TRequest> public: using TBase = TRpcOperationRequestActor<TDerived, TRequest>; - TRateLimiterRequest(IRequestOpCtx* msg, bool trusted = false) - : TBase(msg) - , TrustedZone(trusted) - {} - + TRateLimiterRequest(IRequestOpCtx* msg, bool trusted = false) + : TBase(msg) + , TrustedZone(trusted) + {} + bool ValidateResource(const Ydb::RateLimiter::Resource& resource, Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) { if (!ValidateResourcePath(resource.resource_path(), status, issues)) { return false; @@ -47,27 +47,27 @@ public: } return true; } - - bool ValidateCoordinationNodePath(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) { - auto databaseName = this->Request_->GetDatabaseName() - .GetOrElse(DatabaseFromDomain(AppData())); - - if (!TrustedZone && !GetCoordinationNodePath().StartsWith(databaseName)) { - status = StatusIds::BAD_REQUEST; - issues.AddIssue("Coordination node path not belong to current database."); - return false; - } - return true; - } - -protected: - const TString& GetCoordinationNodePath() const { - return this->GetProtoRequest()->coordination_node_path(); - } - -private: - const bool TrustedZone; -}; + + bool ValidateCoordinationNodePath(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) { + auto databaseName = this->Request_->GetDatabaseName() + .GetOrElse(DatabaseFromDomain(AppData())); + + if (!TrustedZone && !GetCoordinationNodePath().StartsWith(databaseName)) { + status = StatusIds::BAD_REQUEST; + issues.AddIssue("Coordination node path not belong to current database."); + return false; + } + return true; + } + +protected: + const TString& GetCoordinationNodePath() const { + return this->GetProtoRequest()->coordination_node_path(); + } + +private: + const bool TrustedZone; +}; template <class TEvRequest> class TRateLimiterControlRequest : public TRateLimiterRequest<TRateLimiterControlRequest<TEvRequest>, TEvRequest> { @@ -82,12 +82,12 @@ public: Ydb::StatusIds::StatusCode status = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; NYql::TIssues issues; - - if (!this->ValidateCoordinationNodePath(status, issues)) { - this->Reply(status, issues, TActivationContext::ActorContextFor(this->SelfId())); - return; - } - + + if (!this->ValidateCoordinationNodePath(status, issues)) { + this->Reply(status, issues, TActivationContext::ActorContextFor(this->SelfId())); + return; + } + if (!ValidateRequest(status, issues)) { this->Reply(status, issues, TActivationContext::ActorContextFor(this->SelfId())); return; @@ -108,7 +108,7 @@ protected: } void ResolveCoordinationPath() { - TVector<TString> path = NKikimr::SplitPath(this->GetCoordinationNodePath()); + TVector<TString> path = NKikimr::SplitPath(this->GetCoordinationNodePath()); if (path.empty()) { this->Reply(StatusIds::BAD_REQUEST, "Empty path.", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, TActivationContext::ActorContextFor(this->SelfId())); return; @@ -235,14 +235,14 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { - return ValidateResource(GetProtoRequest()->resource(), status, issues); + return ValidateResource(GetProtoRequest()->resource(), status, issues); } void SendRequest() override { Become(&TCreateRateLimiterResourceRPC::StateFunc); THolder<TEvKesus::TEvAddQuoterResource> req = MakeHolder<TEvKesus::TEvAddQuoterResource>(); - CopyProps(GetProtoRequest()->resource(), *req->Record.MutableResource()); + CopyProps(GetProtoRequest()->resource(), *req->Record.MutableResource()); NTabletPipe::SendData(SelfId(), KesusPipeClient, req.Release(), 0); } @@ -266,14 +266,14 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { - return ValidateResource(GetProtoRequest()->resource(), status, issues); + return ValidateResource(GetProtoRequest()->resource(), status, issues); } void SendRequest() override { Become(&TAlterRateLimiterResourceRPC::StateFunc); THolder<TEvKesus::TEvUpdateQuoterResource> req = MakeHolder<TEvKesus::TEvUpdateQuoterResource>(); - CopyProps(GetProtoRequest()->resource(), *req->Record.MutableResource()); + CopyProps(GetProtoRequest()->resource(), *req->Record.MutableResource()); NTabletPipe::SendData(SelfId(), KesusPipeClient, req.Release(), 0); } @@ -297,14 +297,14 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { - return ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues); + return ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues); } void SendRequest() override { Become(&TDropRateLimiterResourceRPC::StateFunc); THolder<TEvKesus::TEvDeleteQuoterResource> req = MakeHolder<TEvKesus::TEvDeleteQuoterResource>(); - req->Record.SetResourcePath(GetProtoRequest()->resource_path()); + req->Record.SetResourcePath(GetProtoRequest()->resource_path()); NTabletPipe::SendData(SelfId(), KesusPipeClient, req.Release(), 0); } @@ -328,7 +328,7 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { - if (const TString& path = GetProtoRequest()->resource_path()) { + if (const TString& path = GetProtoRequest()->resource_path()) { return ValidateResourcePath(path, status, issues); } return true; @@ -338,10 +338,10 @@ public: Become(&TListRateLimiterResourcesRPC::StateFunc); THolder<TEvKesus::TEvDescribeQuoterResources> req = MakeHolder<TEvKesus::TEvDescribeQuoterResources>(); - if (const TString& path = GetProtoRequest()->resource_path()) { + if (const TString& path = GetProtoRequest()->resource_path()) { req->Record.AddResourcePaths(path); } - req->Record.SetRecursive(GetProtoRequest()->recursive()); + req->Record.SetRecursive(GetProtoRequest()->recursive()); NTabletPipe::SendData(SelfId(), KesusPipeClient, req.Release(), 0); } @@ -375,14 +375,14 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { - return ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues); + return ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues); } void SendRequest() override { Become(&TDescribeRateLimiterResourceRPC::StateFunc); THolder<TEvKesus::TEvDescribeQuoterResources> req = MakeHolder<TEvKesus::TEvDescribeQuoterResources>(); - req->Record.AddResourcePaths(GetProtoRequest()->resource_path()); + req->Record.AddResourcePaths(GetProtoRequest()->resource_path()); NTabletPipe::SendData(SelfId(), KesusPipeClient, req.Release(), 0); } @@ -415,19 +415,19 @@ public: Ydb::StatusIds::StatusCode status = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; NYql::TIssues issues; - - if (!ValidateCoordinationNodePath(status, issues)) { - Reply(status, issues, TActivationContext::AsActorContext()); - return; - } - + + if (!ValidateCoordinationNodePath(status, issues)) { + Reply(status, issues, TActivationContext::AsActorContext()); + return; + } + if (!ValidateRequest(status, issues)) { Reply(status, issues, TActivationContext::AsActorContext()); return; } SendRequest(); - } + } STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { @@ -438,11 +438,11 @@ public: } bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) { - if (!ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues)) { + if (!ValidateResourcePath(GetProtoRequest()->resource_path(), status, issues)) { return false; } - if (GetProtoRequest()->units_case() == Ydb::RateLimiter::AcquireResourceRequest::UnitsCase::UNITS_NOT_SET) { + if (GetProtoRequest()->units_case() == Ydb::RateLimiter::AcquireResourceRequest::UnitsCase::UNITS_NOT_SET) { return false; } @@ -452,24 +452,24 @@ public: void SendRequest() { Become(&TAcquireRateLimiterResourceRPC::StateFunc); - if (GetProtoRequest()->units_case() == Ydb::RateLimiter::AcquireResourceRequest::UnitsCase::kRequired) { + if (GetProtoRequest()->units_case() == Ydb::RateLimiter::AcquireResourceRequest::UnitsCase::kRequired) { SendLeaf( - TEvQuota::TResourceLeaf(GetProtoRequest()->coordination_node_path(), - GetProtoRequest()->resource_path(), - GetProtoRequest()->required())); + TEvQuota::TResourceLeaf(GetProtoRequest()->coordination_node_path(), + GetProtoRequest()->resource_path(), + GetProtoRequest()->required())); return; - } + } SendLeaf( - TEvQuota::TResourceLeaf(GetProtoRequest()->coordination_node_path(), - GetProtoRequest()->resource_path(), - GetProtoRequest()->used(), + TEvQuota::TResourceLeaf(GetProtoRequest()->coordination_node_path(), + GetProtoRequest()->resource_path(), + GetProtoRequest()->used(), true)); } void SendLeaf(const TEvQuota::TResourceLeaf& leaf) { Send(MakeQuoterServiceID(), - new TEvQuota::TEvRequest(TEvQuota::EResourceOperator::And, { leaf }, GetOperationTimeout()), 0, 0); + new TEvQuota::TEvRequest(TEvQuota::EResourceOperator::And, { leaf }, GetOperationTimeout()), 0, 0); } void Handle(TEvQuota::TEvClearance::TPtr& ev) { @@ -515,9 +515,9 @@ void TGRpcRequestProxy::Handle(TEvAcquireRateLimiterResource::TPtr& ev, const TA ctx.Register(new TAcquireRateLimiterResourceRPC(ev->Release().Release())); } -template<> -IActor* TEvAcquireRateLimiterResource::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { - return new TAcquireRateLimiterResourceRPC(msg, true); -} - +template<> +IActor* TEvAcquireRateLimiterResource::CreateRpcActor(NKikimr::NGRpcService::IRequestOpCtx* msg) { + return new TAcquireRateLimiterResourceRPC(msg, true); +} + } // namespace NKikimr::NGRpcService diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index a9d7785075..4baae606a9 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -745,7 +745,7 @@ private: void ReplyWithError(StatusIds::StatusCode status, const TString& message, const TActorContext& ctx) { Finished = true; Request->RaiseIssue(NYql::TIssue(message)); - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); // We cannot Die() while scheme cache request is in flight because that request has pointer to // KeyRange member so we must not destroy it before we get the response @@ -757,7 +757,7 @@ private: void ReplyWithError(StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { Finished = true; Request->RaiseIssues(issues); - Request->ReplyWithYdbStatus(status); + Request->ReplyWithYdbStatus(status); if (!WaitingResolveReply) { Die(ctx); diff --git a/ydb/core/grpc_services/rpc_read_table.cpp b/ydb/core/grpc_services/rpc_read_table.cpp index 3409cb140d..bbb8790728 100644 --- a/ydb/core/grpc_services/rpc_read_table.cpp +++ b/ydb/core/grpc_services/rpc_read_table.cpp @@ -1,67 +1,67 @@ -#include "grpc_request_proxy.h" - -#include "rpc_common.h" -#include "rpc_calls.h" -#include "rpc_kqp_base.h" -#include "local_rate_limiter.h" - +#include "grpc_request_proxy.h" + +#include "rpc_common.h" +#include "rpc_calls.h" +#include "rpc_kqp_base.h" +#include "local_rate_limiter.h" + #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/core/base/appdata.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/tx_proxy/read_table.h> #include <ydb/core/tx/tx_processing.h> - + #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/actorlib_impl/async_destroyer.h> - + #include <ydb/core/protos/ydb_table_impl.pb.h> #include <ydb/core/ydb_convert/ydb_convert.h> - -#include <util/generic/size_literals.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +#include <util/generic/size_literals.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; -using namespace NKqp; - -static const TDuration RlMaxDuration = TDuration::Minutes(1); - -static ui64 CalcRuConsumption(const TString& data) { - constexpr ui64 unitSize = 1_MB; - constexpr ui64 unitSizeAdjust = unitSize - 1; - - auto mb = (data.size() + unitSizeAdjust) / unitSize; - return mb * 128; // 128 ru for 1 MiB -} - -static void NullSerializeReadTableResponse(const TString& input, Ydb::StatusIds::StatusCode status, TString* output) { - Ydb::Impl::ReadTableResponse readTableResponse; - readTableResponse.set_status(status); - - readTableResponse.mutable_result()->set_result_set(input.Data(), input.Size()); +using namespace NKqp; + +static const TDuration RlMaxDuration = TDuration::Minutes(1); + +static ui64 CalcRuConsumption(const TString& data) { + constexpr ui64 unitSize = 1_MB; + constexpr ui64 unitSizeAdjust = unitSize - 1; + + auto mb = (data.size() + unitSizeAdjust) / unitSize; + return mb * 128; // 128 ru for 1 MiB +} + +static void NullSerializeReadTableResponse(const TString& input, Ydb::StatusIds::StatusCode status, TString* output) { + Ydb::Impl::ReadTableResponse readTableResponse; + readTableResponse.set_status(status); + + readTableResponse.mutable_result()->set_result_set(input.Data(), input.Size()); Y_PROTOBUF_SUPPRESS_NODISCARD readTableResponse.SerializeToString(output); -} - -static void NullSerializeReadTableResponse(const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, Ydb::StatusIds::StatusCode status, TString* output) { - Ydb::Impl::ReadTableResponse readTableResponse; - readTableResponse.set_status(status); - readTableResponse.mutable_issues()->CopyFrom(message); +} + +static void NullSerializeReadTableResponse(const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, Ydb::StatusIds::StatusCode status, TString* output) { + Ydb::Impl::ReadTableResponse readTableResponse; + readTableResponse.set_status(status); + readTableResponse.mutable_issues()->CopyFrom(message); Y_PROTOBUF_SUPPRESS_NODISCARD readTableResponse.SerializeToString(output); -} - -static NKikimrMiniKQL::TParams ConvertKey(const Ydb::TypedValue& key) { - NKikimrMiniKQL::TParams protobuf; - ConvertYdbTypeToMiniKQLType(key.type(), *protobuf.MutableType()); - ConvertYdbValueToMiniKQLValue(key.type(), key.value(), *protobuf.MutableValue()); - return protobuf; -} - +} + +static NKikimrMiniKQL::TParams ConvertKey(const Ydb::TypedValue& key) { + NKikimrMiniKQL::TParams protobuf; + ConvertYdbTypeToMiniKQLType(key.type(), *protobuf.MutableType()); + ConvertYdbValueToMiniKQLValue(key.type(), key.value(), *protobuf.MutableValue()); + return protobuf; +} + template<class TGetOutput> static void ConvertKeyRange(const Ydb::Table::KeyRange& keyRange, const TGetOutput& getOutput) { - switch (keyRange.from_bound_case()) { + switch (keyRange.from_bound_case()) { case Ydb::Table::KeyRange::kGreaterOrEqual: { auto* output = getOutput(); output->SetFromInclusive(true); @@ -74,10 +74,10 @@ static void ConvertKeyRange(const Ydb::Table::KeyRange& keyRange, const TGetOutp output->MutableFrom()->CopyFrom(ConvertKey(keyRange.greater())); break; } - default: + default: break; - } - switch (keyRange.to_bound_case()) { + } + switch (keyRange.to_bound_case()) { case Ydb::Table::KeyRange::kLessOrEqual: { auto* output = getOutput(); output->SetToInclusive(true); @@ -90,62 +90,62 @@ static void ConvertKeyRange(const Ydb::Table::KeyRange& keyRange, const TGetOutp output->MutableTo()->CopyFrom(ConvertKey(keyRange.less())); break; } - default: + default: break; - } -} - + } +} + class TReadTableRPC : public TActorBootstrapped<TReadTableRPC> { - enum EWakeupTag : ui64 { - TimeoutTimer = 0, - RlSendAllowed = 1, - RlNoResource = 2 - }; - -public: + enum EWakeupTag : ui64 { + TimeoutTimer = 0, + RlSendAllowed = 1, + RlNoResource = 2 + }; + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::GRPC_STREAM_REQ; } - TReadTableRPC(TEvReadTableRequest* msg) - : Request_(msg) - , QuotaLimit_(10) - , QuotaReserved_(0) - , LeftInGRpcAdaptorQueue_(0) - {} - - void Bootstrap(const TActorContext& ctx) { - Become(&TReadTableRPC::StateWork); - - const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); - QuotaLimit_ = Max((size_t)1, (size_t)(cfg.GetMessageBufferSize() / cfg.GetMessageSizeLimit())); - - LastStatusTimestamp_ = ctx.Now(); - LastDataStreamTimestamp_ = ctx.Now(); - - InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout()); - InactiveServerTimeout_ = TDuration::FromValue(cfg.GetInactiveServerTimeout()); - - TDuration timeout; - if (InactiveClientTimeout_) - timeout = InactiveClientTimeout_; - if (InactiveServerTimeout_) - timeout = timeout ? Min(timeout, InactiveServerTimeout_) : InactiveServerTimeout_; - if (timeout) - SetTimeoutTimer(timeout, ctx); - - SendProposeRequest(ctx); - - auto actorId = SelfId(); - const TActorSystem* const as = ctx.ExecutorThread.ActorSystem; + TReadTableRPC(TEvReadTableRequest* msg) + : Request_(msg) + , QuotaLimit_(10) + , QuotaReserved_(0) + , LeftInGRpcAdaptorQueue_(0) + {} + + void Bootstrap(const TActorContext& ctx) { + Become(&TReadTableRPC::StateWork); + + const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); + QuotaLimit_ = Max((size_t)1, (size_t)(cfg.GetMessageBufferSize() / cfg.GetMessageSizeLimit())); + + LastStatusTimestamp_ = ctx.Now(); + LastDataStreamTimestamp_ = ctx.Now(); + + InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout()); + InactiveServerTimeout_ = TDuration::FromValue(cfg.GetInactiveServerTimeout()); + + TDuration timeout; + if (InactiveClientTimeout_) + timeout = InactiveClientTimeout_; + if (InactiveServerTimeout_) + timeout = timeout ? Min(timeout, InactiveServerTimeout_) : InactiveServerTimeout_; + if (timeout) + SetTimeoutTimer(timeout, ctx); + + SendProposeRequest(ctx); + + auto actorId = SelfId(); + const TActorSystem* const as = ctx.ExecutorThread.ActorSystem; auto clientLostCb = [actorId, as]() { - LOG_WARN(*as, NKikimrServices::READ_TABLE_API, "ForgetAction occurred, send TEvPoisonPill"); - as->Send(actorId, new TEvents::TEvPoisonPill()); - }; + LOG_WARN(*as, NKikimrServices::READ_TABLE_API, "ForgetAction occurred, send TEvPoisonPill"); + as->Send(actorId, new TEvents::TEvPoisonPill()); + }; Request_->SetClientLostAction(std::move(clientLostCb)); - - } - + + } + void PassAway() override { if (ReadTableActor) { Send(ReadTableActor, new TEvents::TEvPoison); @@ -154,85 +154,85 @@ public: TActorBootstrapped::PassAway(); } -private: - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleResponseData); - HFunc(TRpcServices::TEvGrpcNextReply, Handle); - HFunc(TEvTxProcessing::TEvStreamQuotaRequest, Handle); - HFunc(TEvTxProcessing::TEvStreamQuotaRelease, Handle); - HFunc(TEvents::TEvPoisonPill, HandlePoison) - HFunc(TEvents::TEvSubscribe, Handle); - HFunc(TEvents::TEvWakeup, Handle); +private: + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleResponseData); + HFunc(TRpcServices::TEvGrpcNextReply, Handle); + HFunc(TEvTxProcessing::TEvStreamQuotaRequest, Handle); + HFunc(TEvTxProcessing::TEvStreamQuotaRelease, Handle); + HFunc(TEvents::TEvPoisonPill, HandlePoison) + HFunc(TEvents::TEvSubscribe, Handle); + HFunc(TEvents::TEvWakeup, Handle); HFunc(TEvDataShard::TEvGetReadTableStreamStateRequest, Handle); - default: - Y_FAIL("TRequestHandler: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - - void HandleResponseData(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { - const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get(); - LastStatusTimestamp_ = ctx.Now(); - - const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus()); - auto issueMessage = msg->Record.GetIssues(); - switch (status) { - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecResponseData: { - // This check need to protect wrong response in case of new grpc -> old tx proxy request. - if (msg->Record.HasReadTableResponseVersion() && - msg->Record.GetReadTableResponseVersion() == NKikimrTxUserProxy::TReadTableTransaction::YDB_V1) { - auto result = msg->Record.GetSerializedReadTableResponse(); - return TryToSendStreamResult(result, msg->Record.GetDataShardTabletId(), - msg->Record.GetTxId(), ctx); - } else { - TStringStream str; - str << "Response version missmatched"; - LOG_ERROR(ctx, NKikimrServices::READ_TABLE_API, + default: + Y_FAIL("TRequestHandler: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + + void HandleResponseData(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { + const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get(); + LastStatusTimestamp_ = ctx.Now(); + + const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus()); + auto issueMessage = msg->Record.GetIssues(); + switch (status) { + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecResponseData: { + // This check need to protect wrong response in case of new grpc -> old tx proxy request. + if (msg->Record.HasReadTableResponseVersion() && + msg->Record.GetReadTableResponseVersion() == NKikimrTxUserProxy::TReadTableTransaction::YDB_V1) { + auto result = msg->Record.GetSerializedReadTableResponse(); + return TryToSendStreamResult(result, msg->Record.GetDataShardTabletId(), + msg->Record.GetTxId(), ctx); + } else { + TStringStream str; + str << "Response version missmatched"; + LOG_ERROR(ctx, NKikimrServices::READ_TABLE_API, "%s", str.Str().data()); - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); - auto tmp = issueMessage.Add(); - NYql::IssueToMessage(issue, tmp); - return ReplyFinishStream(StatusIds::INTERNAL_ERROR, issueMessage, ctx); - } - } - break; - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: { + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); + auto tmp = issueMessage.Add(); + NYql::IssueToMessage(issue, tmp); + return ReplyFinishStream(StatusIds::INTERNAL_ERROR, issueMessage, ctx); + } + } + break; + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: { if (msg->Record.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusSuccess || msg->Record.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusAlreadyExists) - { - // We can't close stream if has not empty send buffer - if (SendBuffer_) { - HasPendingSuccess = true; - } else { - return ReplyFinishStream(Ydb::StatusIds::SUCCESS, issueMessage, ctx); - } - } - break; - } - case TEvTxUserProxy::TResultStatus::AccessDenied: { - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got AccessDenied response from TxProxy"); - auto tmp = issueMessage.Add(); - NYql::IssueToMessage(issue, tmp); + { + // We can't close stream if has not empty send buffer + if (SendBuffer_) { + HasPendingSuccess = true; + } else { + return ReplyFinishStream(Ydb::StatusIds::SUCCESS, issueMessage, ctx); + } + } + break; + } + case TEvTxUserProxy::TResultStatus::AccessDenied: { + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got AccessDenied response from TxProxy"); + auto tmp = issueMessage.Add(); + NYql::IssueToMessage(issue, tmp); return ReplyFinishStream(Ydb::StatusIds::UNAUTHORIZED, issueMessage, ctx); - } - case TEvTxUserProxy::TResultStatus::ResolveError: { - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got ResolveError response from TxProxy"); - auto tmp = issueMessage.Add(); - NYql::IssueToMessage(issue, tmp); + } + case TEvTxUserProxy::TResultStatus::ResolveError: { + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got ResolveError response from TxProxy"); + auto tmp = issueMessage.Add(); + NYql::IssueToMessage(issue, tmp); return ReplyFinishStream(Ydb::StatusIds::SCHEME_ERROR, issueMessage, ctx); - } - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyNotReady: - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardTryLater: - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardUnknown: + } + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyNotReady: + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardTryLater: + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardUnknown: case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::CoordinatorDeclined: - case TEvTxUserProxy::TResultStatus::ProxyShardNotAvailable: { - TStringStream str; - str << "Got " << status << " response from TxProxy"; - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); - auto tmp = issueMessage.Add(); - NYql::IssueToMessage(issue, tmp); - return ReplyFinishStream(Ydb::StatusIds::UNAVAILABLE, issueMessage, ctx); - } + case TEvTxUserProxy::TResultStatus::ProxyShardNotAvailable: { + TStringStream str; + str << "Got " << status << " response from TxProxy"; + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); + auto tmp = issueMessage.Add(); + NYql::IssueToMessage(issue, tmp); + return ReplyFinishStream(Ydb::StatusIds::UNAVAILABLE, issueMessage, ctx); + } case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::CoordinatorUnknown: { TString str = TStringBuilder() << "Got " << status << " response from TxProxy, transaction state unknown"; @@ -248,166 +248,166 @@ private: NYql::IssueToMessage(issue, tmp); return ReplyFinishStream(Ydb::StatusIds::TIMEOUT, issueMessage, ctx); } - default: { - TStringStream str; - str << "Got unknown TEvProposeTransactionStatus (" << status << ") response from TxProxy"; - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); - auto tmp = issueMessage.Add(); - NYql::IssueToMessage(issue, tmp); - return ReplyFinishStream(StatusIds::INTERNAL_ERROR, issueMessage, ctx); - } - } - } - - void Handle(TEvTxProcessing::TEvStreamQuotaRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Adding quota request to queue ShardId: " - << ev->Get()->Record.GetShardId() << ", TxId: " - << ev->Get()->Record.GetTxId()); - - QuotaRequestQueue_.push_back(ev); - TryToAllocateQuota(); - } - - void Handle(TEvTxProcessing::TEvStreamQuotaRelease::TPtr& ev, const TActorContext& ctx) { - ui64 shardId = ev->Get()->Record.GetShardId(); - ui64 txId = ev->Get()->Record.GetTxId(); - - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Release quota for ShardId: " << shardId - << ", TxId: " << txId); - - ReleasedShards_.insert(shardId); - - auto it = QuotaByShard_.find(shardId); - if (it != QuotaByShard_.end()) { - QuotaReserved_ -= it->second; - QuotaByShard_.erase(it); - TryToAllocateQuota(); - } - } - - void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) { + default: { + TStringStream str; + str << "Got unknown TEvProposeTransactionStatus (" << status << ") response from TxProxy"; + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, str.Str()); + auto tmp = issueMessage.Add(); + NYql::IssueToMessage(issue, tmp); + return ReplyFinishStream(StatusIds::INTERNAL_ERROR, issueMessage, ctx); + } + } + } + + void Handle(TEvTxProcessing::TEvStreamQuotaRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Adding quota request to queue ShardId: " + << ev->Get()->Record.GetShardId() << ", TxId: " + << ev->Get()->Record.GetTxId()); + + QuotaRequestQueue_.push_back(ev); + TryToAllocateQuota(); + } + + void Handle(TEvTxProcessing::TEvStreamQuotaRelease::TPtr& ev, const TActorContext& ctx) { + ui64 shardId = ev->Get()->Record.GetShardId(); + ui64 txId = ev->Get()->Record.GetTxId(); + + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Release quota for ShardId: " << shardId + << ", TxId: " << txId); + + ReleasedShards_.insert(shardId); + + auto it = QuotaByShard_.find(shardId); + if (it != QuotaByShard_.end()) { + QuotaReserved_ -= it->second; + QuotaByShard_.erase(it); + TryToAllocateQuota(); + } + } + + void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); if (LeftInGRpcAdaptorQueue_ > 0) LeftInGRpcAdaptorQueue_--; - LastDataStreamTimestamp_ = ctx.Now(); - TryToAllocateQuota(); - } - - void HandlePoison(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext &ctx) { - Y_UNUSED(ev); - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; - auto item = message.Add(); - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, - "Client should not see this message, if so... may the force be with you"); - NYql::IssueToMessage(issue, item); - // We must try to finish stream otherwise grpc will not free allocated memory - // If stream already scheduled to be finished (ReplyFinishStream already called) - // this call do nothing but Die will be called after reply to grpc - ReplyFinishStream(StatusIds::INTERNAL_ERROR, message, ctx); - } - - void Handle(TEvents::TEvSubscribe::TPtr& ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Add stream subscriber " << ev->Sender); - - StreamSubscribers_.insert(ev->Sender); - } - - void Handle(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { - switch ((EWakeupTag) ev->Get()->Tag) { - case EWakeupTag::TimeoutTimer: - return ProcessTimeoutTimer(ctx); - case EWakeupTag::RlSendAllowed: - return ProcessRlSendAllowed(ctx); - case EWakeupTag::RlNoResource: - return ProcessRlNoResource(ctx); - } - } - - void ProcessRlSendAllowed(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Success rate limiter response, we got " - << SendBuffer_.size() << " pending messages," - " pending success# " << HasPendingSuccess); - TBuffEntry& entry = SendBuffer_.front(); - - DoSendStreamResult(std::move(entry.Buf), entry.ShardId, entry.TxId, entry.StartTime, ctx); - - SendBuffer_.pop_front(); - - if (SendBuffer_.empty() && HasPendingSuccess) { - const static google::protobuf::RepeatedPtrField<TYdbIssueMessageType> empty; - return ReplyFinishStream(Ydb::StatusIds::SUCCESS, empty, ctx); - } - } - - void ProcessRlNoResource(const TActorContext& ctx) { - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, - "Throughput limit exceeded for read table request"); - LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Throughput limit exceeded, we got " - << SendBuffer_.size() << " pending messages," - " pending success# " << HasPendingSuccess - << " stream will be terminated"); - - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; - NYql::IssueToMessage(issue, message.Add()); - - return ReplyFinishStream(Ydb::StatusIds::OVERLOADED, message, ctx); - } - - void ProcessTimeoutTimer(const TActorContext &ctx) { - TInstant now = ctx.Now(); - TDuration timeout; - - if (InactiveClientTimeout_ && LeftInGRpcAdaptorQueue_) { - TDuration processTime = now - LastDataStreamTimestamp_; - if (processTime >= InactiveClientTimeout_) { - TStringStream ss; - ss << SelfId() << " Client cannot process data in " << processTime - << " which exceeds client timeout " << InactiveClientTimeout_; - LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, ss.Str()); - return HandleStreamTimeout(ctx, ss.Str()); - } else { - TDuration remain = InactiveClientTimeout_ - processTime; - timeout = timeout ? Min(timeout, remain) : remain; - } - } - - if (InactiveServerTimeout_) { - TDuration processTime = now - LastStatusTimestamp_; - // Ignore server timeout if response buffer is full. - if (LeftInGRpcAdaptorQueue_ == QuotaLimit_) { - timeout = timeout ? Min(timeout, InactiveServerTimeout_) : InactiveServerTimeout_; - } else if (processTime >= InactiveServerTimeout_) { - TStringStream ss; - ss << SelfId() - << " Server doesn't provide data for " << processTime - << " which exceeds server timeout " << InactiveServerTimeout_ - << " (QuotaRequestQueue: " << QuotaRequestQueue_.size() - << " ResponseQueue: " << LeftInGRpcAdaptorQueue_ - << " QuotaLimit: " << QuotaLimit_ - << " QuotaReserved: " << QuotaReserved_ - << " QuotaByShard: " << QuotaByShard_.size() << ")"; - LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, ss.Str()); - return HandleStreamTimeout(ctx, ss.Str()); - } else { - TDuration remain = InactiveServerTimeout_ - processTime; - timeout = timeout ? Min(timeout, remain) : remain; - } - } - - if (timeout) - SetTimeoutTimer(timeout, ctx); - } - + LastDataStreamTimestamp_ = ctx.Now(); + TryToAllocateQuota(); + } + + void HandlePoison(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext &ctx) { + Y_UNUSED(ev); + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; + auto item = message.Add(); + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, + "Client should not see this message, if so... may the force be with you"); + NYql::IssueToMessage(issue, item); + // We must try to finish stream otherwise grpc will not free allocated memory + // If stream already scheduled to be finished (ReplyFinishStream already called) + // this call do nothing but Die will be called after reply to grpc + ReplyFinishStream(StatusIds::INTERNAL_ERROR, message, ctx); + } + + void Handle(TEvents::TEvSubscribe::TPtr& ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Add stream subscriber " << ev->Sender); + + StreamSubscribers_.insert(ev->Sender); + } + + void Handle(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { + switch ((EWakeupTag) ev->Get()->Tag) { + case EWakeupTag::TimeoutTimer: + return ProcessTimeoutTimer(ctx); + case EWakeupTag::RlSendAllowed: + return ProcessRlSendAllowed(ctx); + case EWakeupTag::RlNoResource: + return ProcessRlNoResource(ctx); + } + } + + void ProcessRlSendAllowed(const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Success rate limiter response, we got " + << SendBuffer_.size() << " pending messages," + " pending success# " << HasPendingSuccess); + TBuffEntry& entry = SendBuffer_.front(); + + DoSendStreamResult(std::move(entry.Buf), entry.ShardId, entry.TxId, entry.StartTime, ctx); + + SendBuffer_.pop_front(); + + if (SendBuffer_.empty() && HasPendingSuccess) { + const static google::protobuf::RepeatedPtrField<TYdbIssueMessageType> empty; + return ReplyFinishStream(Ydb::StatusIds::SUCCESS, empty, ctx); + } + } + + void ProcessRlNoResource(const TActorContext& ctx) { + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, + "Throughput limit exceeded for read table request"); + LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Throughput limit exceeded, we got " + << SendBuffer_.size() << " pending messages," + " pending success# " << HasPendingSuccess + << " stream will be terminated"); + + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; + NYql::IssueToMessage(issue, message.Add()); + + return ReplyFinishStream(Ydb::StatusIds::OVERLOADED, message, ctx); + } + + void ProcessTimeoutTimer(const TActorContext &ctx) { + TInstant now = ctx.Now(); + TDuration timeout; + + if (InactiveClientTimeout_ && LeftInGRpcAdaptorQueue_) { + TDuration processTime = now - LastDataStreamTimestamp_; + if (processTime >= InactiveClientTimeout_) { + TStringStream ss; + ss << SelfId() << " Client cannot process data in " << processTime + << " which exceeds client timeout " << InactiveClientTimeout_; + LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, ss.Str()); + return HandleStreamTimeout(ctx, ss.Str()); + } else { + TDuration remain = InactiveClientTimeout_ - processTime; + timeout = timeout ? Min(timeout, remain) : remain; + } + } + + if (InactiveServerTimeout_) { + TDuration processTime = now - LastStatusTimestamp_; + // Ignore server timeout if response buffer is full. + if (LeftInGRpcAdaptorQueue_ == QuotaLimit_) { + timeout = timeout ? Min(timeout, InactiveServerTimeout_) : InactiveServerTimeout_; + } else if (processTime >= InactiveServerTimeout_) { + TStringStream ss; + ss << SelfId() + << " Server doesn't provide data for " << processTime + << " which exceeds server timeout " << InactiveServerTimeout_ + << " (QuotaRequestQueue: " << QuotaRequestQueue_.size() + << " ResponseQueue: " << LeftInGRpcAdaptorQueue_ + << " QuotaLimit: " << QuotaLimit_ + << " QuotaReserved: " << QuotaReserved_ + << " QuotaByShard: " << QuotaByShard_.size() << ")"; + LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, ss.Str()); + return HandleStreamTimeout(ctx, ss.Str()); + } else { + TDuration remain = InactiveServerTimeout_ - processTime; + timeout = timeout ? Min(timeout, remain) : remain; + } + } + + if (timeout) + SetTimeoutTimer(timeout, ctx); + } + void Handle(TEvDataShard::TEvGetReadTableStreamStateRequest::TPtr &ev, const TActorContext &ctx) { auto *response = new TEvDataShard::TEvGetReadTableStreamStateResponse; response->Record.MutableStatus()->SetCode(Ydb::StatusIds::SUCCESS); - + //response->Record.SetReady(IsStreamReady); //response->Record.SetFinished(IsStreamFinished); response->Record.SetResponseQueueSize(LeftInGRpcAdaptorQueue_); @@ -430,39 +430,39 @@ private: ctx.Send(ev->Sender, response); } - void SendProposeRequest(const TActorContext &ctx) { - const auto req = Request_->GetProtoRequest(); - auto actorId = SelfId(); - const TActorSystem* const as = ctx.ExecutorThread.ActorSystem; - auto cb = [actorId, as](size_t left) { - as->Send(actorId, new TRpcServices::TEvGrpcNextReply{left}); - }; - - Request_->SetStreamingNotify(cb); - - if (req->path().empty()) { - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got empty table path"); - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; - auto item = message.Add(); - NYql::IssueToMessage(issue, item); - return ReplyFinishStream(StatusIds::BAD_REQUEST, message, ctx); - } - + void SendProposeRequest(const TActorContext &ctx) { + const auto req = Request_->GetProtoRequest(); + auto actorId = SelfId(); + const TActorSystem* const as = ctx.ExecutorThread.ActorSystem; + auto cb = [actorId, as](size_t left) { + as->Send(actorId, new TRpcServices::TEvGrpcNextReply{left}); + }; + + Request_->SetStreamingNotify(cb); + + if (req->path().empty()) { + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Got empty table path"); + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; + auto item = message.Add(); + NYql::IssueToMessage(issue, item); + return ReplyFinishStream(StatusIds::BAD_REQUEST, message, ctx); + } + const auto& featureFlags = AppData(ctx)->FeatureFlags; - + bool useSnapshot = featureFlags.GetReadTableWithSnapshot(); switch (req->use_snapshot()) { case Ydb::FeatureFlag::STATUS_UNSPECIFIED: break; - + case Ydb::FeatureFlag::ENABLED: useSnapshot = true; break; - + case Ydb::FeatureFlag::DISABLED: useSnapshot = false; break; - + default: { const NYql::TIssue& issue = MakeIssue( NKikimrIssues::TIssuesIds::DEFAULT_ERROR, @@ -472,8 +472,8 @@ private: NYql::IssueToMessage(issue, item); return ReplyFinishStream(StatusIds::BAD_REQUEST, message, ctx); } - } - + } + if (useSnapshot) { NKikimr::NTxProxy::TReadTableSettings settings; @@ -503,7 +503,7 @@ private: return ReplyFinishStream(StatusIds::BAD_REQUEST, message, ctx); } - ReadTableActor = ctx.RegisterWithSameMailbox(NKikimr::NTxProxy::CreateReadTableSnapshotWorker(settings)); + ReadTableActor = ctx.RegisterWithSameMailbox(NKikimr::NTxProxy::CreateReadTableSnapshotWorker(settings)); } else { auto proposeRequest = std::make_unique<TEvTxUserProxy::TEvProposeTransaction>(); @@ -534,229 +534,229 @@ private: readTransaction->AddColumns(col); } ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - } - + } + } + void ReplyFinishStream(Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, - const TActorContext& ctx) { - // Skip sending empty result in case of success status - simplify client logic - if (status != Ydb::StatusIds::SUCCESS) { - TString out; - NullSerializeReadTableResponse(message, status, &out); + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, + const TActorContext& ctx) { + // Skip sending empty result in case of success status - simplify client logic + if (status != Ydb::StatusIds::SUCCESS) { + TString out; + NullSerializeReadTableResponse(message, status, &out); Request_->SendSerializedResult(std::move(out), status); - } - Request_->FinishStream(); - LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Finish grpc stream, status: " << (int)status); - - auto &cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); - - // Answer all pending quota requests. - while (!QuotaRequestQueue_.empty()) { - auto request = QuotaRequestQueue_.front(); - const auto& rec = request->Get()->Record; - QuotaRequestQueue_.pop_front(); - - TAutoPtr<TEvTxProcessing::TEvStreamQuotaResponse> response - = new TEvTxProcessing::TEvStreamQuotaResponse; - response->Record.SetTxId(rec.GetTxId()); - response->Record.SetMessageSizeLimit(cfg.GetMessageSizeLimit()); - response->Record.SetReservedMessages(0); - - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Send zero quota to Shard " << rec.GetShardId() - << ", TxId " << rec.GetTxId()); + } + Request_->FinishStream(); + LOG_NOTICE_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Finish grpc stream, status: " << (int)status); + + auto &cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); + + // Answer all pending quota requests. + while (!QuotaRequestQueue_.empty()) { + auto request = QuotaRequestQueue_.front(); + const auto& rec = request->Get()->Record; + QuotaRequestQueue_.pop_front(); + + TAutoPtr<TEvTxProcessing::TEvStreamQuotaResponse> response + = new TEvTxProcessing::TEvStreamQuotaResponse; + response->Record.SetTxId(rec.GetTxId()); + response->Record.SetMessageSizeLimit(cfg.GetMessageSizeLimit()); + response->Record.SetReservedMessages(0); + + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Send zero quota to Shard " << rec.GetShardId() + << ", TxId " << rec.GetTxId()); ctx.Send(request->Sender, response.Release(), 0, request->Cookie); - } - - for (const auto& id : StreamSubscribers_) { - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Send dead stream notification to subscriber " << id); - ctx.Send(id, new TEvTxProcessing::TEvStreamIsDead); - } - - Die(ctx); - } - - void SetTimeoutTimer(TDuration timeout, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Set stream timeout timer for " << timeout); - - auto *ev = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup(EWakeupTag::TimeoutTimer)); - StreamTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way()); - CreateLongTimer(ctx, timeout, ev, 0, StreamTimerCookieHolder_.Get()); - } - - void HandleStreamTimeout(const TActorContext &ctx, const TString& msg) { - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; - const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, msg); - auto tmp = message.Add(); - NYql::IssueToMessage(issue, tmp); - return ReplyFinishStream(StatusIds::TIMEOUT, message, ctx); - } - - void TryToAllocateQuota() { - if (QuotaRequestQueue_.empty()) - return; - - auto &cfg = AppData()->StreamingConfig.GetOutputStreamConfig(); + } + + for (const auto& id : StreamSubscribers_) { + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Send dead stream notification to subscriber " << id); + ctx.Send(id, new TEvTxProcessing::TEvStreamIsDead); + } + + Die(ctx); + } + + void SetTimeoutTimer(TDuration timeout, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Set stream timeout timer for " << timeout); + + auto *ev = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup(EWakeupTag::TimeoutTimer)); + StreamTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way()); + CreateLongTimer(ctx, timeout, ev, 0, StreamTimerCookieHolder_.Get()); + } + + void HandleStreamTimeout(const TActorContext &ctx, const TString& msg) { + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> message; + const NYql::TIssue& issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, msg); + auto tmp = message.Add(); + NYql::IssueToMessage(issue, tmp); + return ReplyFinishStream(StatusIds::TIMEOUT, message, ctx); + } + + void TryToAllocateQuota() { + if (QuotaRequestQueue_.empty()) + return; + + auto &cfg = AppData()->StreamingConfig.GetOutputStreamConfig(); if (QuotaLimit_ <= QuotaReserved_ + LeftInGRpcAdaptorQueue_) return; - ui32 freeQuota = QuotaLimit_ - QuotaReserved_ - LeftInGRpcAdaptorQueue_; - - // Allow to ignore MinQuotaSize if limit is smaller. - if (freeQuota < cfg.GetMinQuotaSize() && freeQuota != QuotaLimit_) - return; - - // Limit number of concurrently streaming shards. - if (QuotaByShard_.size() >= cfg.GetMaxStreamingShards()) - return; - - // Limit numbers of pending chunks - if (SendBuffer_.size() >= cfg.GetMaxStreamingShards()) - return; - - auto request = QuotaRequestQueue_.front(); - const auto& rec = request->Get()->Record; - QuotaRequestQueue_.pop_front(); - - if (ReleasedShards_.contains(rec.GetShardId())) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, - "Quota request from released shard " << rec.GetShardId()); - return TryToAllocateQuota(); - } - - ui32 quotaSize = Min(cfg.GetMaxQuotaSize(), freeQuota); - - QuotaReserved_ += quotaSize; - QuotaByShard_[rec.GetShardId()] += quotaSize; - - TAutoPtr<TEvTxProcessing::TEvStreamQuotaResponse> response - = new TEvTxProcessing::TEvStreamQuotaResponse; - response->Record.SetTxId(rec.GetTxId()); - response->Record.SetMessageSizeLimit(cfg.GetMessageSizeLimit()); - response->Record.SetReservedMessages(quotaSize); - - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, - SelfId() << " Assign stream quota to Shard " << rec.GetShardId() - << ", Quota " << quotaSize << ", TxId " << rec.GetTxId() - << " Reserved: " << QuotaReserved_ << " of " << QuotaLimit_ - << ", Queued: " << LeftInGRpcAdaptorQueue_); - - Send(request->Sender, response.Release(), 0, request->Cookie); - } - - void TryToSendStreamResult(const TString& data, ui64 shardId, ui64 txId, const TActorContext& ctx) { - auto ru = CalcRuConsumption(data); - - auto getRlPAth = [this] { - if (auto path = Request_->GetRlPath()) { - return Sprintf("rate limiter found, coordination node: %s, resource: %s", - path->CoordinationNode.c_str(), path->ResourcePath.c_str()); - } else { - return TString("rate limiter absent"); - } - }; - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " got stream part, size: " << data.size() - << ", RU required: " << ru << " " << getRlPAth()); - - TString out; - NullSerializeReadTableResponse(data, StatusIds::SUCCESS, &out); - TInstant startTime = ctx.Now(); - - if (Request_->GetRlPath()) { - auto selfId = this->SelfId(); - auto as = TActivationContext::ActorSystem(); - - auto onSendAllowed = [selfId, as]() mutable { - as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::RlSendAllowed)); - }; - - auto onSendTimeout = [selfId, as] { - as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::RlNoResource)); - }; - - SendBuffer_.emplace_back(TBuffEntry{std::move(out), shardId, txId, startTime}); - auto rlActor = NRpcService::RateLimiterAcquireUseSameMailbox( - *Request_.get(), ru, RlMaxDuration, - std::move(onSendAllowed), std::move(onSendTimeout), ctx); - - LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, - SelfId() << " Launch rate limiter actor: " - << rlActor); - } else { - DoSendStreamResult(std::move(out), shardId, txId, TInstant(), ctx); - } - } - - void DoSendStreamResult(TString&& out, ui64 shardId, ui64 txId, TInstant startTime, const TActorContext& ctx) { - auto it = QuotaByShard_.find(shardId); - if (it != QuotaByShard_.end()) { - --it->second; - if (!it->second) - QuotaByShard_.erase(it); - --QuotaReserved_; - } else { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, - SelfId() << " Response out of quota from Shard " << shardId - << ", TxId " << txId); - } - - if (startTime) { - auto delay = ctx.Now() - startTime; - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, - SelfId() << " Data response has been delayed for " << delay << " seconds"); - } - + ui32 freeQuota = QuotaLimit_ - QuotaReserved_ - LeftInGRpcAdaptorQueue_; + + // Allow to ignore MinQuotaSize if limit is smaller. + if (freeQuota < cfg.GetMinQuotaSize() && freeQuota != QuotaLimit_) + return; + + // Limit number of concurrently streaming shards. + if (QuotaByShard_.size() >= cfg.GetMaxStreamingShards()) + return; + + // Limit numbers of pending chunks + if (SendBuffer_.size() >= cfg.GetMaxStreamingShards()) + return; + + auto request = QuotaRequestQueue_.front(); + const auto& rec = request->Get()->Record; + QuotaRequestQueue_.pop_front(); + + if (ReleasedShards_.contains(rec.GetShardId())) { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, + "Quota request from released shard " << rec.GetShardId()); + return TryToAllocateQuota(); + } + + ui32 quotaSize = Min(cfg.GetMaxQuotaSize(), freeQuota); + + QuotaReserved_ += quotaSize; + QuotaByShard_[rec.GetShardId()] += quotaSize; + + TAutoPtr<TEvTxProcessing::TEvStreamQuotaResponse> response + = new TEvTxProcessing::TEvStreamQuotaResponse; + response->Record.SetTxId(rec.GetTxId()); + response->Record.SetMessageSizeLimit(cfg.GetMessageSizeLimit()); + response->Record.SetReservedMessages(quotaSize); + + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, + SelfId() << " Assign stream quota to Shard " << rec.GetShardId() + << ", Quota " << quotaSize << ", TxId " << rec.GetTxId() + << " Reserved: " << QuotaReserved_ << " of " << QuotaLimit_ + << ", Queued: " << LeftInGRpcAdaptorQueue_); + + Send(request->Sender, response.Release(), 0, request->Cookie); + } + + void TryToSendStreamResult(const TString& data, ui64 shardId, ui64 txId, const TActorContext& ctx) { + auto ru = CalcRuConsumption(data); + + auto getRlPAth = [this] { + if (auto path = Request_->GetRlPath()) { + return Sprintf("rate limiter found, coordination node: %s, resource: %s", + path->CoordinationNode.c_str(), path->ResourcePath.c_str()); + } else { + return TString("rate limiter absent"); + } + }; + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " got stream part, size: " << data.size() + << ", RU required: " << ru << " " << getRlPAth()); + + TString out; + NullSerializeReadTableResponse(data, StatusIds::SUCCESS, &out); + TInstant startTime = ctx.Now(); + + if (Request_->GetRlPath()) { + auto selfId = this->SelfId(); + auto as = TActivationContext::ActorSystem(); + + auto onSendAllowed = [selfId, as]() mutable { + as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::RlSendAllowed)); + }; + + auto onSendTimeout = [selfId, as] { + as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::RlNoResource)); + }; + + SendBuffer_.emplace_back(TBuffEntry{std::move(out), shardId, txId, startTime}); + auto rlActor = NRpcService::RateLimiterAcquireUseSameMailbox( + *Request_.get(), ru, RlMaxDuration, + std::move(onSendAllowed), std::move(onSendTimeout), ctx); + + LOG_DEBUG_S(ctx, NKikimrServices::READ_TABLE_API, + SelfId() << " Launch rate limiter actor: " + << rlActor); + } else { + DoSendStreamResult(std::move(out), shardId, txId, TInstant(), ctx); + } + } + + void DoSendStreamResult(TString&& out, ui64 shardId, ui64 txId, TInstant startTime, const TActorContext& ctx) { + auto it = QuotaByShard_.find(shardId); + if (it != QuotaByShard_.end()) { + --it->second; + if (!it->second) + QuotaByShard_.erase(it); + --QuotaReserved_; + } else { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, + SelfId() << " Response out of quota from Shard " << shardId + << ", TxId " << txId); + } + + if (startTime) { + auto delay = ctx.Now() - startTime; + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, + SelfId() << " Data response has been delayed for " << delay << " seconds"); + } + Request_->SendSerializedResult(std::move(out), StatusIds::SUCCESS); - - LeftInGRpcAdaptorQueue_++; - if (LeftInGRpcAdaptorQueue_ > QuotaLimit_) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, - SelfId() << " TMessageBusServerRequest: response queue limit is exceeded."); - } - - TryToAllocateQuota(); - } - - std::unique_ptr<TEvReadTableRequest> Request_; - + + LeftInGRpcAdaptorQueue_++; + if (LeftInGRpcAdaptorQueue_ > QuotaLimit_) { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::READ_TABLE_API, + SelfId() << " TMessageBusServerRequest: response queue limit is exceeded."); + } + + TryToAllocateQuota(); + } + + std::unique_ptr<TEvReadTableRequest> Request_; + TActorId ReadTableActor; - TList<TEvTxProcessing::TEvStreamQuotaRequest::TPtr> QuotaRequestQueue_; - - size_t QuotaLimit_; - size_t QuotaReserved_; - THashMap<ui64, size_t> QuotaByShard_; - THashSet<ui64> ReleasedShards_; - - size_t LeftInGRpcAdaptorQueue_; - + TList<TEvTxProcessing::TEvStreamQuotaRequest::TPtr> QuotaRequestQueue_; + + size_t QuotaLimit_; + size_t QuotaReserved_; + THashMap<ui64, size_t> QuotaByShard_; + THashSet<ui64> ReleasedShards_; + + size_t LeftInGRpcAdaptorQueue_; + THashSet<TActorId> StreamSubscribers_; - - TDuration InactiveClientTimeout_; - TDuration InactiveServerTimeout_; - TInstant LastDataStreamTimestamp_; - TInstant LastStatusTimestamp_; - - TSchedulerCookieHolder StreamTimerCookieHolder_; - - struct TBuffEntry - { - TString Buf; - ui64 ShardId; - ui64 TxId; - TInstant StartTime; - }; - TDeque<TBuffEntry> SendBuffer_; - bool HasPendingSuccess = false; -}; - -void TGRpcRequestProxy::Handle(TEvReadTableRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TReadTableRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + + TDuration InactiveClientTimeout_; + TDuration InactiveServerTimeout_; + TInstant LastDataStreamTimestamp_; + TInstant LastStatusTimestamp_; + + TSchedulerCookieHolder StreamTimerCookieHolder_; + + struct TBuffEntry + { + TString Buf; + ui64 ShardId; + ui64 TxId; + TInstant StartTime; + }; + TDeque<TBuffEntry> SendBuffer_; + bool HasPendingSuccess = false; +}; + +void TGRpcRequestProxy::Handle(TEvReadTableRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TReadTableRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_remove_directory.cpp b/ydb/core/grpc_services/rpc_remove_directory.cpp index c4d7646b7e..9bdc2ad7b2 100644 --- a/ydb/core/grpc_services/rpc_remove_directory.cpp +++ b/ydb/core/grpc_services/rpc_remove_directory.cpp @@ -1,67 +1,67 @@ -#include "service_scheme.h" - +#include "service_scheme.h" + #include "rpc_scheme_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/protos/ydb_scheme.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace Ydb; - -using TEvRemoveDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::RemoveDirectoryRequest, - Ydb::Scheme::RemoveDirectoryResponse>; - + +using TEvRemoveDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::RemoveDirectoryRequest, + Ydb::Scheme::RemoveDirectoryResponse>; + class TRemoveDirectoryRPC : public TRpcSchemeRequestActor<TRemoveDirectoryRPC, TEvRemoveDirectoryRequest> { using TBase = TRpcSchemeRequestActor<TRemoveDirectoryRPC, TEvRemoveDirectoryRequest>; -public: - TRemoveDirectoryRPC(IRequestOpCtx* msg) +public: + TRemoveDirectoryRPC(IRequestOpCtx* msg) : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { + + void Bootstrap(const TActorContext &ctx) { TBase::Bootstrap(ctx); - SendProposeRequest(ctx); - Become(&TRemoveDirectoryRPC::StateWork); - } - -private: - void SendProposeRequest(const TActorContext &ctx) { - const auto req = GetProtoRequest(); - std::pair<TString, TString> pathPair; - try { - pathPair = SplitPath(req->path()); - } catch (const std::exception& ex) { - Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - + SendProposeRequest(ctx); + Become(&TRemoveDirectoryRPC::StateWork); + } + +private: + void SendProposeRequest(const TActorContext &ctx) { + const auto req = GetProtoRequest(); + std::pair<TString, TString> pathPair; + try { + pathPair = SplitPath(req->path()); + } catch (const std::exception& ex) { + Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); - modifyScheme->SetWorkingDir(workingDir); + modifyScheme->SetWorkingDir(workingDir); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpRmDir); - modifyScheme->MutableDrop()->SetName(name); - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - - void ReplyWithResult(const StatusIds::StatusCode status, - const TActorContext &ctx) { - Request_->ReplyWithYdbStatus(status); - Die(ctx); - } -}; - -void DoRemoveDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TRemoveDirectoryRPC(p.release())); -} - -} // namespace NGRpcService -} // namespace NKikimr - + modifyScheme->MutableDrop()->SetName(name); + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + + void ReplyWithResult(const StatusIds::StatusCode status, + const TActorContext &ctx) { + Request_->ReplyWithYdbStatus(status); + Die(ctx); + } +}; + +void DoRemoveDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TRemoveDirectoryRPC(p.release())); +} + +} // namespace NGRpcService +} // namespace NKikimr + diff --git a/ydb/core/grpc_services/rpc_request_base.h b/ydb/core/grpc_services/rpc_request_base.h index e6175f99db..b5cb84fc0b 100644 --- a/ydb/core/grpc_services/rpc_request_base.h +++ b/ydb/core/grpc_services/rpc_request_base.h @@ -30,7 +30,7 @@ protected: const TResponse& response, const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS ) { - TProtoResponseHelper::SendProtoResponse(response, status, Request); + TProtoResponseHelper::SendProtoResponse(response, status, Request); this->PassAway(); } @@ -69,8 +69,8 @@ protected: } void Reply(const Ydb::Operations::Operation& operation) { - Request->SendOperation(operation); - this->PassAway(); + Request->SendOperation(operation); + this->PassAway(); } bool CheckAccess(const TString& path, TIntrusivePtr<TSecurityObject> securityObject, ui32 access) { diff --git a/ydb/core/grpc_services/rpc_rollback_transaction.cpp b/ydb/core/grpc_services/rpc_rollback_transaction.cpp index 7deb30425a..b6cdfe2817 100644 --- a/ydb/core/grpc_services/rpc_rollback_transaction.cpp +++ b/ydb/core/grpc_services/rpc_rollback_transaction.cpp @@ -2,7 +2,7 @@ #include "rpc_calls.h" #include "rpc_kqp_base.h" -#include "rpc_common.h" +#include "rpc_common.h" #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -37,14 +37,14 @@ public: private: void RollbackTransactionImpl(const TActorContext &ctx) { - const auto req = GetProtoRequest(); + const auto req = GetProtoRequest(); const auto traceId = Request_->GetTraceId(); TString sessionId; auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + NYql::TIssues issues; if (CheckSession(req->session_id(), issues)) { ev->Record.MutableRequest()->SetSessionId(req->session_id()); @@ -83,7 +83,7 @@ private: } void ReplyWithResult(StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx) { Request_->SendResult(status, message); Die(ctx); diff --git a/ydb/core/grpc_services/rpc_scheme_base.h b/ydb/core/grpc_services/rpc_scheme_base.h index 2826cb8ad6..0be68e802e 100644 --- a/ydb/core/grpc_services/rpc_scheme_base.h +++ b/ydb/core/grpc_services/rpc_scheme_base.h @@ -14,7 +14,7 @@ protected: using TBase = TRpcOperationRequestActor<TDerived, TRequest>; public: - TRpcSchemeRequestActor(IRequestOpCtx* request) + TRpcSchemeRequestActor(IRequestOpCtx* request) : TBase(request) {} protected: @@ -107,8 +107,8 @@ protected: } case NKikimrScheme::EStatus::StatusResourceExhausted: case NKikimrScheme::EStatus::StatusPreconditionFailed: { - return this->ReplyWithResult(Ydb::StatusIds::PRECONDITION_FAILED, issueMessage, ctx); - } + return this->ReplyWithResult(Ydb::StatusIds::PRECONDITION_FAILED, issueMessage, ctx); + } default: { return this->ReplyWithResult(Ydb::StatusIds::GENERIC_ERROR, issueMessage, ctx); } diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp index f2183febbd..5f95f263c6 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp @@ -309,7 +309,7 @@ private: auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); SetAuthToken(ev, *Request_); SetDatabase(ev, *Request_); - SetRlPath(ev, *Request_); + SetRlPath(ev, *Request_); if (traceId) { ev->Record.SetTraceId(traceId.GetRef()); diff --git a/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp index 99bd0441b6..aa936e7b6e 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp @@ -1,8 +1,8 @@ -#include "service_yql_scripting.h" +#include "service_yql_scripting.h" #include "rpc_kqp_base.h" #include <ydb/public/api/protos/ydb_scripting.pb.h> - + #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/kikimr_issue.h> @@ -16,9 +16,9 @@ namespace NKikimr { namespace NGRpcService { -using TEvStreamExecuteYqlScriptRequest = - TGrpcRequestNoOperationCall<Ydb::Scripting::ExecuteYqlRequest, Ydb::Scripting::ExecuteYqlPartialResponse>; - +using TEvStreamExecuteYqlScriptRequest = + TGrpcRequestNoOperationCall<Ydb::Scripting::ExecuteYqlRequest, Ydb::Scripting::ExecuteYqlPartialResponse>; + namespace { using namespace NActors; @@ -109,7 +109,7 @@ public: return NKikimrServices::TActivity::GRPC_STREAM_REQ; } - TStreamExecuteYqlScriptRPC(IRequestNoOpCtx* request, ui64 rpcBufferSize) + TStreamExecuteYqlScriptRPC(IRequestNoOpCtx* request, ui64 rpcBufferSize) : TBase(request) , RpcBufferSize_(rpcBufferSize) {} @@ -505,9 +505,9 @@ private: } // namespace -void DoStreamExecuteYqlScript(std::unique_ptr<IRequestNoOpCtx> p, const IFacilityProvider& facility) { - ui64 rpcBufferSize = facility.GetAppConfig().GetTableServiceConfig().GetResourceManager().GetChannelBufferSize(); - TActivationContext::AsActorContext().Register(new TStreamExecuteYqlScriptRPC(p.release(), rpcBufferSize)); +void DoStreamExecuteYqlScript(std::unique_ptr<IRequestNoOpCtx> p, const IFacilityProvider& facility) { + ui64 rpcBufferSize = facility.GetAppConfig().GetTableServiceConfig().GetResourceManager().GetChannelBufferSize(); + TActivationContext::AsActorContext().Register(new TStreamExecuteYqlScriptRPC(p.release(), rpcBufferSize)); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_whoami.cpp b/ydb/core/grpc_services/rpc_whoami.cpp index 4d02dec9b4..b8b2dadfcf 100644 --- a/ydb/core/grpc_services/rpc_whoami.cpp +++ b/ydb/core/grpc_services/rpc_whoami.cpp @@ -45,7 +45,7 @@ public: private: void Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev) { const TEvTicketParser::TEvAuthorizeTicketResult& result(*ev->Get()); - auto *response = TEvWhoAmIRequest::AllocateResult<Ydb::Discovery::WhoAmIResult>(Request); + auto *response = TEvWhoAmIRequest::AllocateResult<Ydb::Discovery::WhoAmIResult>(Request); if (!result.Error) { if (result.Token != nullptr) { response->set_user(result.Token->GetUserSID()); @@ -72,7 +72,7 @@ private: void ReplyError(const TString& error) { auto issue = NYql::TIssue(error); Request->RaiseIssue(issue); - Request->ReplyWithYdbStatus(Ydb::StatusIds::GENERIC_ERROR); + Request->ReplyWithYdbStatus(Ydb::StatusIds::GENERIC_ERROR); } THolder<TEvWhoAmIRequest> Request; diff --git a/ydb/core/grpc_services/service_analytics_internal.h b/ydb/core/grpc_services/service_analytics_internal.h index abd00fadb3..f3d1ed966d 100644 --- a/ydb/core/grpc_services/service_analytics_internal.h +++ b/ydb/core/grpc_services/service_analytics_internal.h @@ -1,17 +1,17 @@ -#pragma once - -#include <memory> - -namespace NKikimr { -namespace NGRpcService { - -class IRequestOpCtx; -class IFacilityProvider; - +#pragma once + +#include <memory> + +namespace NKikimr { +namespace NGRpcService { + +class IRequestOpCtx; +class IFacilityProvider; + void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); - -} // namespace NGRpcService -} // namespace NKikimr + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/service_logstore.h b/ydb/core/grpc_services/service_logstore.h index b4c56a5062..3384ca445f 100644 --- a/ydb/core/grpc_services/service_logstore.h +++ b/ydb/core/grpc_services/service_logstore.h @@ -1,21 +1,21 @@ -#pragma once - -#include <memory> - -namespace NKikimr { -namespace NGRpcService { - -class IRequestOpCtx; -class IFacilityProvider; - -void DoCreateLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoDescribeLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoDropLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); - -void DoCreateLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoDescribeLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoDropLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoAlterLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); - -} -} +#pragma once + +#include <memory> + +namespace NKikimr { +namespace NGRpcService { + +class IRequestOpCtx; +class IFacilityProvider; + +void DoCreateLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoDescribeLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoDropLogStoreRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoCreateLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoDescribeLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoDropLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoAlterLogTableRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +} +} diff --git a/ydb/core/grpc_services/service_scheme.h b/ydb/core/grpc_services/service_scheme.h index 7dc86e85ba..def61fdab9 100644 --- a/ydb/core/grpc_services/service_scheme.h +++ b/ydb/core/grpc_services/service_scheme.h @@ -1,18 +1,18 @@ -#pragma once - -#include <memory> - -namespace NKikimr { -namespace NGRpcService { - -class IRequestOpCtx; -class IFacilityProvider; - -void DoMakeDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoRemoveDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoListDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoDescribePathRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoModifyPermissionsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); - -} -} +#pragma once + +#include <memory> + +namespace NKikimr { +namespace NGRpcService { + +class IRequestOpCtx; +class IFacilityProvider; + +void DoMakeDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoRemoveDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoListDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoDescribePathRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoModifyPermissionsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +} +} diff --git a/ydb/core/grpc_services/service_yql_scripting.h b/ydb/core/grpc_services/service_yql_scripting.h index 23a1255d51..5a54bf3b54 100644 --- a/ydb/core/grpc_services/service_yql_scripting.h +++ b/ydb/core/grpc_services/service_yql_scripting.h @@ -1,17 +1,17 @@ -#pragma once - -#include <memory> - -namespace NKikimr { -namespace NGRpcService { - -class IRequestOpCtx; -class IRequestNoOpCtx; -class IFacilityProvider; - -void DoExecuteYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); -void DoStreamExecuteYqlScript(std::unique_ptr<IRequestNoOpCtx> p, const IFacilityProvider& facility); -void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); - -} // namespace NGRpcService -} // namespace NKikimr +#pragma once + +#include <memory> + +namespace NKikimr { +namespace NGRpcService { + +class IRequestOpCtx; +class IRequestNoOpCtx; +class IFacilityProvider; + +void DoExecuteYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); +void DoStreamExecuteYqlScript(std::unique_ptr<IRequestNoOpCtx> p, const IFacilityProvider& facility); +void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/table_profiles.cpp b/ydb/core/grpc_services/table_profiles.cpp index 6a8d6618a4..367de4cc89 100644 --- a/ydb/core/grpc_services/table_profiles.cpp +++ b/ydb/core/grpc_services/table_profiles.cpp @@ -75,10 +75,10 @@ NKikimrSchemeOp::TStorageConfig *TTableProfiles::GetDefaultStorageConfig(NKikimr return GetDefaultFamilyDescription(policy)->MutableStorageConfig(); } -bool TTableProfiles::HasPresetName(const TString &presetName) const { +bool TTableProfiles::HasPresetName(const TString &presetName) const { return TableProfiles.contains(presetName); -} - +} + bool TTableProfiles::ApplyTableProfile(const Ydb::Table::TableProfile &profile, NKikimrSchemeOp::TTableDescription &tableDesc, Ydb::StatusIds::StatusCode &code, diff --git a/ydb/core/grpc_services/table_profiles.h b/ydb/core/grpc_services/table_profiles.h index fa11def951..2acec6f115 100644 --- a/ydb/core/grpc_services/table_profiles.h +++ b/ydb/core/grpc_services/table_profiles.h @@ -17,8 +17,8 @@ public: void Load(const NKikimrConfig::TTableProfilesConfig &config); - bool HasPresetName(const TString& presetName) const; - + bool HasPresetName(const TString& presetName) const; + bool ApplyTableProfile(const Ydb::Table::TableProfile &profile, NKikimrSchemeOp::TTableDescription &tableDesc, Ydb::StatusIds::StatusCode &code, diff --git a/ydb/core/grpc_services/ut/ya.make b/ydb/core/grpc_services/ut/ya.make index d2c77945ec..b1888784b2 100644 --- a/ydb/core/grpc_services/ut/ya.make +++ b/ydb/core/grpc_services/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/core/grpc_services) - + OWNER( dcherednik g:kikimr ) -FORK_SUBTESTS() - +FORK_SUBTESTS() + IF (SANITIZER_TYPE OR WITH_VALGRIND) SIZE(MEDIUM) ENDIF() -SRCS( - rpc_calls_ut.cpp - operation_helpers_ut.cpp -) - -PEERDIR( +SRCS( + rpc_calls_ut.cpp + operation_helpers_ut.cpp +) + +PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion ydb/core/client/scheme_cache_lib ydb/core/testlib -) - -END() +) + +END() diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 68a5f1592c..05b156bf04 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( grpc_endpoint_publish_actor.cpp - grpc_helper.cpp + grpc_helper.cpp grpc_mon.cpp grpc_publisher_service_actor.cpp - grpc_request_proxy.cpp - local_rate_limiter.cpp - operation_helpers.cpp + grpc_request_proxy.cpp + local_rate_limiter.cpp + operation_helpers.cpp resolve_local_db_table.cpp rpc_alter_coordination_node.cpp rpc_alter_table.cpp @@ -25,9 +25,9 @@ SRCS( rpc_copy_tables.cpp rpc_export.cpp rpc_create_coordination_node.cpp - rpc_create_session.cpp + rpc_create_session.cpp rpc_create_table.cpp - rpc_delete_session.cpp + rpc_delete_session.cpp rpc_describe_coordination_node.cpp rpc_describe_path.cpp rpc_describe_table.cpp @@ -45,7 +45,7 @@ SRCS( rpc_get_shard_locations.cpp rpc_import.cpp rpc_import_data.cpp - rpc_keep_alive.cpp + rpc_keep_alive.cpp rpc_kh_describe.cpp rpc_kh_snapshots.cpp rpc_kqp_base.cpp @@ -61,7 +61,7 @@ SRCS( rpc_rate_limiter_api.cpp rpc_read_columns.cpp rpc_read_table.cpp - rpc_remove_directory.cpp + rpc_remove_directory.cpp rpc_rename_tables.cpp rpc_rollback_transaction.cpp rpc_s3_listing.cpp @@ -73,9 +73,9 @@ SRCS( table_profiles.cpp table_settings.cpp rpc_analytics_internal.cpp -) - -PEERDIR( +) + +PEERDIR( contrib/libs/xxhash library/cpp/cgiparam library/cpp/digest/old_crc @@ -109,8 +109,8 @@ PEERDIR( ydb/public/api/protos ydb/public/lib/operation_id ydb/public/sdk/cpp/client/resources -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/kesus/proxy/events.h b/ydb/core/kesus/proxy/events.h index 3ea03a65f0..2062661bf6 100644 --- a/ydb/core/kesus/proxy/events.h +++ b/ydb/core/kesus/proxy/events.h @@ -56,7 +56,7 @@ struct TEvKesusProxy { TEvProxyError(Ydb::StatusIds::StatusCode status, const TString& reason) { Error.SetStatus(status); - Error.AddIssues()->set_message(reason); + Error.AddIssues()->set_message(reason); } }; }; diff --git a/ydb/core/kesus/proxy/proxy.cpp b/ydb/core/kesus/proxy/proxy.cpp index ef8f2d42f5..9633816e60 100644 --- a/ydb/core/kesus/proxy/proxy.cpp +++ b/ydb/core/kesus/proxy/proxy.cpp @@ -103,7 +103,7 @@ private: // Recheck schemecache for changes LOG_TRACE_S(ctx, NKikimrServices::KESUS_PROXY, "Starting resolve for kesus " << msg->KesusPath.Quote()); - RegisterWithSameMailbox(CreateResolveActor(msg->KesusPath)); + RegisterWithSameMailbox(CreateResolveActor(msg->KesusPath)); entry.State = CACHE_STATE_RESOLVING; [[fallthrough]]; @@ -135,7 +135,7 @@ private: "Received an OK result for " << msg->KesusPath.Quote() << " without KesusInfo: not found"); entry.LastError.SetStatus(Ydb::StatusIds::NOT_FOUND); - entry.LastError.AddIssues()->set_message("Kesus not found"); + entry.LastError.AddIssues()->set_message("Kesus not found"); break; } const auto& desc = result.KesusInfo->Description; @@ -176,7 +176,7 @@ private: "Resolve did not find path " << msg->KesusPath.Quote() << ": " << result.Status); entry.LastError.SetStatus(Ydb::StatusIds::NOT_FOUND); - entry.LastError.AddIssues()->set_message("Kesus not found"); + entry.LastError.AddIssues()->set_message("Kesus not found"); break; default: LOG_ERROR_S(ctx, NKikimrServices::KESUS_PROXY, diff --git a/ydb/core/kesus/proxy/proxy_actor.cpp b/ydb/core/kesus/proxy/proxy_actor.cpp index a58d3340b2..e93d90355f 100644 --- a/ydb/core/kesus/proxy/proxy_actor.cpp +++ b/ydb/core/kesus/proxy/proxy_actor.cpp @@ -254,7 +254,7 @@ private: void HandleLinkFailure() { NKikimrKesus::TKesusError error; error.SetStatus(Ydb::StatusIds::UNAVAILABLE); - error.AddIssues()->set_message("Link failure"); + error.AddIssues()->set_message("Link failure"); HandleLinkError(error); } @@ -650,7 +650,7 @@ private: NKikimrKesus::TKesusError error; error.SetStatus(Ydb::StatusIds::BAD_SESSION); - error.AddIssues()->set_message("Proxy session expired"); + error.AddIssues()->set_message("Proxy session expired"); HandleSessionError(error); if (SessionNeeded()) { diff --git a/ydb/core/kesus/tablet/events.h b/ydb/core/kesus/tablet/events.h index 68f9c024eb..872f0149c9 100644 --- a/ydb/core/kesus/tablet/events.h +++ b/ydb/core/kesus/tablet/events.h @@ -99,7 +99,7 @@ struct TEvKesus { inline static void FillError(NKikimrKesus::TKesusError* error, Ydb::StatusIds::StatusCode status, const TString& reason) { Y_VERIFY(status != Ydb::StatusIds::SUCCESS, "Attempting to set error to SUCCESS"); error->SetStatus(status); - error->AddIssues()->set_message(reason); + error->AddIssues()->set_message(reason); } template<class TDerived, class TRecord, ui32 Event> diff --git a/ydb/core/keyvalue/keyvalue_flat_impl.h b/ydb/core/keyvalue/keyvalue_flat_impl.h index 68dcd9269d..292f2feff9 100644 --- a/ydb/core/keyvalue/keyvalue_flat_impl.h +++ b/ydb/core/keyvalue/keyvalue_flat_impl.h @@ -322,7 +322,7 @@ protected: State.SetIsSpringCleanupDone(); } } - CollectorActorId = ctx.RegisterWithSameMailbox(CreateKeyValueCollector(ctx.SelfID, State.GetCollectOperation(), + CollectorActorId = ctx.RegisterWithSameMailbox(CreateKeyValueCollector(ctx.SelfID, State.GetCollectOperation(), Info(), Executor()->Generation(), State.GetPerGenerationCounter(), isSpringCleanup)); State.OnEvCollectDone(perGenerationCounterStepSize, ctx); } else { diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h index 46bfa0f2cc..1e06beca39 100644 --- a/ydb/core/kqp/common/kqp_gateway.h +++ b/ydb/core/kqp/common/kqp_gateway.h @@ -122,7 +122,7 @@ public: bool LlvmEnabled = true; TKqpSnapshot Snapshot = TKqpSnapshot(); NKikimrKqp::EIsolationLevel IsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; - TMaybe<NKikimrKqp::TRlPath> RlPath; + TMaybe<NKikimrKqp::TRlPath> RlPath; }; struct TExecPhysicalResult : public TGenericResult { diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h index 53f39a579d..517d1bc451 100644 --- a/ydb/core/kqp/common/kqp_resolve.h +++ b/ydb/core/kqp/common/kqp_resolve.h @@ -19,8 +19,8 @@ enum class ETableKind { Olap }; -class TKqpTableKeys { -public: +class TKqpTableKeys { +public: struct TColumn { ui32 Id; ui32 Type; @@ -36,8 +36,8 @@ public: TTable* FindTablePtr(const TTableId& id) { return TablesById.FindPtr(id); - } - + } + const TTable* FindTablePtr(const TTableId& id) const { return TablesById.FindPtr(id); } @@ -52,23 +52,23 @@ public: } return table; - } - + } + const TTable& GetTable(const TTableId& id) const { auto table = TablesById.FindPtr(id); MKQL_ENSURE_S(table); return *table; } - size_t Size() const { + size_t Size() const { return TablesById.size(); - } - + } + const THashMap<TTableId, TTable>& Get() const { return TablesById; - } + } -private: +private: THashMap<TTableId, TTable> TablesById; }; diff --git a/ydb/core/kqp/common/kqp_ru_calc.cpp b/ydb/core/kqp/common/kqp_ru_calc.cpp index d3083a51e8..a58f57f8e2 100644 --- a/ydb/core/kqp/common/kqp_ru_calc.cpp +++ b/ydb/core/kqp/common/kqp_ru_calc.cpp @@ -1,94 +1,94 @@ -#include "kqp_ru_calc.h" - +#include "kqp_ru_calc.h" + #include <ydb/core/protos/kqp_stats.pb.h> - + #include <ydb/core/kqp/executer/kqp_executer_stats.h> - + #include <ydb/core/kqp/kqp.h> - -#include <util/generic/size_literals.h> - -#include <cmath> - -namespace NKikimr { -namespace NKqp { -namespace NRuCalc { - -namespace { - -ui64 CalcReadIORu(const TTableStat& stat) { - constexpr ui64 bytesPerUnit = 4_KB; - constexpr ui64 bytesPerUnitAdjust = bytesPerUnit - 1; - - auto bytes = (stat.Bytes + bytesPerUnitAdjust) / bytesPerUnit; - return std::max(bytes, stat.Rows); -} - -class TIoReadStat: public TTableStat { -public: + +#include <util/generic/size_literals.h> + +#include <cmath> + +namespace NKikimr { +namespace NKqp { +namespace NRuCalc { + +namespace { + +ui64 CalcReadIORu(const TTableStat& stat) { + constexpr ui64 bytesPerUnit = 4_KB; + constexpr ui64 bytesPerUnitAdjust = bytesPerUnit - 1; + + auto bytes = (stat.Bytes + bytesPerUnitAdjust) / bytesPerUnit; + return std::max(bytes, stat.Rows); +} + +class TIoReadStat: public TTableStat { +public: void Add(const NYql::NDqProto::TDqTableStats& tableAggr) { Rows += tableAggr.GetReadRows(); Bytes += tableAggr.GetReadBytes(); - } - - ui64 CalcRu() const { - return CalcReadIORu(*this); - } -}; - -class TIoWriteStat: public TTableStat { -public: + } + + ui64 CalcRu() const { + return CalcReadIORu(*this); + } +}; + +class TIoWriteStat: public TTableStat { +public: void Add(const NYql::NDqProto::TDqTableStats& tableAggr) { Rows += tableAggr.GetWriteRows(); Rows += tableAggr.GetEraseRows(); Bytes += tableAggr.GetWriteBytes(); - } - - ui64 CalcRu() const { - constexpr ui64 bytesPerUnit = 1_KB; - constexpr ui64 bytesPerUnitAdjust = bytesPerUnit - 1; - - auto bytes = (Bytes + bytesPerUnitAdjust) / bytesPerUnit; - return 2 * std::max(bytes, Rows); - } -}; - -} - -ui64 CpuTimeToUnit(TDuration cpuTime) { - return std::floor(cpuTime.MicroSeconds() / 1500.0); -} - -ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats) { - TDuration totalCpuTime; - TIoReadStat totalReadStat; - TIoWriteStat totalWriteStat; - - for (const auto& exec : stats.GetExecutions()) { + } + + ui64 CalcRu() const { + constexpr ui64 bytesPerUnit = 1_KB; + constexpr ui64 bytesPerUnitAdjust = bytesPerUnit - 1; + + auto bytes = (Bytes + bytesPerUnitAdjust) / bytesPerUnit; + return 2 * std::max(bytes, Rows); + } +}; + +} + +ui64 CpuTimeToUnit(TDuration cpuTime) { + return std::floor(cpuTime.MicroSeconds() / 1500.0); +} + +ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats) { + TDuration totalCpuTime; + TIoReadStat totalReadStat; + TIoWriteStat totalWriteStat; + + for (const auto& exec : stats.GetExecutions()) { totalCpuTime += TDuration::MicroSeconds(exec.GetCpuTimeUs()); - + for (auto& table : exec.GetTables()) { - totalReadStat.Add(table); - } - } - - if (stats.HasCompilation()) { - totalCpuTime += TDuration::MicroSeconds(stats.GetCompilation().GetCpuTimeUs()); - } - - totalCpuTime += TDuration::MicroSeconds(stats.GetWorkerCpuTimeUs()); - - auto totalIoRu = totalReadStat.CalcRu() + totalWriteStat.CalcRu(); - - return std::max(std::max(CpuTimeToUnit(totalCpuTime), totalIoRu), (ui64)1); -} - -ui64 CalcRequestUnit(const TProgressStatEntry& stats) { + totalReadStat.Add(table); + } + } + + if (stats.HasCompilation()) { + totalCpuTime += TDuration::MicroSeconds(stats.GetCompilation().GetCpuTimeUs()); + } + + totalCpuTime += TDuration::MicroSeconds(stats.GetWorkerCpuTimeUs()); + + auto totalIoRu = totalReadStat.CalcRu() + totalWriteStat.CalcRu(); + + return std::max(std::max(CpuTimeToUnit(totalCpuTime), totalIoRu), (ui64)1); +} + +ui64 CalcRequestUnit(const TProgressStatEntry& stats) { auto ioRu = CalcReadIORu(stats.ReadIOStat); - + return std::max(std::max(CpuTimeToUnit(stats.ComputeTime), ioRu), (ui64)1); -} - -} // namespace NRuCalc -} // namespace NKqp -} // namespace NKikimr +} + +} // namespace NRuCalc +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/common/kqp_ru_calc.h b/ydb/core/kqp/common/kqp_ru_calc.h index f1c69ea843..b701204959 100644 --- a/ydb/core/kqp/common/kqp_ru_calc.h +++ b/ydb/core/kqp/common/kqp_ru_calc.h @@ -1,23 +1,23 @@ -#pragma once - -#include <util/system/types.h> -#include <util/datetime/base.h> - -namespace NKqpProto { - class TKqpStatsQuery; -} - -namespace NKikimr { -namespace NKqp { - -struct TProgressStatEntry; - -namespace NRuCalc { - -ui64 CpuTimeToUnit(TDuration cpuTimeUs); -ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats); -ui64 CalcRequestUnit(const TProgressStatEntry& stats); - -} // namespace NRuCalc -} // namespace NKqp -} // namespace NKikimr +#pragma once + +#include <util/system/types.h> +#include <util/datetime/base.h> + +namespace NKqpProto { + class TKqpStatsQuery; +} + +namespace NKikimr { +namespace NKqp { + +struct TProgressStatEntry; + +namespace NRuCalc { + +ui64 CpuTimeToUnit(TDuration cpuTimeUs); +ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats); +ui64 CalcRequestUnit(const TProgressStatEntry& stats); + +} // namespace NRuCalc +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/common/ya.make b/ydb/core/kqp/common/ya.make index 82ea4bb9c5..1e694fda12 100644 --- a/ydb/core/kqp/common/ya.make +++ b/ydb/core/kqp/common/ya.make @@ -10,7 +10,7 @@ SRCS( kqp_common.h kqp_resolve.cpp kqp_resolve.h - kqp_ru_calc.cpp + kqp_ru_calc.cpp kqp_transform.cpp kqp_transform.h kqp_yql.cpp diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp index d720f99c4e..0413044698 100644 --- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp +++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp @@ -311,7 +311,7 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& } YQL_ENSURE(keySet.empty()); - YQL_ENSURE(tableMeta.KeyColumnNames.size() + upsertSet.size() == upsertRows.Columns().Size()); + YQL_ENSURE(tableMeta.KeyColumnNames.size() + upsertSet.size() == upsertRows.Columns().Size()); TVector<TStringBuf> upsertColumns(upsertSet.begin(), upsertSet.end()); auto result = ctx.PgmBuilder().KqpUpsertRows(MakeTableId(upsertRows.Table()), rows, diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp index 3d27fd9779..f8e3cd0c77 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp @@ -52,7 +52,7 @@ static constexpr TDuration MAX_RETRY_DELAY = TDuration::Seconds(2); static constexpr ui64 MAX_SHARD_RETRIES = 5; // retry after: 0, 250, 500, 1000, 2000 static constexpr ui64 MAX_TOTAL_SHARD_RETRIES = 20; static constexpr ui64 MAX_SHARD_RESOLVES = 3; -static constexpr TDuration RL_MAX_BATCH_DELAY = TDuration::Seconds(50); +static constexpr TDuration RL_MAX_BATCH_DELAY = TDuration::Seconds(50); class TKqpScanComputeActor : public TDqComputeActorBase<TKqpScanComputeActor> { @@ -173,23 +173,23 @@ public: ReportEventElapsedTime(); } - void HandleEvWakeup(EEvWakeupTag tag) { - switch (tag) { - case RlSendAllowedTag: - ProcessScanData(); - break; - case RlNoResourceTag: - ProcessRlNoResourceAndDie(); - break; - case TimeoutTag: - Y_FAIL("TimeoutTag must be handled in base class"); - break; - case PeriodicStatsTag: - Y_FAIL("PeriodicStatsTag must be handled in base class"); - break; - } - } - + void HandleEvWakeup(EEvWakeupTag tag) { + switch (tag) { + case RlSendAllowedTag: + ProcessScanData(); + break; + case RlNoResourceTag: + ProcessRlNoResourceAndDie(); + break; + case TimeoutTag: + Y_FAIL("TimeoutTag must be handled in base class"); + break; + case PeriodicStatsTag: + Y_FAIL("PeriodicStatsTag must be handled in base class"); + break; + } + } + void FillExtraStats(NDqProto::TDqComputeActorStats* dst, bool last) { if (last && ScanData && dst->TasksSize() > 0) { YQL_ENSURE(dst->TasksSize() == 1); @@ -232,17 +232,17 @@ protected: } private: - void ProcessRlNoResourceAndDie() { - const NYql::TIssue issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, - "Throughput limit exceeded for query"); - CA_LOG_E("Throughput limit exceeded, we got " - << PendingScanData.size() << " pending messages," - << " stream will be terminated"); - + void ProcessRlNoResourceAndDie() { + const NYql::TIssue issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, + "Throughput limit exceeded for query"); + CA_LOG_E("Throughput limit exceeded, we got " + << PendingScanData.size() << " pending messages," + << " stream will be terminated"); + State = NDqProto::COMPUTE_STATE_FAILURE; ReportStateAndMaybeDie(Ydb::StatusIds::OVERLOADED, TIssues({issue})); - } - + } + void HandleExecute(TEvKqpCompute::TEvScanInitActor::TPtr& ev) { Y_VERIFY_DEBUG(ScanData); Y_VERIFY_DEBUG(!Shards.empty()); @@ -280,7 +280,7 @@ private: case EShardState::Initial: case EShardState::Running: - case EShardState::PostRunning: + case EShardState::PostRunning: case EShardState::Resolving: { TerminateExpiredScan(scanActorId, "Got unexpected/expired EvScanInitActor, terminate it"); return; @@ -296,84 +296,84 @@ private: auto& state = Shards.front(); switch (state.State) { - case EShardState::Running: { + case EShardState::Running: { + if (state.Generation == msg.Generation) { + YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender); + + TInstant startTime = TActivationContext::Now(); + if (ev->Get()->Finished) { + state.State = EShardState::PostRunning; + } + PendingScanData.emplace_back(std::make_pair(ev, startTime)); + + if (auto rlPath = GetRlPath()) { + auto selfId = this->SelfId(); + auto as = TActivationContext::ActorSystem(); + + auto onSendAllowed = [selfId, as]() mutable { + as->Send(selfId, new TEvents::TEvWakeup(EEvWakeupTag::RlSendAllowedTag)); + }; + + auto onSendTimeout = [selfId, as]() { + as->Send(selfId, new TEvents::TEvWakeup(EEvWakeupTag::RlNoResourceTag)); + }; + + const NRpcService::TRlFullPath rlFullPath { + .CoordinationNode = rlPath->GetCoordinationNode(), + .ResourcePath = rlPath->GetResourcePath(), + .DatabaseName = rlPath->GetDatabase(), + .Token = rlPath->GetToken() + }; + + auto rlActor = NRpcService::RateLimiterAcquireUseSameMailbox( + rlFullPath, 0, RL_MAX_BATCH_DELAY, + std::move(onSendAllowed), std::move(onSendTimeout), TActivationContext::AsActorContext()); + + CA_LOG_D("Launch rate limiter actor: " << rlActor); + } else { + ProcessScanData(); + } + + } else if (state.Generation > msg.Generation) { + TerminateExpiredScan(ev->Sender, "Cancel expired scan"); + } else { + YQL_ENSURE(false, "EvScanData from the future, expected: " << state.Generation << ", got: " << msg.Generation); + } + break; + } + case EShardState::PostRunning: + TerminateExpiredScan(ev->Sender, "Unexpected data after finish"); + break; + case EShardState::Initial: + case EShardState::Starting: + case EShardState::Resolving: + TerminateExpiredScan(ev->Sender, "Cancel unexpected scan"); + break; + } + } + + void ProcessScanData() { + Y_VERIFY_DEBUG(ScanData); + Y_VERIFY_DEBUG(!Shards.empty()); + Y_VERIFY(PendingScanData); + + auto& ev = PendingScanData.front().first; + + TDuration latency; + if (PendingScanData.front().second != TInstant::Zero()) { + latency = TActivationContext::Now() - PendingScanData.front().second; + Counters->ScanQueryRateLimitLatency->Collect(latency.MilliSeconds()); + } + + auto& msg = *ev->Get(); + auto& state = Shards.front(); + + switch (state.State) { + case EShardState::Running: + case EShardState::PostRunning: { if (state.Generation == msg.Generation) { YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender); - TInstant startTime = TActivationContext::Now(); - if (ev->Get()->Finished) { - state.State = EShardState::PostRunning; - } - PendingScanData.emplace_back(std::make_pair(ev, startTime)); - - if (auto rlPath = GetRlPath()) { - auto selfId = this->SelfId(); - auto as = TActivationContext::ActorSystem(); - - auto onSendAllowed = [selfId, as]() mutable { - as->Send(selfId, new TEvents::TEvWakeup(EEvWakeupTag::RlSendAllowedTag)); - }; - - auto onSendTimeout = [selfId, as]() { - as->Send(selfId, new TEvents::TEvWakeup(EEvWakeupTag::RlNoResourceTag)); - }; - - const NRpcService::TRlFullPath rlFullPath { - .CoordinationNode = rlPath->GetCoordinationNode(), - .ResourcePath = rlPath->GetResourcePath(), - .DatabaseName = rlPath->GetDatabase(), - .Token = rlPath->GetToken() - }; - - auto rlActor = NRpcService::RateLimiterAcquireUseSameMailbox( - rlFullPath, 0, RL_MAX_BATCH_DELAY, - std::move(onSendAllowed), std::move(onSendTimeout), TActivationContext::AsActorContext()); - - CA_LOG_D("Launch rate limiter actor: " << rlActor); - } else { - ProcessScanData(); - } - - } else if (state.Generation > msg.Generation) { - TerminateExpiredScan(ev->Sender, "Cancel expired scan"); - } else { - YQL_ENSURE(false, "EvScanData from the future, expected: " << state.Generation << ", got: " << msg.Generation); - } - break; - } - case EShardState::PostRunning: - TerminateExpiredScan(ev->Sender, "Unexpected data after finish"); - break; - case EShardState::Initial: - case EShardState::Starting: - case EShardState::Resolving: - TerminateExpiredScan(ev->Sender, "Cancel unexpected scan"); - break; - } - } - - void ProcessScanData() { - Y_VERIFY_DEBUG(ScanData); - Y_VERIFY_DEBUG(!Shards.empty()); - Y_VERIFY(PendingScanData); - - auto& ev = PendingScanData.front().first; - - TDuration latency; - if (PendingScanData.front().second != TInstant::Zero()) { - latency = TActivationContext::Now() - PendingScanData.front().second; - Counters->ScanQueryRateLimitLatency->Collect(latency.MilliSeconds()); - } - - auto& msg = *ev->Get(); - auto& state = Shards.front(); - - switch (state.State) { - case EShardState::Running: - case EShardState::PostRunning: { - if (state.Generation == msg.Generation) { - YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender); - LastKey = std::move(msg.LastKey); ui64 bytes = 0; @@ -400,10 +400,10 @@ private: } CA_LOG_D("Got EvScanData, rows: " << rowsCount << ", bytes: " << bytes << ", finished: " << msg.Finished - << ", from: " << ev->Sender << ", shards remain: " << Shards.size() - << ", delayed for: " << latency.SecondsFloat() << " seconds by ratelimitter"); + << ", from: " << ev->Sender << ", shards remain: " << Shards.size() + << ", delayed for: " << latency.SecondsFloat() << " seconds by ratelimitter"); - if (rowsCount == 0 && !msg.Finished && state.State != EShardState::PostRunning) { + if (rowsCount == 0 && !msg.Finished && state.State != EShardState::PostRunning) { ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes() ? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes() : 0ul; @@ -441,22 +441,22 @@ private: DoExecute(); - } else if (state.Generation > msg.Generation) { + } else if (state.Generation > msg.Generation) { TerminateExpiredScan(ev->Sender, "Cancel expired scan"); - } else { - YQL_ENSURE(false, "EvScanData from the future, expected: " << state.Generation << ", got: " << msg.Generation); + } else { + YQL_ENSURE(false, "EvScanData from the future, expected: " << state.Generation << ", got: " << msg.Generation); } - break; + break; } - case EShardState::Initial: - case EShardState::Starting: - case EShardState::Resolving: { + case EShardState::Initial: + case EShardState::Starting: + case EShardState::Resolving: { TerminateExpiredScan(ev->Sender, "Cancel unexpected scan"); - break; + break; } } - PendingScanData.pop_front(); + PendingScanData.pop_front(); } void HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) { @@ -472,7 +472,7 @@ private: auto& state = Shards.front(); switch (state.State) { - case EShardState::Starting: { + case EShardState::Starting: { if (state.Generation == msg.GetGeneration()) { CA_LOG_W("Got EvScanError while starting scan, status: " << Ydb::StatusIds_StatusCode_Name(status) << ", reason: " << issues.ToString()); @@ -511,11 +511,11 @@ private: YQL_ENSURE(false, "Got EvScanError from the future, expected: " << state.Generation << ", got: " << msg.GetGeneration()); - break; + break; } - case EShardState::PostRunning: - case EShardState::Running: { + case EShardState::PostRunning: + case EShardState::Running: { if (state.Generation == msg.GetGeneration()) { CA_LOG_W("Got EvScanError while running scan, status: " << Ydb::StatusIds_StatusCode_Name(status) << ", reason: " << issues.ToString() << ", restart"); @@ -536,8 +536,8 @@ private: << ", got: " << msg.GetGeneration()); } - case EShardState::Initial: - case EShardState::Resolving: { + case EShardState::Initial: + case EShardState::Resolving: { // do nothing return; } @@ -561,8 +561,8 @@ private: } switch (state.State) { - case EShardState::Starting: - case EShardState::Running: { + case EShardState::Starting: + case EShardState::Running: { Counters->ScanQueryShardDisconnect->Inc(); if (state.TotalRetries >= MAX_TOTAL_SHARD_RETRIES) { @@ -604,9 +604,9 @@ private: return; } - case EShardState::Initial: - case EShardState::Resolving: - case EShardState::PostRunning: { + case EShardState::Initial: + case EShardState::Resolving: + case EShardState::PostRunning: { CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId << ", state: " << (int) state.State); return; @@ -753,8 +753,8 @@ private: } auto& shard = Shards.front(); if (shard.State == EShardState::Running && ev->Sender == shard.ActorId) { - CA_LOG_E("TEvScanDataAck lost while running scan, terminate execution. DataShard actor: " - << shard.ActorId); + CA_LOG_E("TEvScanDataAck lost while running scan, terminate execution. DataShard actor: " + << shard.ActorId); InternalError(TIssuesIds::DEFAULT_ERROR, "Delivery problem: EvScanDataAck lost."); } else { CA_LOG_D("Skip lost TEvScanDataAck to " << ev->Sender << ", active scan actor: " << shard.ActorId); @@ -986,9 +986,9 @@ private: CA_LOG_T("Scan over tablet " << state.TabletId << " finished: " << ScanData->IsFinished() << ", prevFreeSpace: " << prevFreeSpace << ", freeSpace: " << freeSpace << ", peer: " << state.ActorId); - if (!ScanData->IsFinished() && state.State != EShardState::PostRunning - && prevFreeSpace < freeSpace && state.ActorId) - { + if (!ScanData->IsFinished() && state.State != EShardState::PostRunning + && prevFreeSpace < freeSpace && state.ActorId) + { CA_LOG_T("[poll] Send EvScanDataAck to " << state.ActorId << ", gen: " << state.Generation << ", freeSpace: " << freeSpace); SendScanDataAck(state, freeSpace); @@ -1041,13 +1041,13 @@ private: NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr; TOwnedCellVec LastKey; - TDeque<std::pair<TEvKqpCompute::TEvScanData::TPtr, TInstant>> PendingScanData; + TDeque<std::pair<TEvKqpCompute::TEvScanData::TPtr, TInstant>> PendingScanData; - enum class EShardState { + enum class EShardState { Initial, Starting, Running, - PostRunning, //We already recieve all data, we has not processed it yet. + PostRunning, //We already recieve all data, we has not processed it yet. Resolving, }; diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp index 0265881cb3..26a28a0249 100644 --- a/ydb/core/kqp/counters/kqp_counters.cpp +++ b/ydb/core/kqp/counters/kqp_counters.cpp @@ -738,10 +738,10 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons /* Scan queries */ ScanQueryShardDisconnect = KqpGroup->GetCounter("ScanQuery/ShardDisconnect", true); ScanQueryShardResolve = KqpGroup->GetCounter("ScanQuery/ShardResolve", true); - ScanQueryRateLimitLatency = KqpGroup->GetHistogram( - "ScanQuery/RateLimitLatency", NMonitoring::ExponentialHistogram(20, 2, 1)); + ScanQueryRateLimitLatency = KqpGroup->GetHistogram( + "ScanQuery/RateLimitLatency", NMonitoring::ExponentialHistogram(20, 2, 1)); + - // NewEngine NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( "Query/ExecuteLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1)); diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h index dc75a4ab4e..53a8f7d2c2 100644 --- a/ydb/core/kqp/counters/kqp_counters.h +++ b/ydb/core/kqp/counters/kqp_counters.h @@ -340,7 +340,7 @@ public: // Scan queries counters NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardDisconnect; NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardResolve; - NMonitoring::THistogramPtr ScanQueryRateLimitLatency; + NMonitoring::THistogramPtr ScanQueryRateLimitLatency; // NewEngine vs OldEngine THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies; diff --git a/ydb/core/kqp/executer/kqp_data_executer.cpp b/ydb/core/kqp/executer/kqp_data_executer.cpp index 77f1bd989c..003d4ffa20 100644 --- a/ydb/core/kqp/executer/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer/kqp_data_executer.cpp @@ -127,9 +127,9 @@ public: return NKikimrServices::TActivity::KQP_DATA_EXECUTER_ACTOR; } - TKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, + TKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters) - : TBase(std::move(request), database, userToken, counters) + : TBase(std::move(request), database, userToken, counters) { YQL_ENSURE(Request.IsolationLevel != NKikimrKqp::ISOLATION_LEVEL_UNDEFINED); @@ -918,7 +918,7 @@ private: auto sb = TStringBuilder() << "Waiting for " << PendingComputeActors.size() << " compute actor(s) and " << notFinished << " datashard(s): "; for (auto shardId : PendingComputeActors) { - sb << "CA " << shardId.first << ", "; + sb << "CA " << shardId.first << ", "; } for (auto& [shardId, shardState] : ShardStates) { if (shardState.State != TShardState::EState::Finished) { @@ -1241,7 +1241,7 @@ private: LOG_D("Executing task: " << taskId << " on compute actor: " << task.ComputeActorId); - auto result = PendingComputeActors.emplace(task.ComputeActorId, TProgressStat()); + auto result = PendingComputeActors.emplace(task.ComputeActorId, TProgressStat()); YQL_ENSURE(result.second); } @@ -1758,10 +1758,10 @@ private: } // namespace -IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, +IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters) { - return new TKqpDataExecuter(std::move(request), database, userToken, counters); + return new TKqpDataExecuter(std::move(request), database, userToken, counters); } } // namespace NKqp diff --git a/ydb/core/kqp/executer/kqp_executer_impl.cpp b/ydb/core/kqp/executer/kqp_executer_impl.cpp index a3357e918b..0931e5d2c7 100644 --- a/ydb/core/kqp/executer/kqp_executer_impl.cpp +++ b/ydb/core/kqp/executer/kqp_executer_impl.cpp @@ -4,7 +4,7 @@ #include <ydb/core/kqp/runtime/kqp_transport.h> #include <ydb/public/api/protos/ydb_rate_limiter.pb.h> - + #include <ydb/library/yql/dq/runtime/dq_transport.h> #include <ydb/library/yql/dq/runtime/dq_arrow_helpers.h> @@ -77,7 +77,7 @@ std::pair<TString, TString> SerializeKqpTasksParametersForOlap(const NKqpProto:: const NYql::NDq::TMkqlValueRef* mkqlValue = stageInfo.Meta.Tx.Params.Values.FindPtr(name); - auto [type, value] = ImportValueFromProto(mkqlValue->GetType(), mkqlValue->GetValue(), typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(mkqlValue->GetType(), mkqlValue->GetValue(), typeEnv, holderFactory); YQL_ENSURE(NYql::NArrow::IsArrowCompatible(type), "Incompatible parameter type. Can't convert to arrow"); @@ -104,25 +104,25 @@ std::pair<TString, TString> SerializeKqpTasksParametersForOlap(const NKqpProto:: ); } -TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken, - const NKikimrKqp::TRlPath& path) -{ - Ydb::RateLimiter::AcquireResourceRequest req; - req.set_coordination_node_path(path.GetCoordinationNode()); - req.set_resource_path(path.GetResourcePath()); - req.set_used(ru); - - // No need to handle result of rate limiter response on the response hook - // just report ru usage - auto noop = [](Ydb::RateLimiter::AcquireResourceResponse) {}; - return NKikimr::NRpcService::RateLimiterAcquireUseSameMailbox( - std::move(req), - database, - userToken, - std::move(noop), - TActivationContext::AsActorContext()); -} - +TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken, + const NKikimrKqp::TRlPath& path) +{ + Ydb::RateLimiter::AcquireResourceRequest req; + req.set_coordination_node_path(path.GetCoordinationNode()); + req.set_resource_path(path.GetResourcePath()); + req.set_used(ru); + + // No need to handle result of rate limiter response on the response hook + // just report ru usage + auto noop = [](Ydb::RateLimiter::AcquireResourceResponse) {}; + return NKikimr::NRpcService::RateLimiterAcquireUseSameMailbox( + std::move(req), + database, + userToken, + std::move(noop), + TActivationContext::AsActorContext()); +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, @@ -131,7 +131,7 @@ IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TSt if (request.Transactions.empty()) { // commit-only or rollback-only data transaction YQL_ENSURE(request.EraseLocks); - return CreateKqpDataExecuter(std::move(request), database, userToken, counters); + return CreateKqpDataExecuter(std::move(request), database, userToken, counters); } if (request.Locks.empty()) { @@ -181,7 +181,7 @@ IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TSt } return data - ? CreateKqpDataExecuter(std::move(request), database, userToken, counters) + ? CreateKqpDataExecuter(std::move(request), database, userToken, counters) : CreateKqpScanExecuter(std::move(request), database, userToken, counters); } diff --git a/ydb/core/kqp/executer/kqp_executer_impl.h b/ydb/core/kqp/executer/kqp_executer_impl.h index 08e4589c0c..6ecf78ab05 100644 --- a/ydb/core/kqp/executer/kqp_executer_impl.h +++ b/ydb/core/kqp/executer/kqp_executer_impl.h @@ -6,7 +6,7 @@ #include "kqp_table_resolver.h" #include <ydb/core/kqp/common/kqp_ru_calc.h> - + #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/kikimr_issue.h> @@ -71,16 +71,16 @@ inline bool IsDebugLogEnabled() { TlsActivationContext->LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER); } -TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken, - const NKikimrKqp::TRlPath& path); - +TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken, + const NKikimrKqp::TRlPath& path); + template <class TDerived, EExecType ExecType> class TKqpExecuterBase : public TActorBootstrapped<TDerived> { public: - TKqpExecuterBase(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, + TKqpExecuterBase(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters) : Request(std::move(request)) - , Database(database) + , Database(database) , UserToken(userToken) , Counters(counters) { @@ -254,55 +254,55 @@ protected: } } - void UpdateResourcesUsage(bool force) { - TInstant now = TActivationContext::Now(); - if ((now - LastResourceUsageUpdate < ResourceUsageUpdateInterval) && !force) - return; - - LastResourceUsageUpdate = now; - - TProgressStat::TEntry consumption; - for (const auto& p : PendingComputeActors) { - const auto& t = p.second.GetLastUsage(); - consumption += t; - } - - for (const auto& p : LastStats) { - const auto& t = p.GetLastUsage(); - consumption += t; - } - - auto ru = NRuCalc::CalcRequestUnit(consumption); - - // Some heuristic to reduce overprice due to round part stats - if (ru <= 100 && !force) - return; - - for (auto& p : PendingComputeActors) { - p.second.Update(); - } - - for (auto& p : LastStats) { - p.Update(); - } - - if (Request.RlPath) { - auto actorId = ReportToRl(ru, Database, UserToken.GetOrElse(""), - Request.RlPath.GetRef()); - - LOG_D("Resource usage for last stat interval: " << consumption - << " ru: " << ru << " rl path: " << Request.RlPath.GetRef() - << " rl actor: " << actorId - << " force flag: " << force); - } else { - LOG_D("Resource usage for last stat interval: " << consumption - << " ru: " << ru << " rate limiter was not found" - << " force flag: " << force); - } - } - - - + void UpdateResourcesUsage(bool force) { + TInstant now = TActivationContext::Now(); + if ((now - LastResourceUsageUpdate < ResourceUsageUpdateInterval) && !force) + return; + + LastResourceUsageUpdate = now; + + TProgressStat::TEntry consumption; + for (const auto& p : PendingComputeActors) { + const auto& t = p.second.GetLastUsage(); + consumption += t; + } + + for (const auto& p : LastStats) { + const auto& t = p.GetLastUsage(); + consumption += t; + } + + auto ru = NRuCalc::CalcRequestUnit(consumption); + + // Some heuristic to reduce overprice due to round part stats + if (ru <= 100 && !force) + return; + + for (auto& p : PendingComputeActors) { + p.second.Update(); + } + + for (auto& p : LastStats) { + p.Update(); + } + + if (Request.RlPath) { + auto actorId = ReportToRl(ru, Database, UserToken.GetOrElse(""), + Request.RlPath.GetRef()); + + LOG_D("Resource usage for last stat interval: " << consumption + << " ru: " << ru << " rl path: " << Request.RlPath.GetRef() + << " rl actor: " << actorId + << " force flag: " << force); + } else { + LOG_D("Resource usage for last stat interval: " << consumption + << " ru: " << ru << " rate limiter was not found" + << " force flag: " << force); + } + } + + + protected: void BuildSysViewScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) @@ -462,7 +462,7 @@ protected: auto* itemsLimitParam = stageInfo.Meta.Tx.Params.Values.FindPtr(itemsLimitParamName); YQL_ENSURE(itemsLimitParam); - auto [type, value] = NMiniKQL::ImportValueFromProto(itemsLimitParam->GetType(), itemsLimitParam->GetValue(), typeEnv, holderFactory); + auto [type, value] = NMiniKQL::ImportValueFromProto(itemsLimitParam->GetType(), itemsLimitParam->GetValue(), typeEnv, holderFactory); YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::Data); itemsLimit = value.Get<ui64>(); @@ -532,10 +532,10 @@ protected: google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues) { for (auto computeActor : PendingComputeActors) { - LOG_D("terminate compute actor " << computeActor.first); + LOG_D("terminate compute actor " << computeActor.first); auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(status, "Terminate execution"); - this->Send(computeActor.first, ev.Release()); + this->Send(computeActor.first, ev.Release()); } auto& response = *ResponseEv->Record.MutableResponse(); @@ -599,8 +599,8 @@ protected: protected: IKqpGateway::TExecPhysicalRequest Request; - const TString Database; - const TMaybe<TString> UserToken; + const TString Database; + const TMaybe<TString> UserToken; TKqpRequestCounters::TPtr Counters; std::unique_ptr<TQueryExecutionStats> Stats; TInstant StartTime; @@ -615,23 +615,23 @@ protected: TKqpTableKeys TableKeys; TActorId KqpTableResolverId; - THashMap<TActorId, TProgressStat> PendingComputeActors; // Running compute actors (pure and DS) - TVector<TProgressStat> LastStats; + THashMap<TActorId, TProgressStat> PendingComputeActors; // Running compute actors (pure and DS) + TVector<TProgressStat> LastStats; TInstant StartResolveTime; - TInstant LastResourceUsageUpdate; + TInstant LastResourceUsageUpdate; std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ResponseEv; -private: - static constexpr TDuration ResourceUsageUpdateInterval = TDuration::MilliSeconds(100); +private: + static constexpr TDuration ResourceUsageUpdateInterval = TDuration::MilliSeconds(100); }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// IActor* CreateKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters); -IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, - const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters); +IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, + const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters); IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters); diff --git a/ydb/core/kqp/executer/kqp_executer_stats.cpp b/ydb/core/kqp/executer/kqp_executer_stats.cpp index 462b84200f..8c3605fd91 100644 --- a/ydb/core/kqp/executer/kqp_executer_stats.cpp +++ b/ydb/core/kqp/executer/kqp_executer_stats.cpp @@ -11,15 +11,15 @@ namespace { TTableStat operator - (const TTableStat& l, const TTableStat& r) { return {.Rows = l.Rows - r.Rows, .Bytes = l.Bytes - r.Bytes}; -} - -TProgressStatEntry operator - (const TProgressStatEntry& l, const TProgressStatEntry& r) { - return TProgressStatEntry { +} + +TProgressStatEntry operator - (const TProgressStatEntry& l, const TProgressStatEntry& r) { + return TProgressStatEntry { .ComputeTime = l.ComputeTime - r.ComputeTime, .ReadIOStat = l.ReadIOStat - r.ReadIOStat - }; -} - + }; +} + void UpdateAggr(NDqProto::TDqStatsAggr* aggr, ui64 value) noexcept { if (aggr->GetMin() == 0) { aggr->SetMin(value); @@ -252,37 +252,37 @@ void TQueryExecutionStats::Finish() { // Cerr << (TStringBuilder() << "TQueryExecutionStats::Finish" << Endl << Result->DebugString() << Endl); } -TTableStat& TTableStat::operator +=(const TTableStat& rhs) { - Rows += rhs.Rows; - Bytes += rhs.Bytes; - return *this; -} - -TTableStat& TTableStat::operator -=(const TTableStat& rhs) { - Rows -= rhs.Rows; - Bytes -= rhs.Bytes; - return *this; -} - -TProgressStatEntry& TProgressStatEntry::operator +=(const TProgressStatEntry& rhs) { +TTableStat& TTableStat::operator +=(const TTableStat& rhs) { + Rows += rhs.Rows; + Bytes += rhs.Bytes; + return *this; +} + +TTableStat& TTableStat::operator -=(const TTableStat& rhs) { + Rows -= rhs.Rows; + Bytes -= rhs.Bytes; + return *this; +} + +TProgressStatEntry& TProgressStatEntry::operator +=(const TProgressStatEntry& rhs) { ComputeTime += rhs.ComputeTime; ReadIOStat += rhs.ReadIOStat; - - return *this; -} - + + return *this; +} + TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry) { return entry.ReadIOStat; -} - -TDuration CalcCumComputeTime(const TProgressStatEntry& entry) { +} + +TDuration CalcCumComputeTime(const TProgressStatEntry& entry) { return entry.ComputeTime; -} - -void TProgressStatEntry::Out(IOutputStream& o) const { +} + +void TProgressStatEntry::Out(IOutputStream& o) const { o << "ComputeTime: " << ComputeTime << " ReadRows: " << ReadIOStat.Rows << " ReadBytes: " << ReadIOStat.Bytes; -} - +} + void TProgressStat::Set(const NDqProto::TDqComputeActorStats& stats) { Cur.ComputeTime += TDuration::MicroSeconds(stats.GetCpuTimeUs()); for (auto& task : stats.GetTasks()) { @@ -290,16 +290,16 @@ void TProgressStat::Set(const NDqProto::TDqComputeActorStats& stats) { Cur.ReadIOStat.Rows += table.GetReadRows(); Cur.ReadIOStat.Bytes += table.GetReadBytes(); } - } -} - -TProgressStat::TEntry TProgressStat::GetLastUsage() const { - return Cur - Total; -} - -void TProgressStat::Update() { - Total = Cur; - Cur = TEntry(); -} - + } +} + +TProgressStat::TEntry TProgressStat::GetLastUsage() const { + return Cur - Total; +} + +void TProgressStat::Update() { + Total = Cur; + Cur = TEntry(); +} + } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/executer/kqp_executer_stats.h b/ydb/core/kqp/executer/kqp_executer_stats.h index 1317523dbd..56c7921318 100644 --- a/ydb/core/kqp/executer/kqp_executer_stats.h +++ b/ydb/core/kqp/executer/kqp_executer_stats.h @@ -49,47 +49,47 @@ struct TQueryExecutionStats { void Finish(); }; -struct TTableStat { - ui64 Rows = 0; - ui64 Bytes = 0; +struct TTableStat { + ui64 Rows = 0; + ui64 Bytes = 0; TTableStat& operator+=(const TTableStat& rhs); TTableStat& operator-=(const TTableStat& rhs); -}; - -struct TProgressStatEntry { +}; + +struct TProgressStatEntry { TDuration ComputeTime; TTableStat ReadIOStat; TProgressStatEntry& operator+=(const TProgressStatEntry& rhs); - void Out(IOutputStream& o) const; -}; - + void Out(IOutputStream& o) const; +}; + TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry); -TDuration CalcCumComputeTime(const TProgressStatEntry& entry); - -class TProgressStat { -public: - using TEntry = TProgressStatEntry; +TDuration CalcCumComputeTime(const TProgressStatEntry& entry); - TProgressStat() = default; +class TProgressStat { +public: + using TEntry = TProgressStatEntry; + + TProgressStat() = default; void Set(const NYql::NDqProto::TDqComputeActorStats& stats); - void Update(); + void Update(); + + TEntry GetLastUsage() const; - TEntry GetLastUsage() const; +private: + TEntry Total; + TEntry Cur; +}; -private: - TEntry Total; - TEntry Cur; -}; - } // namespace NKqp } // namespace NKikimr - -template<> -inline void Out<NKikimr::NKqp::TProgressStatEntry>(IOutputStream& o, const NKikimr::NKqp::TProgressStatEntry& x) { - return x.Out(o); -} + +template<> +inline void Out<NKikimr::NKqp::TProgressStatEntry>(IOutputStream& o, const NKikimr::NKqp::TProgressStatEntry& x) { + return x.Out(o); +} diff --git a/ydb/core/kqp/executer/kqp_literal_executer.cpp b/ydb/core/kqp/executer/kqp_literal_executer.cpp index 153d3120eb..3aabd41548 100644 --- a/ydb/core/kqp/executer/kqp_literal_executer.cpp +++ b/ydb/core/kqp/executer/kqp_literal_executer.cpp @@ -223,12 +223,12 @@ private: } if (auto* param = stageInfo.Meta.Tx.Params.Values.FindPtr(name)) { - NMiniKQL::TType* typeFromProto; - std::tie(typeFromProto, value) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory); + NMiniKQL::TType* typeFromProto; + std::tie(typeFromProto, value) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory); #ifndef NDEBUG YQL_ENSURE(ToString(*type) == ToString(*typeFromProto), "" << *type << " != " << *typeFromProto); #else - Y_UNUSED(typeFromProto); + Y_UNUSED(typeFromProto); #endif return true; } diff --git a/ydb/core/kqp/executer/kqp_partition_helper.cpp b/ydb/core/kqp/executer/kqp_partition_helper.cpp index 2b989501e7..ac786f6c24 100644 --- a/ydb/core/kqp/executer/kqp_partition_helper.cpp +++ b/ydb/core/kqp/executer/kqp_partition_helper.cpp @@ -33,7 +33,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKey(const NDq::TMkqlV THashMap<ui64, TShardParamValuesAndRanges> ret; THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues; - auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List); auto* itemType = static_cast<NMiniKQL::TListType*>(type)->GetItemType(); @@ -93,7 +93,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq:: THashMap<ui64, TShardParamValuesAndRanges> ret; THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues; - auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List); auto itemType = static_cast<NMiniKQL::TListType&>(*type).GetItemType(); @@ -205,7 +205,7 @@ TVector<TCell> FillKeyValues(const TVector<NUdf::TDataTypeId>& keyColumnTypes, c protoValue = &protoValue->GetTuple(*paramIndex); } - auto [type, value] = ImportValueFromProto(*protoType, *protoValue, typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(*protoType, *protoValue, typeEnv, holderFactory); keyValues.emplace_back(NMiniKQL::MakeCell(keyColumnTypes[i], value, typeEnv, /* copy */ true)); } @@ -340,7 +340,7 @@ TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDa const auto* protoType = ¶m->GetType(); const auto* protoValue = ¶m->GetValue(); - auto [type, value] = ImportValueFromProto(*protoType, *protoValue, typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(*protoType, *protoValue, typeEnv, holderFactory); // First element is Flow wrapping Ranges List YQL_ENSURE(value.IsBoxed()); @@ -631,7 +631,7 @@ THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRow paramDesc = &iter.first->second; paramDesc->MkqlValueRef = param; - std::tie(paramDesc->MkqlType, paramDesc->MkqlValue) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory); + std::tie(paramDesc->MkqlType, paramDesc->MkqlValue) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory); } mkqlType = paramDesc->MkqlType; @@ -643,7 +643,7 @@ THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRow case NKqpProto::TKqpPhyRowsList_TValue::kLiteralValue: { const auto& literal = columnValue.GetLiteralValue(); - std::tie(mkqlType, mkqlValue) = ImportValueFromProto(literal.GetType(), literal.GetValue(), typeEnv, holderFactory); + std::tie(mkqlType, mkqlValue) = ImportValueFromProto(literal.GetType(), literal.GetValue(), typeEnv, holderFactory); break; } diff --git a/ydb/core/kqp/executer/kqp_planner.cpp b/ydb/core/kqp/executer/kqp_planner.cpp index 5e938d8210..6bcfa85c66 100644 --- a/ydb/core/kqp/executer/kqp_planner.cpp +++ b/ydb/core/kqp/executer/kqp_planner.cpp @@ -31,13 +31,13 @@ TKqpPlanner::TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto:: , ScanTasks(std::move(scanTasks)) , Snapshot(snapshot) , Database(database) - , UserToken(userToken) + , UserToken(userToken) , Deadline(deadline) , StatsMode(statsMode) , DisableLlvmForUdfStages(disableLlvmForUdfStages) , EnableLlvm(enableLlvm) , WithSpilling(withSpilling) - , RlPath(rlPath) + , RlPath(rlPath) { if (!Database) { // a piece of magic for tests @@ -266,15 +266,15 @@ THolder<TEvKqpNode::TEvStartKqpTasksRequest> TKqpPlanner::PrepareKqpNodeRequest( ev->Record.MutableRuntimeSettings()->SetUseLLVM(withLLVM); ev->Record.MutableRuntimeSettings()->SetUseSpilling(WithSpilling); - if (RlPath) { - auto rlPath = ev->Record.MutableRuntimeSettings()->MutableRlPath(); - rlPath->SetCoordinationNode(RlPath->GetCoordinationNode()); - rlPath->SetResourcePath(RlPath->GetResourcePath()); - rlPath->SetDatabase(Database); - if (UserToken) - rlPath->SetToken(UserToken.GetRef()); - } - + if (RlPath) { + auto rlPath = ev->Record.MutableRuntimeSettings()->MutableRlPath(); + rlPath->SetCoordinationNode(RlPath->GetCoordinationNode()); + rlPath->SetResourcePath(RlPath->GetResourcePath()); + rlPath->SetDatabase(Database); + if (UserToken) + rlPath->SetToken(UserToken.GetRef()); + } + ev->Record.SetStartAllOrFail(true); return ev; diff --git a/ydb/core/kqp/executer/kqp_planner.h b/ydb/core/kqp/executer/kqp_planner.h index 50d7e4fa56..dc5e993a74 100644 --- a/ydb/core/kqp/executer/kqp_planner.h +++ b/ydb/core/kqp/executer/kqp_planner.h @@ -61,13 +61,13 @@ private: THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks; const IKqpGateway::TKqpSnapshot Snapshot; TString Database; - const TMaybe<TString> UserToken; + const TMaybe<TString> UserToken; const TInstant Deadline; const NYql::NDqProto::EDqStatsMode StatsMode; const bool DisableLlvmForUdfStages; const bool EnableLlvm; const bool WithSpilling; - const TMaybe<NKikimrKqp::TRlPath> RlPath; + const TMaybe<NKikimrKqp::TRlPath> RlPath; THashSet<ui32> TrackingNodes; }; diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp index f053ff04aa..07c29f64e2 100644 --- a/ydb/core/kqp/executer/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp @@ -44,8 +44,8 @@ public: TKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters) - : TBase(std::move(request), database, userToken, counters) - { + : TBase(std::move(request), database, userToken, counters) + { YQL_ENSURE(Request.Transactions.size() == 1); YQL_ENSURE(Request.Locks.empty()); YQL_ENSURE(!Request.ValidateLocks); @@ -178,8 +178,8 @@ private: LOG_D("Got execution state from compute actor: " << computeActor << ", task: " << taskId - << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()) - << ", stats: " << state.GetStats()); + << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()) + << ", stats: " << state.GetStats()); switch (state.GetState()) { case NDqProto::COMPUTE_STATE_UNKNOWN: { @@ -196,26 +196,26 @@ private: // initial TEvState event from Compute Actor // there can be race with RM answer if (PendingComputeTasks.erase(taskId)) { - auto it = PendingComputeActors.emplace(computeActor, TProgressStat()); + auto it = PendingComputeActors.emplace(computeActor, TProgressStat()); YQL_ENSURE(it.second); if (state.HasStats()) { it.first->second.Set(state.GetStats()); - } - + } + auto& task = TasksGraph.GetTask(taskId); task.ComputeActorId = computeActor; THashMap<TActorId, THashSet<ui64>> updates; CollectTaskChannelsUpdates(task, updates); PropagateChannelsUpdates(updates); - } else { - auto it = PendingComputeActors.find(computeActor); - if (it != PendingComputeActors.end()) { + } else { + auto it = PendingComputeActors.find(computeActor); + if (it != PendingComputeActors.end()) { if (state.HasStats()) { it->second.Set(state.GetStats()); - } - } + } + } } break; } @@ -225,8 +225,8 @@ private: Stats->AddComputeActorStats(computeActor.NodeId(), std::move(*state.MutableStats())); } - auto it = PendingComputeActors.find(computeActor); - if (it == PendingComputeActors.end()) { + auto it = PendingComputeActors.find(computeActor); + if (it == PendingComputeActors.end()) { LOG_W("Got execution state for compute actor: " << computeActor << ", task: " << taskId << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()) @@ -238,13 +238,13 @@ private: << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState())); return; } - } else { + } else { if (state.HasStats()) { it->second.Set(state.GetStats()); - } - LastStats.emplace_back(std::move(it->second)); - PendingComputeActors.erase(it); - YQL_ENSURE(PendingComputeTasks.find(taskId) == PendingComputeTasks.end()); + } + LastStats.emplace_back(std::move(it->second)); + PendingComputeActors.erase(it); + YQL_ENSURE(PendingComputeTasks.find(taskId) == PendingComputeTasks.end()); } } } @@ -328,7 +328,7 @@ private: if (PendingComputeTasks.erase(taskId) == 0) { LOG_D("Executing task: " << taskId << ", compute actor: " << task.ComputeActorId << ", already finished"); } else { - auto result = PendingComputeActors.emplace(std::make_pair(task.ComputeActorId, TProgressStat())); + auto result = PendingComputeActors.emplace(std::make_pair(task.ComputeActorId, TProgressStat())); YQL_ENSURE(result.second); CollectTaskChannelsUpdates(task, channelsUpdates); @@ -357,7 +357,7 @@ private: LOG_N("Disconnected node " << nodeId); for (auto computeActor : PendingComputeActors) { - if (computeActor.first.NodeId() == nodeId) { + if (computeActor.first.NodeId() == nodeId) { return ReplyUnavailable(TStringBuilder() << "Connection with node " << nodeId << " lost."); } } @@ -920,9 +920,9 @@ private: auto planner = CreateKqpPlanner(TxId, SelfId(), std::move(computeTasks), std::move(scanTasks), Request.Snapshot, - Database, UserToken, Deadline.GetOrElse(TInstant::Zero()), Request.StatsMode, + Database, UserToken, Deadline.GetOrElse(TInstant::Zero()), Request.StatsMode, Request.DisableLlvmForUdfStages, Request.LlvmEnabled, AppData()->EnableKqpSpilling, Request.RlPath); - RegisterWithSameMailbox(planner); + RegisterWithSameMailbox(planner); } void Finalize() { @@ -1008,12 +1008,12 @@ private: bool CheckExecutionComplete() { if (PendingComputeActors.empty() && PendingComputeTasks.empty()) { Finalize(); - UpdateResourcesUsage(true); + UpdateResourcesUsage(true); return true; } - UpdateResourcesUsage(false); - + UpdateResourcesUsage(false); + if (IsDebugLogEnabled()) { TStringBuilder sb; sb << "Waiting for: "; @@ -1021,7 +1021,7 @@ private: sb << "CT " << ct << ", "; } for (auto ca : PendingComputeActors) { - sb << "CA " << ca.first << ", "; + sb << "CA " << ca.first << ", "; } LOG_D(sb); } diff --git a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp index ec4e95c0e7..3148a2ec59 100644 --- a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp @@ -52,7 +52,7 @@ TKikimrParamsMap GetParamsMap(const NYdb::TParams& params) { auto paramValues = params.GetValues(); for (auto& pair : paramValues) { - Ydb::TypedValue protoParam; + Ydb::TypedValue protoParam; protoParam.mutable_type()->CopyFrom(NYdb::TProtoAccessor::GetProto(pair.second.GetType())); protoParam.mutable_value()->CopyFrom(NYdb::TProtoAccessor::GetProto(pair.second)); @@ -102,7 +102,7 @@ Y_UNIT_TEST_SUITE(KqpExecuter) { LogTxPlan(kikimr, tx); - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto params = db.GetParamsBuilder() .AddParam("$items") .BeginList() diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json index fc735c22ff..2faa33d5e1 100644 --- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json +++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json @@ -58,14 +58,14 @@ "Match": {"Type": "Callable", "Name": "KqlReadTable"} }, { - "Name": "TKqlReadTableIndex", - "Base": "TKqlReadTableBase", - "Match": {"Type": "Callable", "Name": "KqlReadTableIndex"}, - "Children": [ - {"Index": 4, "Name": "Index", "Type": "TCoAtom"} - ] - }, - { + "Name": "TKqlReadTableIndex", + "Base": "TKqlReadTableBase", + "Match": {"Type": "Callable", "Name": "KqlReadTableIndex"}, + "Children": [ + {"Index": 4, "Name": "Index", "Type": "TCoAtom"} + ] + }, + { "Name": "TKqpReadTable", "Base": "TKqlReadTableBase", "Match": {"Type": "Callable", "Name": "KqpReadTable"} @@ -136,14 +136,14 @@ "Match": {"Type": "Callable", "Name": "KqlLookupTable"} }, { - "Name": "TKqlLookupIndex", - "Base": "TKqlLookupTableBase", - "Match": {"Type": "Callable", "Name": "KqlLookupIndex"}, - "Children": [ - {"Index": 3, "Name": "Index", "Type": "TCoAtom"} - ] - }, - { + "Name": "TKqlLookupIndex", + "Base": "TKqlLookupTableBase", + "Match": {"Type": "Callable", "Name": "KqlLookupIndex"}, + "Children": [ + {"Index": 3, "Name": "Index", "Type": "TCoAtom"} + ] + }, + { "Name": "TKqpLookupTable", "Base": "TKqlLookupTableBase", "Match": {"Type": "Callable", "Name": "KqpLookupTable"} @@ -172,11 +172,11 @@ "Match": {"Type": "Callable", "Name": "KqlUpsertRows"} }, { - "Name": "TKqlUpsertRowsIndex", - "Base": "TKqlUpsertRowsBase", - "Match": {"Type": "Callable", "Name": "KqlUpsertRowsIndex"} - }, - { + "Name": "TKqlUpsertRowsIndex", + "Base": "TKqlUpsertRowsBase", + "Match": {"Type": "Callable", "Name": "KqlUpsertRowsIndex"} + }, + { "Name": "TKqpUpsertRows", "Base": "TKqlUpsertRowsBase", "Match": {"Type": "Callable", "Name": "KqpUpsertRows"}, @@ -185,9 +185,9 @@ ] }, { - "Name": "TKqlInsertRowsBase", + "Name": "TKqlInsertRowsBase", "Base": "TKqlTableEffect", - "Match": {"Type": "CallableBase"}, + "Match": {"Type": "CallableBase"}, "Children": [ {"Index": 1, "Name": "Input", "Type": "TExprBase"}, {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}, @@ -195,35 +195,35 @@ ] }, { - "Name": "TKqlUpdateRowsBase", + "Name": "TKqlUpdateRowsBase", "Base": "TKqlTableEffect", - "Match": {"Type": "CallableBase"}, + "Match": {"Type": "CallableBase"}, "Children": [ {"Index": 1, "Name": "Input", "Type": "TExprBase"}, {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"} ] }, { - "Name": "TKqlUpdateRows", - "Base": "TKqlUpdateRowsBase", - "Match": {"Type": "Callable", "Name": "TKqlUpdateRows"} - }, - { - "Name": "TKqlUpdateRowsIndex", - "Base": "TKqlUpdateRowsBase", - "Match": {"Type": "Callable", "Name": "TKqlUpdateRowsIndex"} - }, - { - "Name": "TKqlInsertRows", - "Base": "TKqlInsertRowsBase", - "Match": {"Type": "Callable", "Name": "TKqlInsertRows"} - }, - { - "Name": "TKqlInsertRowsIndex", - "Base": "TKqlInsertRowsBase", - "Match": {"Type": "Callable", "Name": "TKqlInsertRowsIndex"} - }, - { + "Name": "TKqlUpdateRows", + "Base": "TKqlUpdateRowsBase", + "Match": {"Type": "Callable", "Name": "TKqlUpdateRows"} + }, + { + "Name": "TKqlUpdateRowsIndex", + "Base": "TKqlUpdateRowsBase", + "Match": {"Type": "Callable", "Name": "TKqlUpdateRowsIndex"} + }, + { + "Name": "TKqlInsertRows", + "Base": "TKqlInsertRowsBase", + "Match": {"Type": "Callable", "Name": "TKqlInsertRows"} + }, + { + "Name": "TKqlInsertRowsIndex", + "Base": "TKqlInsertRowsBase", + "Match": {"Type": "Callable", "Name": "TKqlInsertRowsIndex"} + }, + { "Name": "TKqpParamBinding", "Base": "TExprBase", "Match": {"Type": "Tuple"}, @@ -349,11 +349,11 @@ "Match": {"Type": "Callable", "Name": "KqlDeleteRows"} }, { - "Name": "TKqlDeleteRowsIndex", - "Base": "TKqlDeleteRowsBase", - "Match": {"Type": "Callable", "Name": "KqlDeleteRowsIndex"} - }, - { + "Name": "TKqlDeleteRowsIndex", + "Base": "TKqlDeleteRowsBase", + "Match": {"Type": "Callable", "Name": "KqlDeleteRowsIndex"} + }, + { "Name": "TKqpDeleteRows", "Base": "TKqlDeleteRowsBase", "Match": {"Type": "Callable", "Name": "KqpDeleteRows"} diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 22f2c0bca6..44af6d1168 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -205,10 +205,10 @@ public: void FillResult(TResult& queryResult) const override { for (auto& resultStr : ResultProviderConfig.CommittedResults) { - queryResult.Results.emplace_back( + queryResult.Results.emplace_back( google::protobuf::Arena::CreateMessage<NKikimrMiniKQL::TResult>(queryResult.ProtobufArenaPtr.get())); - NKikimrMiniKQL::TResult* result = queryResult.Results.back(); - + NKikimrMiniKQL::TResult* result = queryResult.Results.back(); + if (!result->ParseFromArray(resultStr.data(), resultStr.size())) { queryResult = ResultFromError<TResult>("Failed to parse run result."); return; @@ -762,7 +762,7 @@ TIntrusivePtr<IKikimrAsyncResult<TResult>> CheckedProcess(TExprContext& ctx, TLa ? asyncResult : MakeKikimrResultHolder(ResultFromErrors<TResult>(ctx.IssueManager.GetIssues())); } - catch (const std::exception& e) { + catch (const std::exception& e) { return MakeKikimrResultHolder(ResultFromException<TResult>(e)); } } @@ -773,7 +773,7 @@ TResult CheckedSyncProcess(TLambda&& getResultFunc) { auto asyncResult = getResultFunc(); return SyncProcess(asyncResult); } - catch (const std::exception& e) { + catch (const std::exception& e) { return ResultFromException<TResult>(e); } } @@ -1083,7 +1083,7 @@ public: .AddTypeAnnotation() .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters") .AddPostTypeAnnotation() - .AddOptimization(true, false) + .AddOptimization(true, false) .Add(TLogExprTransformer::Sync("Optimized expr"), "LogExpr") .AddRun(&NullProgressWriter) .Build(); @@ -1879,10 +1879,10 @@ private: SetupExecutePreparedTransformer(settings, tx); - if (issues) { - ctx.IssueManager.AddIssues(issues); - } - + if (issues) { + ctx.IssueManager.AddIssues(issues); + } + if (!ParseParameters(std::move(parameters), SessionCtx->Query().Parameters, ctx)) { return nullptr; } @@ -2120,7 +2120,7 @@ private: SessionCtx->Query().Deadlines = settings.Deadlines; SessionCtx->Query().Limits = settings.Limits; SessionCtx->Query().StatsMode = settings.StatsMode; - SessionCtx->Query().RlPath = settings.RlPath; + SessionCtx->Query().RlPath = settings.RlPath; return MakeIntrusive<TAsyncExecutePreparedResult>(FakeWorld.Get(), ctx, *ExecutePreparedTransformer, SessionCtx, FillSettings, ExecuteCtx, nullptr, sqlVersion); @@ -2128,7 +2128,7 @@ private: void SetupSession(TIntrusivePtr<IKikimrTransactionContext> txCtx) { ExprCtx->Reset(); - ExprCtx->Step.Done(TExprStep::ExprEval); // KIKIMR-8067 + ExprCtx->Step.Done(TExprStep::ExprEval); // KIKIMR-8067 TypesCtx->DeprecatedSQL = false; TypesCtx->CachedNow.reset(); diff --git a/ydb/core/kqp/host/kqp_run_prepared.cpp b/ydb/core/kqp/host/kqp_run_prepared.cpp index 16c33eaf0a..e71a3bacee 100644 --- a/ydb/core/kqp/host/kqp_run_prepared.cpp +++ b/ydb/core/kqp/host/kqp_run_prepared.cpp @@ -21,8 +21,8 @@ public: , Cluster(cluster) , TxState(txState) , TransformCtx(transformCtx) - , CurrentMkqlIndex(0) - {} + , CurrentMkqlIndex(0) + {} TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { output = input; @@ -30,7 +30,7 @@ public: const auto& kql = TransformCtx->GetPreparedKql(); if (CurrentMkqlIndex >= kql.MkqlsSize()) { - return Finish(ctx); + return Finish(ctx); } const auto& mkql = kql.GetMkqls(CurrentMkqlIndex); @@ -92,29 +92,29 @@ public: } private: - TStatus Finish(TExprContext& ctx) { - const auto& kql = TransformCtx->GetPreparedKql(); - if (!kql.GetEffects().empty()) { - YQL_ENSURE(kql.GetEffects().size() == 1); - - auto& effect = kql.GetEffects(0); - - TExprNode::TPtr expr; - if (!GetExpr(effect.GetNodeAst(), expr, ctx)) { - return TStatus::Error; - } - - TVector<NKikimrKqp::TParameterBinding> bindings(effect.GetBindings().begin(), - effect.GetBindings().end()); - - bool preserveParams = !TransformCtx->Settings.GetCommitTx(); - if (!AddDeferredEffect(TExprBase(expr), bindings, ctx, *TxState, *TransformCtx, preserveParams)) { - return TStatus::Error; - } - } - return TStatus::Ok; - } - + TStatus Finish(TExprContext& ctx) { + const auto& kql = TransformCtx->GetPreparedKql(); + if (!kql.GetEffects().empty()) { + YQL_ENSURE(kql.GetEffects().size() == 1); + + auto& effect = kql.GetEffects(0); + + TExprNode::TPtr expr; + if (!GetExpr(effect.GetNodeAst(), expr, ctx)) { + return TStatus::Error; + } + + TVector<NKikimrKqp::TParameterBinding> bindings(effect.GetBindings().begin(), + effect.GetBindings().end()); + + bool preserveParams = !TransformCtx->Settings.GetCommitTx(); + if (!AddDeferredEffect(TExprBase(expr), bindings, ctx, *TxState, *TransformCtx, preserveParams)) { + return TStatus::Error; + } + } + return TStatus::Ok; + } + bool Execute(const NKikimrKqp::TPreparedMkql& mkql, TFuture<IKqpGateway::TMkqlResult>& future) { if (YQL_CLOG_ACTIVE(DEBUG, ProviderKqp)) { TString mkqlText; diff --git a/ydb/core/kqp/host/kqp_run_scan.cpp b/ydb/core/kqp/host/kqp_run_scan.cpp index c6845eb767..7c91ae8200 100644 --- a/ydb/core/kqp/host/kqp_run_scan.cpp +++ b/ydb/core/kqp/host/kqp_run_scan.cpp @@ -24,8 +24,8 @@ protected: IKqpGateway::TExecPhysicalRequest request; request.Transactions.emplace_back(*tx, PrepareParameters(*tx)); - - request.RlPath = TransformCtx->QueryCtx->RlPath; + + request.RlPath = TransformCtx->QueryCtx->RlPath; request.Timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime(); if (!request.Timeout) { // TODO: Just cancel request. diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index 01be6f891d..42b4eca7c6 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -179,7 +179,7 @@ public: .Add(CreateKqpExecTransformer(Gateway, Cluster, TxState, TransformCtx), "Prepare") .Add(CreateKqpSubstituteTransformer(TxState, TransformCtx), "Substitute") .Add(CreateKqpFinalizeTransformer(Gateway, Cluster, TxState, TransformCtx), "Finalize") - .Build(false); + .Build(false); PreparedRunTransformer = TTransformationPipeline(typesCtx) .Add(TLogExprTransformer::Sync("PreparedRun iteration", logComp, logLevel), "KqlPreparedRun") @@ -470,20 +470,20 @@ private: TransformCtx->QueryCtx->PreparingQuery->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1); auto kql = TransformCtx->QueryCtx->PreparingQuery->AddKqls(); kql->MutableSettings()->CopyFrom(TransformCtx->Settings); - + FillAstAndPlan(*kql, finalProgram, ctx); auto operations = TableOperationsToProto(dataQuery.Operations(), ctx); for (auto& op : operations) { const auto& tableName = op.GetTable(); - + kql->AddOperations()->CopyFrom(op); const auto& desc = TransformCtx->Tables->GetTable(cluster, tableName); TableDescriptionToTableInfo(desc, kql->AddTableInfo()); } TransformCtx->PreparingKql = kql; - + return MakeIntrusive<TAsyncRunResult>(finalProgram, ctx, *KqlPrepareTransformer, *TransformCtx); } diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h index 2b1f176735..09f794591d 100644 --- a/ydb/core/kqp/kqp.h +++ b/ydb/core/kqp/kqp.h @@ -251,83 +251,83 @@ struct TEvKqp { struct TEvDataQueryStreamPartAck : public TEventLocal<TEvDataQueryStreamPartAck, TKqpEvents::EvDataQueryStreamPartAck> {}; - // Wrapper to use Arena allocated protobuf with ActorSystem (for serialization path). - // Arena deserialization is not supported. - // TODO: Add arena support to actor system TEventPB? - template<typename TProto> - class TProtoArenaHolder : public TNonCopyable { - public: - TProtoArenaHolder() - : Protobuf_(google::protobuf::Arena::CreateMessage<TProto>(nullptr)) - {} - - ~TProtoArenaHolder() { - // Deallocate message only if it was "normal" allocation - // In case of protobuf arena memory will be freed during arena deallocation - if (!Protobuf_->GetArena()) { - delete Protobuf_; - } - } - - void Realloc(std::shared_ptr<google::protobuf::Arena> arena) { - // Allow realloc only if previous allocation was made using "normal" allocator - // and no data was writen. It prevents ineffective using of protobuf. - Y_ASSERT(!Protobuf_->GetArena()); - Y_ASSERT(ByteSize() == 0); - delete Protobuf_; - Protobuf_ = google::protobuf::Arena::CreateMessage<TProto>(arena.get()); - // Make sure arena is alive - Arena_ = arena; - } - - bool ParseFromString(const TString& data) { - return Protobuf_->ParseFromString(data); - } - + // Wrapper to use Arena allocated protobuf with ActorSystem (for serialization path). + // Arena deserialization is not supported. + // TODO: Add arena support to actor system TEventPB? + template<typename TProto> + class TProtoArenaHolder : public TNonCopyable { + public: + TProtoArenaHolder() + : Protobuf_(google::protobuf::Arena::CreateMessage<TProto>(nullptr)) + {} + + ~TProtoArenaHolder() { + // Deallocate message only if it was "normal" allocation + // In case of protobuf arena memory will be freed during arena deallocation + if (!Protobuf_->GetArena()) { + delete Protobuf_; + } + } + + void Realloc(std::shared_ptr<google::protobuf::Arena> arena) { + // Allow realloc only if previous allocation was made using "normal" allocator + // and no data was writen. It prevents ineffective using of protobuf. + Y_ASSERT(!Protobuf_->GetArena()); + Y_ASSERT(ByteSize() == 0); + delete Protobuf_; + Protobuf_ = google::protobuf::Arena::CreateMessage<TProto>(arena.get()); + // Make sure arena is alive + Arena_ = arena; + } + + bool ParseFromString(const TString& data) { + return Protobuf_->ParseFromString(data); + } + bool ParseFromZeroCopyStream(google::protobuf::io::ZeroCopyInputStream* input) { return Protobuf_->ParseFromZeroCopyStream(input); } - bool SerializeToZeroCopyStream(google::protobuf::io::ZeroCopyOutputStream* output) const { - return Protobuf_->SerializeToZeroCopyStream(output); - } - - bool SerializeToString(TString* output) const { - return Protobuf_->SerializeToString(output); - } - - int ByteSize() const { - return Protobuf_->ByteSize(); - } - - TString DebugString() const { - return Protobuf_->DebugString(); - } - + bool SerializeToZeroCopyStream(google::protobuf::io::ZeroCopyOutputStream* output) const { + return Protobuf_->SerializeToZeroCopyStream(output); + } + + bool SerializeToString(TString* output) const { + return Protobuf_->SerializeToString(output); + } + + int ByteSize() const { + return Protobuf_->ByteSize(); + } + + TString DebugString() const { + return Protobuf_->DebugString(); + } + TString ShortDebugString() const { return Protobuf_->ShortDebugString(); } - TString GetTypeName() const { - return Protobuf_->GetTypeName(); - } - - const TProto& GetRef() const { - return *Protobuf_; - } - - TProto& GetRef() { - return *Protobuf_; - } - - private: - TProtoArenaHolder(TProtoArenaHolder&&) = default; - TProtoArenaHolder& operator=(TProtoArenaHolder&&) = default; - TProto* Protobuf_; - std::shared_ptr<google::protobuf::Arena> Arena_; - }; - - struct TEvQueryResponse : public TEventPB<TEvQueryResponse, TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>, + TString GetTypeName() const { + return Protobuf_->GetTypeName(); + } + + const TProto& GetRef() const { + return *Protobuf_; + } + + TProto& GetRef() { + return *Protobuf_; + } + + private: + TProtoArenaHolder(TProtoArenaHolder&&) = default; + TProtoArenaHolder& operator=(TProtoArenaHolder&&) = default; + TProto* Protobuf_; + std::shared_ptr<google::protobuf::Arena> Arena_; + }; + + struct TEvQueryResponse : public TEventPB<TEvQueryResponse, TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>, TKqpEvents::EvQueryResponse> {}; struct TEvCreateSessionResponse : public TEventPB<TEvCreateSessionResponse, diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp index 45158d7c8e..27cf296a03 100644 --- a/ydb/core/kqp/kqp_compile_actor.cpp +++ b/ydb/core/kqp/kqp_compile_actor.cpp @@ -18,7 +18,7 @@ #include <util/string/escape.h> #include <ydb/core/base/cputime.h> - + namespace NKikimr { namespace NKqp { @@ -102,13 +102,13 @@ public: IKqpHost::TPrepareSettings prepareSettings; // prepareSettings.UseNewEngine = use default settings prepareSettings.DocumentApiRestricted = Query.Settings.DocumentApiRestricted; - - NCpuTime::TCpuTimer timer(CompileCpuTime); - + + NCpuTime::TCpuTimer timer(CompileCpuTime); + AsyncCompileResult = KqpHost->PrepareDataQuery(Query.Text, prepareSettings); Continue(ctx); - + Become(&TKqpCompileActor::CompileState); } @@ -208,7 +208,7 @@ private: auto& stats = responseEv->Stats; stats.SetFromCache(false); stats.SetDurationUs((TInstant::Now() - StartTime).MicroSeconds()); - stats.SetCpuTimeUs(CompileCpuTime.MicroSeconds()); + stats.SetCpuTimeUs(CompileCpuTime.MicroSeconds()); ctx.Send(Owner, responseEv.Release()); Counters->ReportCompileFinish(DbCounters); @@ -243,7 +243,7 @@ private: TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); if (!ev->Get()->Finished) { - NCpuTime::TCpuTimer timer(CompileCpuTime); + NCpuTime::TCpuTimer timer(CompileCpuTime); Continue(ctx); return; } @@ -380,7 +380,7 @@ private: TDuration CompilationTimeout; bool RecompileWithNewEngine; TInstant StartTime; - TDuration CompileCpuTime; + TDuration CompileCpuTime; TInstant RecompileStartTime; TActorId TimeoutTimerActorId; TIntrusivePtr<IKqpGateway> Gateway; diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index e0caa17f6b..47489a9bd5 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -48,18 +48,18 @@ using NKikimrTxUserProxy::TMiniKQLTransaction; constexpr const IKqpGateway::TKqpSnapshot IKqpGateway::TKqpSnapshot::InvalidSnapshot = TKqpSnapshot(); -#define STATIC_ASSERT_STATE_EQUAL(name) \ - static_assert(static_cast<ui32>(NYql::TIndexDescription::EIndexState::name) \ +#define STATIC_ASSERT_STATE_EQUAL(name) \ + static_assert(static_cast<ui32>(NYql::TIndexDescription::EIndexState::name) \ == NKikimrSchemeOp::EIndexState::EIndexState##name, \ - "index state missmatch, flag: ## name"); - -STATIC_ASSERT_STATE_EQUAL(Invalid) -STATIC_ASSERT_STATE_EQUAL(Ready) -STATIC_ASSERT_STATE_EQUAL(NotReady) -STATIC_ASSERT_STATE_EQUAL(WriteOnly) - -#undef STATIC_ASSERT_STATE_EQUAL - + "index state missmatch, flag: ## name"); + +STATIC_ASSERT_STATE_EQUAL(Invalid) +STATIC_ASSERT_STATE_EQUAL(Ready) +STATIC_ASSERT_STATE_EQUAL(NotReady) +STATIC_ASSERT_STATE_EQUAL(WriteOnly) + +#undef STATIC_ASSERT_STATE_EQUAL + namespace { template <class TResult> @@ -536,7 +536,7 @@ public: if (!result.Errors.Empty()) { Promise.SetValue(ResultFromIssues<TResult>(GetMkqlCompileStatus(result.Errors), - "Query compilation failed", result.Errors)); + "Query compilation failed", result.Errors)); this->Die(ctx); return; @@ -685,7 +685,7 @@ public: auto& response = ev->Get()->Record; auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(response.GetStatus()); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvProposeTransactionStatus for scheme request" + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvProposeTransactionStatus for scheme request" << ", TxId: " << response.GetTxId() << ", status: " << status << ", scheme shard status: " << response.GetSchemeShardStatus()); @@ -701,14 +701,14 @@ public: request->Record.SetTxId(response.GetTxId()); NTabletPipe::SendData(ctx, ShemePipeActorId, request.Release()); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Sent TEvNotifyTxCompletion request" + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Sent TEvNotifyTxCompletion request" << ", TxId: " << response.GetTxId()); return; } case TEvTxUserProxy::TResultStatus::AccessDenied: { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Access denied for scheme request" + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Access denied for scheme request" << ", TxId: " << response.GetTxId()); TIssue issue(NYql::TPosition(), "Access denied."); @@ -722,7 +722,7 @@ public: if (response.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusSuccess || response.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusAlreadyExists) { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request" + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request" << ", TxId: " << response.GetTxId()); IKqpGateway::TGenericResult result; @@ -757,11 +757,11 @@ public: return; } case NKikimrScheme::EStatus::StatusPathDoesNotExist: { - Promise.SetValue(ResultFromIssues<TResult>(TIssuesIds::KIKIMR_SCHEME_ERROR, - response.GetSchemeShardReason(), {})); - this->Die(ctx); - return; - } + Promise.SetValue(ResultFromIssues<TResult>(TIssuesIds::KIKIMR_SCHEME_ERROR, + response.GetSchemeShardReason(), {})); + this->Die(ctx); + return; + } default: break; @@ -980,14 +980,14 @@ private: : Serialized(serialized) , Data(serialized) {} }; - using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TNavigate = NSchemeCache::TSchemeCacheNavigate; public: TKikimrIcGateway(const TString& cluster, const TString& database, std::shared_ptr<IKqpTableMetadataLoader>&& metadataLoader, TActorSystem* actorSystem, ui32 nodeId, TKqpRequestCounters::TPtr counters, const TActorId& mkqlComplileService) : Cluster(cluster) , Database(database) - , ActorSystem(actorSystem) + , ActorSystem(actorSystem) , NodeId(nodeId) , Counters(counters) , MetadataLoader(std::move(metadataLoader)) @@ -1028,10 +1028,10 @@ public: return MetadataLoader->GetCollectedSchemeData(); } - TString GetTokenCompat() const { - return UserToken ? UserToken->Serialized : TString(); - } - + TString GetTokenCompat() const { + return UserToken ? UserToken->Serialized : TString(); + } + TFuture<TListPathResult> ListPath(const TString& cluster, const TString &path) override { using TRequest = TEvTxUserProxy::TEvNavigate; @@ -1064,26 +1064,26 @@ public: } } - static ui64 GetExpectedVersion(const std::pair<TIndexId, TString>& pathId) { - return pathId.first.SchemaVersion; - } - - static ui64 GetExpectedVersion(const TString&) { - return 0; - } - + static ui64 GetExpectedVersion(const std::pair<TIndexId, TString>& pathId) { + return pathId.first.SchemaVersion; + } + + static ui64 GetExpectedVersion(const TString&) { + return 0; + } + TFuture<TTableMetadataResult> LoadTableMetadata(const TString& cluster, const TString& table, TLoadTableMetadataSettings settings) override { try { - if (!CheckCluster(cluster)) { + if (!CheckCluster(cluster)) { return InvalidCluster<TTableMetadataResult>(cluster); - } + } if (UserToken) { return MetadataLoader->LoadTableMetadata(cluster, table, settings, Database, UserToken->Data); - } else { + } else { return MetadataLoader->LoadTableMetadata(cluster, table, settings, Database, Nothing()); - } - + } + } catch (yexception& e) { return MakeFuture(ResultFromException<TTableMetadataResult>(e)); } @@ -1112,7 +1112,7 @@ public: using TConfigRequest = NConsole::TEvConfigsDispatcher::TEvGetConfigRequest; using TConfigResponse = NConsole::TEvConfigsDispatcher::TEvGetConfigResponse; - + ui32 configKind = (ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem; auto ev = MakeHolder<TConfigRequest>(configKind); @@ -1148,12 +1148,12 @@ public: auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); schemeTx.SetWorkingDir(pathPair.first); NKikimrSchemeOp::TTableDescription* tableDesc = nullptr; - if (!metadata->Indexes.empty()) { + if (!metadata->Indexes.empty()) { schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateIndexedTable); - tableDesc = schemeTx.MutableCreateIndexedTable()->MutableTableDescription(); - for (const auto& index : metadata->Indexes) { - auto indexDesc = schemeTx.MutableCreateIndexedTable()->AddIndexDescription(); - indexDesc->SetName(index.Name); + tableDesc = schemeTx.MutableCreateIndexedTable()->MutableTableDescription(); + for (const auto& index : metadata->Indexes) { + auto indexDesc = schemeTx.MutableCreateIndexedTable()->AddIndexDescription(); + indexDesc->SetName(index.Name); switch (index.Type) { case NYql::TIndexDescription::EType::GlobalSync: indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobal); @@ -1164,20 +1164,20 @@ public: } indexDesc->SetState(static_cast<::NKikimrSchemeOp::EIndexState>(index.State)); - for (const auto& col : index.KeyColumns) { - indexDesc->AddKeyColumnNames(col); - } - for (const auto& col : index.DataColumns) { - indexDesc->AddDataColumnNames(col); - } - } + for (const auto& col : index.KeyColumns) { + indexDesc->AddKeyColumnNames(col); + } + for (const auto& col : index.DataColumns) { + indexDesc->AddDataColumnNames(col); + } + } FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); - } else { + } else { schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateTable); - tableDesc = schemeTx.MutableCreateTable(); + tableDesc = schemeTx.MutableCreateTable(); FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); - } - + } + Ydb::StatusIds::StatusCode code; TString error; TList<TString> warnings; @@ -1189,7 +1189,7 @@ public: tablePromise.SetValue(errResult); return; } - + SendSchemeRequest(ev.Release()).Apply( [tablePromise, warnings{std::move(warnings)}](const TFuture<TGenericResult>& future) mutable { if (warnings.size()) { @@ -1215,22 +1215,22 @@ public: } TFuture<TGenericResult> AlterTable(Ydb::Table::AlterTableRequest&& req, const TString& cluster) override { - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } // FIXME: should be defined in grpc_services/rpc_calls.h, but cause cyclic dependency using namespace NGRpcService; using TEvAlterTableRequest = TGRpcRequestValidationWrapper<TRpcServices::EvAlterTable, Ydb::Table::AlterTableRequest, Ydb::Table::AlterTableResponse, true, TRateLimiterMode::Rps>; return SendLocalRpcRequestNoResult<TEvAlterTableRequest>(std::move(req), Database, GetTokenCompat()); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + TFuture<TGenericResult> RenameTable(const TString& src, const TString& dst, const TString& cluster) override { using TRequest = TEvTxUserProxy::TEvProposeTransaction; @@ -1271,8 +1271,8 @@ public: return InvalidCluster<TGenericResult>(cluster); } - Ydb::Table::DropTableRequest dropTable; - dropTable.set_path(table); + Ydb::Table::DropTableRequest dropTable; + dropTable.set_path(table); // FIXME: should be defined in grpc_services/rpc_calls.h, but cause cyclic dependency using namespace NGRpcService; @@ -2007,30 +2007,30 @@ private: return promise.GetFuture(); } - template<typename TRpc> - TFuture<TGenericResult> SendLocalRpcRequestNoResult(typename TRpc::TRequest&& proto, const TString& databse, const TString& token) { - return NRpcService::DoLocalRpc<TRpc>(std::move(proto), databse, token, ActorSystem).Apply([](NThreading::TFuture<typename TRpc::TResponse> future) { - auto r = future.ExtractValue(); - NYql::TIssues issues; - NYql::IssuesFromMessage(r.operation().issues(), issues); - - if (r.operation().ready() != true) { - issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder() - << "Unexpected operation for \"sync\" mode")); - - const auto& yqlStatus = TIssuesIds::DEFAULT_ERROR; - - auto result = ResultFromIssues<TGenericResult>(yqlStatus, issues); - return NThreading::MakeFuture(result); - } - - const auto& yqlStatus = NYql::YqlStatusFromYdbStatus(r.operation().status()); - - auto result = ResultFromIssues<TGenericResult>(yqlStatus, issues); - return NThreading::MakeFuture(result); - }); - } - + template<typename TRpc> + TFuture<TGenericResult> SendLocalRpcRequestNoResult(typename TRpc::TRequest&& proto, const TString& databse, const TString& token) { + return NRpcService::DoLocalRpc<TRpc>(std::move(proto), databse, token, ActorSystem).Apply([](NThreading::TFuture<typename TRpc::TResponse> future) { + auto r = future.ExtractValue(); + NYql::TIssues issues; + NYql::IssuesFromMessage(r.operation().issues(), issues); + + if (r.operation().ready() != true) { + issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder() + << "Unexpected operation for \"sync\" mode")); + + const auto& yqlStatus = TIssuesIds::DEFAULT_ERROR; + + auto result = ResultFromIssues<TGenericResult>(yqlStatus, issues); + return NThreading::MakeFuture(result); + } + + const auto& yqlStatus = NYql::YqlStatusFromYdbStatus(r.operation().status()); + + auto result = ResultFromIssues<TGenericResult>(yqlStatus, issues); + return NThreading::MakeFuture(result); + }); + } + bool CheckCluster(const TString& cluster) { return cluster == Cluster; } @@ -2349,9 +2349,9 @@ private: case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::CoordinatorUnknown: status = TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN; break; - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAborted: - status = TIssuesIds::KIKIMR_OPERATION_ABORTED; - break; + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAborted: + status = TIssuesIds::KIKIMR_OPERATION_ABORTED; + break; default: break; } @@ -2589,7 +2589,7 @@ private: } if (metadata->TableSettings.ReadReplicasSettings) { - if (!NYql::ConvertReadReplicasSettingsToProto(metadata->TableSettings.ReadReplicasSettings.GetRef(), + if (!NYql::ConvertReadReplicasSettingsToProto(metadata->TableSettings.ReadReplicasSettings.GetRef(), *proto.mutable_read_replicas_settings(), code, error)) { return false; } diff --git a/ydb/core/kqp/kqp_impl.h b/ydb/core/kqp/kqp_impl.h index 3673fdd696..7d6f172899 100644 --- a/ydb/core/kqp/kqp_impl.h +++ b/ydb/core/kqp/kqp_impl.h @@ -10,7 +10,7 @@ #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/protobuf/util/pb_io.h> - + namespace NKikimr { namespace NKqp { @@ -88,7 +88,7 @@ TIntrusivePtr<IKqpGateway> CreateKikimrIcGateway(const TString& cluster, const T Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& queryResult); void AddQueryIssues(NKikimrKqp::TQueryResponse& response, const NYql::TIssues& issues); -bool HasSchemeOrFatalIssues(const NYql::TIssues& issues); +bool HasSchemeOrFatalIssues(const NYql::TIssues& issues); // for tests only void FailForcedNewEngineCompilationForTests(bool fail = true); diff --git a/ydb/core/kqp/kqp_response.cpp b/ydb/core/kqp/kqp_response.cpp index ac964cbeb1..d3859cd39c 100644 --- a/ydb/core/kqp/kqp_response.cpp +++ b/ydb/core/kqp/kqp_response.cpp @@ -46,9 +46,9 @@ TMaybe<Ydb::StatusIds::StatusCode> GetYdbStatus(const TIssue& issue) { case TIssuesIds::KIKIMR_TOO_MANY_TRANSACTIONS: return Ydb::StatusIds::BAD_SESSION; - case TIssuesIds::KIKIMR_ACCESS_DENIED: - return Ydb::StatusIds::UNAUTHORIZED; - + case TIssuesIds::KIKIMR_ACCESS_DENIED: + return Ydb::StatusIds::UNAUTHORIZED; + case TIssuesIds::KIKIMR_TIMEOUT: return Ydb::StatusIds::TIMEOUT; @@ -92,11 +92,11 @@ void CollectYdbStatuses(const TIssue& issue, TSet<Ydb::StatusIds::StatusCode>& s } } -bool HasSchemeOrFatalIssues(const TIssue& issue) { - if (issue.GetSeverity() == TSeverityIds::S_FATAL) { - return true; - } - +bool HasSchemeOrFatalIssues(const TIssue& issue) { + if (issue.GetSeverity() == TSeverityIds::S_FATAL) { + return true; + } + switch (issue.GetCode()) { case TIssuesIds::KIKIMR_SCHEME_MISMATCH: case TIssuesIds::KIKIMR_SCHEME_ERROR: @@ -104,7 +104,7 @@ bool HasSchemeOrFatalIssues(const TIssue& issue) { } for (auto& subIssue : issue.GetSubIssues()) { - if (HasSchemeOrFatalIssues(*subIssue)) { + if (HasSchemeOrFatalIssues(*subIssue)) { return true; } } @@ -125,9 +125,9 @@ Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& q } if (statuses.contains(Ydb::StatusIds::UNAUTHORIZED)) { - return Ydb::StatusIds::UNAUTHORIZED; - } - + return Ydb::StatusIds::UNAUTHORIZED; + } + if (statuses.contains(Ydb::StatusIds::INTERNAL_ERROR)) { return Ydb::StatusIds::INTERNAL_ERROR; } @@ -149,9 +149,9 @@ void AddQueryIssues(NKikimrKqp::TQueryResponse& response, const TIssues& issues) } } -bool HasSchemeOrFatalIssues(const TIssues& issues) { +bool HasSchemeOrFatalIssues(const TIssues& issues) { for (auto& issue : issues) { - if (HasSchemeOrFatalIssues(issue)) { + if (HasSchemeOrFatalIssues(issue)) { return true; } } diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index e330b4c01c..cccb8a51e8 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -32,7 +32,7 @@ using namespace NKikimrConfig; using namespace NThreading; using namespace NYql; using namespace NYql::NDq; -using namespace NRuCalc; +using namespace NRuCalc; static std::atomic<bool> FailForcedNewEngineExecution = false; void FailForcedNewEngineExecutionForTests(bool fail) { @@ -58,7 +58,7 @@ struct TKqpQueryState { TString UserToken; TActorId RequestActorId; TInstant StartTime; - TDuration CpuTime; + TDuration CpuTime; NYql::TKikimrQueryDeadlines QueryDeadlines; TString TxId; TKqpCompileResult::TConstPtr QueryCompileResult; @@ -73,7 +73,7 @@ struct TKqpQueryState { TKqpForceNewEngineState ForceNewEngineState; std::optional<TQueryTraits> QueryTraits; - TMaybe<NKikimrKqp::TRlPath> RlPath; + TMaybe<NKikimrKqp::TRlPath> RlPath; }; @@ -287,11 +287,11 @@ public: QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST; } - if (event.HasRlPath()) { - QueryState->RlPath = event.GetRlPath(); - } - - NCpuTime::TCpuTimer timer; + if (event.HasRlPath()) { + QueryState->RlPath = event.GetRlPath(); + } + + NCpuTime::TCpuTimer timer; if (queryRequest.GetCancelAfterMs()) { QueryState->QueryDeadlines.CancelAt = now + TDuration::MilliSeconds(queryRequest.GetCancelAfterMs()); @@ -414,14 +414,14 @@ public: if (CompileQuery(ctx)) { if (QueryState) { - QueryState->CpuTime += timer.GetTime(); + QueryState->CpuTime += timer.GetTime(); } return; } PerformQuery(ctx); if (QueryState) { - QueryState->CpuTime += timer.GetTime(); + QueryState->CpuTime += timer.GetTime(); } } @@ -506,12 +506,12 @@ public: break; } - NCpuTime::TCpuTimer timer; + NCpuTime::TCpuTimer timer; PerformQuery(ctx); - // PerformQuery can reset QueryState + // PerformQuery can reset QueryState if (QueryState) { - QueryState->CpuTime += timer.GetTime(); + QueryState->CpuTime += timer.GetTime(); } } @@ -610,7 +610,7 @@ public: QueryCleanup(ctx); } else { - NCpuTime::TCpuTimer timer(QueryState->CpuTime); + NCpuTime::TCpuTimer timer(QueryState->CpuTime); ContinueQueryProcess(ctx); } } @@ -739,7 +739,7 @@ public: HFunc(TEvKqp::TEvQueryRequest, HandlePerformQuery); HFunc(TEvKqp::TEvCompileResponse, HandlePerformQuery); HFunc(TEvKqp::TEvCloseSessionRequest, HandlePerformQuery); - HFunc(TEvKqp::TEvPingSessionRequest, HandlePerformQuery); + HFunc(TEvKqp::TEvPingSessionRequest, HandlePerformQuery); HFunc(TEvKqp::TEvContinueProcess, HandlePerformQuery); HFunc(TEvKqp::TEvIdleTimeout, HandlePerformQuery); HFunc(TEvKqp::TEvInitiateSessionShutdown, HandleInitiateShutdown); @@ -754,7 +754,7 @@ public: HFunc(TEvKqp::TEvQueryRequest, HandlePerformCleanup); HFunc(TEvKqp::TEvCompileResponse, HandlePerformCleanup); HFunc(TEvKqp::TEvCloseSessionRequest, HandlePerformCleanup); - HFunc(TEvKqp::TEvPingSessionRequest, HandlePerformCleanup); + HFunc(TEvKqp::TEvPingSessionRequest, HandlePerformCleanup); HFunc(TEvKqp::TEvContinueProcess, HandlePerformCleanup); HFunc(TEvKqp::TEvIdleTimeout, HandlePerformCleanup); HFunc(TEvKqp::TEvInitiateSessionShutdown, HandleInitiateShutdown); @@ -982,7 +982,7 @@ private: case NKikimrKqp::QUERY_ACTION_EXPLAIN: { // Force reply flags QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_PLAN | NKikimrKqp::QUERY_REPLY_FLAG_AST; - if (!ExplainQuery(ctx, queryRequest.GetQuery(), queryType)) { + if (!ExplainQuery(ctx, queryRequest.GetQuery(), queryType)) { onBadRequest(QueryState->Error); return; } @@ -995,7 +995,7 @@ private: } case NKikimrKqp::QUERY_ACTION_VALIDATE: { - if (!ValidateQuery(ctx, queryRequest.GetQuery(), queryType)) { + if (!ValidateQuery(ctx, queryRequest.GetQuery(), queryType)) { onBadRequest(QueryState->Error); return; } @@ -1326,8 +1326,8 @@ private: execSettings.StatsMode = statsMode; execSettings.Deadlines = QueryState->QueryDeadlines; execSettings.Limits = GetQueryLimits(Settings); - execSettings.RlPath = QueryState->RlPath; - + execSettings.RlPath = QueryState->RlPath; + QueryState->AsyncQueryResult = KqpHost->ExecuteScanQuery(query, isSql, std::move(*parameters), requestActorId, execSettings); break; @@ -1554,10 +1554,10 @@ private: bool ReplyPrepareResult(const TKqpCompileResult::TConstPtr& compileResult, const TActorContext &ctx) { auto responseEv = MakeHolder<TEvKqp::TEvQueryResponse>(); FillCompileStatus(compileResult, responseEv->Record); - - auto ru = CpuTimeToUnit(TDuration::MicroSeconds(QueryState->CompileStats.GetCpuTimeUs())); - responseEv->Record.GetRef().SetConsumedRu(ru); - + + auto ru = CpuTimeToUnit(TDuration::MicroSeconds(QueryState->CompileStats.GetCpuTimeUs())); + responseEv->Record.GetRef().SetConsumedRu(ru); + return Reply(std::move(responseEv), ctx); } @@ -1568,8 +1568,8 @@ private: KqpHost->AbortTransaction(QueryState->TxId); FillTxInfo(responseEv->Record); - responseEv->Record.GetRef().SetConsumedRu(1); - + responseEv->Record.GetRef().SetConsumedRu(1); + return Reply(std::move(responseEv), ctx); } @@ -1670,7 +1670,7 @@ private: if (Settings.LongSession && status != Ydb::StatusIds::SUCCESS) { bool isQueryById = queryRequest.GetAction() == NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED; - + TMaybe<TString> invalidatedId; if (HasSchemeOrFatalIssues(queryResult.Issues())) { auto compileResult = QueryState->QueryCompileResult; @@ -1703,15 +1703,15 @@ private: auto& stats = queryResult.QueryStats; stats.SetDurationUs(queryDuration.MicroSeconds()); - stats.SetWorkerCpuTimeUs(QueryState->CpuTime.MicroSeconds()); + stats.SetWorkerCpuTimeUs(QueryState->CpuTime.MicroSeconds()); if (QueryState->QueryCompileResult) { stats.MutableCompilation()->Swap(&QueryState->CompileStats); } auto requestInfo = TKqpRequestInfo(QueryState->TraceId, SessionId); if (IsExecuteAction(queryRequest.GetAction())) { - auto ru = CalcRequestUnit(stats); - record.SetConsumedRu(ru); + auto ru = CalcRequestUnit(stats); + record.SetConsumedRu(ru); CollectSystemViewQueryStats(ctx, &stats, queryDuration, queryRequest.GetDatabase(), ru); SlowLogQuery(ctx, requestInfo, queryDuration, status, [&record](){ ui64 resultsSize = 0; @@ -1932,11 +1932,11 @@ private: auto& queryRequest = QueryState->Request; auto& queryResult = QueryState->QueryResult; - auto arena = queryResult.ProtobufArenaPtr; - if (arena) { - record.Realloc(arena); - } - auto& ev = record.GetRef(); + auto arena = queryResult.ProtobufArenaPtr; + if (arena) { + record.Realloc(arena); + } + auto& ev = record.GetRef(); bool replyResults = IsExecuteAction(queryRequest.GetAction()); bool replyPlan = true; @@ -1958,10 +1958,10 @@ private: if (replyResults) { for (auto& result : queryResult.Results) { - // If we have result it must be allocated on protobuf arena - Y_ASSERT(result->GetArena()); - Y_ASSERT(ev.MutableResponse()->GetArena() == result->GetArena()); - ev.MutableResponse()->AddResults()->Swap(result); + // If we have result it must be allocated on protobuf arena + Y_ASSERT(result->GetArena()); + Y_ASSERT(ev.MutableResponse()->GetArena() == result->GetArena()); + ev.MutableResponse()->AddResults()->Swap(result); } } @@ -1994,7 +1994,7 @@ private: case NKikimrKqp::QUERY_ACTION_EXECUTE: replyQueryParameters = replyQueryId = queryRequest.GetQueryCachePolicy().keep_in_cache(); break; - + case NKikimrKqp::QUERY_ACTION_PARSE: case NKikimrKqp::QUERY_ACTION_VALIDATE: replyQueryParameters = true; diff --git a/ydb/core/kqp/node/kqp_node.cpp b/ydb/core/kqp/node/kqp_node.cpp index a1f0f96947..24f8f2ac79 100644 --- a/ydb/core/kqp/node/kqp_node.cpp +++ b/ydb/core/kqp/node/kqp_node.cpp @@ -32,11 +32,11 @@ namespace { #define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream) #define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream) -// Min interval between stats send from scan/compute actor to executor -constexpr TDuration MinStatInterval = TDuration::MilliSeconds(20); -// Max interval in case of no activety -constexpr TDuration MaxStatInterval = TDuration::MilliSeconds(100); - +// Min interval between stats send from scan/compute actor to executor +constexpr TDuration MinStatInterval = TDuration::MilliSeconds(20); +// Max interval in case of no activety +constexpr TDuration MaxStatInterval = TDuration::MilliSeconds(100); + template <class TTasksCollection> TString TasksIdsStr(const TTasksCollection& tasks) { TVector<ui64> ids; @@ -272,12 +272,12 @@ private: runtimeSettingsBase.UseLLVM = msgRtSettings.GetUseLLVM(); runtimeSettingsBase.UseSpilling = msgRtSettings.GetUseSpilling(); - if (msgRtSettings.HasRlPath()) { - runtimeSettingsBase.RlPath = msgRtSettings.GetRlPath(); - } - - runtimeSettingsBase.ReportStatsSettings = NYql::NDq::TReportStatsSettings{MinStatInterval, MaxStatInterval}; - + if (msgRtSettings.HasRlPath()) { + runtimeSettingsBase.RlPath = msgRtSettings.GetRlPath(); + } + + runtimeSettingsBase.ReportStatsSettings = NYql::NDq::TReportStatsSettings{MinStatInterval, MaxStatInterval}; + auto actorSystem = TlsActivationContext->ActorSystem(); // start compute actors diff --git a/ydb/core/kqp/opt/kqp_opt.cpp b/ydb/core/kqp/opt/kqp_opt.cpp index 0bfb595a4b..1bd771ece1 100644 --- a/ydb/core/kqp/opt/kqp_opt.cpp +++ b/ydb/core/kqp/opt/kqp_opt.cpp @@ -74,7 +74,7 @@ TExprBase ProjectColumns(const TExprBase& input, const THashSet<TStringBuf>& col return ProjectColumnsInternal(input, columnNames, ctx); } -TKqpTable BuildTableMeta(const TKikimrTableMetadata& meta, const TPositionHandle& pos, TExprContext& ctx) { +TKqpTable BuildTableMeta(const TKikimrTableMetadata& meta, const TPositionHandle& pos, TExprContext& ctx) { return Build<TKqpTable>(ctx, pos) .Path().Build(meta.Name) .PathId().Build(meta.PathId.ToString()) @@ -83,9 +83,9 @@ TKqpTable BuildTableMeta(const TKikimrTableMetadata& meta, const TPositionHandle .Done(); } -TKqpTable BuildTableMeta(const TKikimrTableDescription& tableDesc, const TPositionHandle& pos, TExprContext& ctx) { - YQL_ENSURE(tableDesc.Metadata); - return BuildTableMeta(*tableDesc.Metadata, pos, ctx); -} - +TKqpTable BuildTableMeta(const TKikimrTableDescription& tableDesc, const TPositionHandle& pos, TExprContext& ctx) { + YQL_ENSURE(tableDesc.Metadata); + return BuildTableMeta(*tableDesc.Metadata, pos, ctx); +} + } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/kqp_opt.h b/ydb/core/kqp/opt/kqp_opt.h index 6222d3779a..a86561d0c4 100644 --- a/ydb/core/kqp/opt/kqp_opt.h +++ b/ydb/core/kqp/opt/kqp_opt.h @@ -45,7 +45,7 @@ struct TKqpBuildQueryContext : TThrRefBase { bool IsKqpEffectsStage(const NYql::NNodes::TDqStageBase& stage); -TMaybe<NYql::NNodes::TKqlQuery> BuildKqlQuery(NYql::NNodes::TKiDataQuery query, const NYql::TKikimrTablesData& tablesData, +TMaybe<NYql::NNodes::TKqlQuery> BuildKqlQuery(NYql::NNodes::TKiDataQuery query, const NYql::TKikimrTablesData& tablesData, NYql::TExprContext& ctx, bool withSystemColumns, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx); TAutoPtr<NYql::IGraphTransformer> CreateKqpFinalizingOptTransformer(); diff --git a/ydb/core/kqp/opt/kqp_opt_effects.cpp b/ydb/core/kqp/opt/kqp_opt_effects.cpp index 577dd34f32..f5f2401ce9 100644 --- a/ydb/core/kqp/opt/kqp_opt_effects.cpp +++ b/ydb/core/kqp/opt/kqp_opt_effects.cpp @@ -271,20 +271,20 @@ TMaybeNode<TKqlQuery> BuildEffects(const TKqlQuery& query, TExprContext& ctx, if constexpr (GroupEffectsByTable) { TMap<TStringBuf, TVector<TKqlTableEffect>> tableEffectsMap; - for (const auto& maybeEffect: query.Effects()) { - if (const auto maybeList = maybeEffect.Maybe<TExprList>()) { - for (const auto effect : maybeList.Cast()) { - YQL_ENSURE(effect.Maybe<TKqlTableEffect>()); - auto tableEffect = effect.Cast<TKqlTableEffect>(); - - tableEffectsMap[tableEffect.Table().Path()].push_back(tableEffect); - } - } else { - YQL_ENSURE(maybeEffect.Maybe<TKqlTableEffect>()); - auto tableEffect = maybeEffect.Cast<TKqlTableEffect>(); - - tableEffectsMap[tableEffect.Table().Path()].push_back(tableEffect); - } + for (const auto& maybeEffect: query.Effects()) { + if (const auto maybeList = maybeEffect.Maybe<TExprList>()) { + for (const auto effect : maybeList.Cast()) { + YQL_ENSURE(effect.Maybe<TKqlTableEffect>()); + auto tableEffect = effect.Cast<TKqlTableEffect>(); + + tableEffectsMap[tableEffect.Table().Path()].push_back(tableEffect); + } + } else { + YQL_ENSURE(maybeEffect.Maybe<TKqlTableEffect>()); + auto tableEffect = maybeEffect.Cast<TKqlTableEffect>(); + + tableEffectsMap[tableEffect.Table().Path()].push_back(tableEffect); + } } for (const auto& pair: tableEffectsMap) { @@ -293,25 +293,25 @@ TMaybeNode<TKqlQuery> BuildEffects(const TKqlQuery& query, TExprContext& ctx, } } } else { - builtEffects.reserve(query.Effects().Size() * 2); - - for (const auto& maybeEffect : query.Effects()) { - if (const auto maybeList = maybeEffect.Maybe<TExprList>()) { - for (const auto effect : maybeList.Cast()) { - YQL_ENSURE(effect.Maybe<TKqlTableEffect>()); - auto tableEffect = effect.Cast<TKqlTableEffect>(); - - if (!BuildEffects(query.Pos(), {tableEffect}, ctx, kqpCtx, builtEffects)) { - return {}; - } - } - } else { - YQL_ENSURE(maybeEffect.Maybe<TKqlTableEffect>()); - auto tableEffect = maybeEffect.Cast<TKqlTableEffect>(); - - if (!BuildEffects(query.Pos(), {tableEffect}, ctx, kqpCtx, builtEffects)) { - return {}; - } + builtEffects.reserve(query.Effects().Size() * 2); + + for (const auto& maybeEffect : query.Effects()) { + if (const auto maybeList = maybeEffect.Maybe<TExprList>()) { + for (const auto effect : maybeList.Cast()) { + YQL_ENSURE(effect.Maybe<TKqlTableEffect>()); + auto tableEffect = effect.Cast<TKqlTableEffect>(); + + if (!BuildEffects(query.Pos(), {tableEffect}, ctx, kqpCtx, builtEffects)) { + return {}; + } + } + } else { + YQL_ENSURE(maybeEffect.Maybe<TKqlTableEffect>()); + auto tableEffect = maybeEffect.Cast<TKqlTableEffect>(); + + if (!BuildEffects(query.Pos(), {tableEffect}, ctx, kqpCtx, builtEffects)) { + return {}; + } } } } diff --git a/ydb/core/kqp/opt/kqp_opt_impl.h b/ydb/core/kqp/opt/kqp_opt_impl.h index 5a6054c5e4..927aa910e8 100644 --- a/ydb/core/kqp/opt/kqp_opt_impl.h +++ b/ydb/core/kqp/opt/kqp_opt_impl.h @@ -49,7 +49,7 @@ NYql::NNodes::TExprBase KqpBuildJoin(const NYql::NNodes::TExprBase& node, NYql:: const TKqpOptimizeContext& kqpCtx, NYql::IOptimizationContext& optCtx, const NYql::TParentsMap& parentsMap, bool allowStageMultiUsage); -TIntrusivePtr<NYql::TKikimrTableMetadata> GetIndexMetadata(const NYql::NNodes::TKqlReadTableIndex& index, - const NYql::TKikimrTablesData& tables, TStringBuf cluster); - +TIntrusivePtr<NYql::TKikimrTableMetadata> GetIndexMetadata(const NYql::NNodes::TKqlReadTableIndex& index, + const NYql::TKikimrTablesData& tables, TStringBuf cluster); + } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index 9f039c59ae..1eeb95daee 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -97,30 +97,30 @@ TExprBase BuildReadTable(const TKiReadTable& read, const TKikimrTableDescription : readNode; } -TExprBase BuildReadTableIndex(const TKiReadTable& read, const TKikimrTableDescription& tableData, - const TString& indexName, bool withSystemColumns, TExprContext& ctx) -{ - bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); - - auto kqlReadTable = Build<TKqlReadTableIndex>(ctx, read.Pos()) - .Table(BuildTableMeta(tableData, read.Pos(), ctx)) - .Range() - .From<TKqlKeyInc>() - .Build() - .To<TKqlKeyInc>() - .Build() - .Build() - .Columns(read.GetSelectColumns(ctx, tableData, withSystemColumns)) - .Settings() - .Build() - .Index().Build(indexName) - .Done(); - - return unwrapValues - ? UnwrapKiReadTableValues(kqlReadTable, tableData, kqlReadTable.Columns(), ctx) - : kqlReadTable; -} - +TExprBase BuildReadTableIndex(const TKiReadTable& read, const TKikimrTableDescription& tableData, + const TString& indexName, bool withSystemColumns, TExprContext& ctx) +{ + bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); + + auto kqlReadTable = Build<TKqlReadTableIndex>(ctx, read.Pos()) + .Table(BuildTableMeta(tableData, read.Pos(), ctx)) + .Range() + .From<TKqlKeyInc>() + .Build() + .To<TKqlKeyInc>() + .Build() + .Build() + .Columns(read.GetSelectColumns(ctx, tableData, withSystemColumns)) + .Settings() + .Build() + .Index().Build(indexName) + .Done(); + + return unwrapValues + ? UnwrapKiReadTableValues(kqlReadTable, tableData, kqlReadTable.Columns(), ctx) + : kqlReadTable; +} + TExprBase BuildUpsertTable(const TKiWriteTable& write, const TCoAtomList& inputColumns, const TKikimrTableDescription& tableData, TExprContext& ctx) { @@ -133,33 +133,33 @@ TExprBase BuildUpsertTable(const TKiWriteTable& write, const TCoAtomList& inputC return effect; } -TExprBase BuildUpsertTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, +TExprBase BuildUpsertTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, const TKikimrTableDescription& tableData, TExprContext& ctx) { - auto effect = Build<TKqlUpsertRowsIndex>(ctx, write.Pos()) - .Table(BuildTableMeta(tableData, write.Pos(), ctx)) - .Input(write.Input()) - .Columns(inputColumns) + auto effect = Build<TKqlUpsertRowsIndex>(ctx, write.Pos()) + .Table(BuildTableMeta(tableData, write.Pos(), ctx)) + .Input(write.Input()) + .Columns(inputColumns) .Done(); - return effect; -} + return effect; +} -TExprBase BuildReplaceTable(const TKiWriteTable& write, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ - const auto [data, columns] = CreateRowsToReplace(write.Input(), inputColumns, tableData, write.Pos(), ctx); +TExprBase BuildReplaceTable(const TKiWriteTable& write, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ + const auto [data, columns] = CreateRowsToReplace(write.Input(), inputColumns, tableData, write.Pos(), ctx); - return Build<TKqlUpsertRows>(ctx, write.Pos()) - .Table(BuildTableMeta(tableData, write.Pos(), ctx)) - .Input(data) - .Columns(columns) + return Build<TKqlUpsertRows>(ctx, write.Pos()) + .Table(BuildTableMeta(tableData, write.Pos(), ctx)) + .Input(data) + .Columns(columns) .Done(); -} +} -TExprBase BuildReplaceTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ +TExprBase BuildReplaceTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ const auto [data, columns] = CreateRowsToReplace(write.Input(), inputColumns, tableData, write.Pos(), ctx); auto effect = Build<TKqlUpsertRowsIndex>(ctx, write.Pos()) @@ -186,21 +186,21 @@ TExprBase BuildInsertTable(const TKiWriteTable& write, bool abort, const TCoAtom return effect; } -TExprBase BuildInsertTableWithIndex(const TKiWriteTable& write, bool abort, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ - auto effect = Build<TKqlInsertRowsIndex>(ctx, write.Pos()) - .Table(BuildTableMeta(tableData, write.Pos(), ctx)) - .Input(write.Input()) - .Columns(inputColumns) - .OnConflict() - .Value(abort ? "abort"sv : "revert"sv) - .Build() - .Done(); - - return effect; -} - +TExprBase BuildInsertTableWithIndex(const TKiWriteTable& write, bool abort, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ + auto effect = Build<TKqlInsertRowsIndex>(ctx, write.Pos()) + .Table(BuildTableMeta(tableData, write.Pos(), ctx)) + .Input(write.Input()) + .Columns(inputColumns) + .OnConflict() + .Value(abort ? "abort"sv : "revert"sv) + .Build() + .Done(); + + return effect; +} + TExprBase BuildUpdateOnTable(const TKiWriteTable& write, const TCoAtomList& inputColumns, const TKikimrTableDescription& tableData, TExprContext& ctx) { @@ -211,16 +211,16 @@ TExprBase BuildUpdateOnTable(const TKiWriteTable& write, const TCoAtomList& inpu .Done(); } -TExprBase BuildUpdateOnTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ - return Build<TKqlUpdateRowsIndex>(ctx, write.Pos()) - .Table(BuildTableMeta(tableData, write.Pos(), ctx)) - .Input(write.Input()) - .Columns(inputColumns) - .Done(); -} - +TExprBase BuildUpdateOnTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ + return Build<TKqlUpdateRowsIndex>(ctx, write.Pos()) + .Table(BuildTableMeta(tableData, write.Pos(), ctx)) + .Input(write.Input()) + .Columns(inputColumns) + .Done(); +} + TExprBase BuildDeleteTable(const TKiWriteTable& write, const TKikimrTableDescription& tableData, TExprContext& ctx) { const auto keysToDelete = ProjectColumns(write.Input(), tableData.Metadata->KeyColumnNames, ctx); @@ -230,15 +230,15 @@ TExprBase BuildDeleteTable(const TKiWriteTable& write, const TKikimrTableDescrip .Done(); } -TExprBase BuildDeleteTableWithIndex(const TKiWriteTable& write, const TKikimrTableDescription& tableData, TExprContext& ctx) { - const auto keysToDelete = ProjectColumns(write.Input(), tableData.Metadata->KeyColumnNames, ctx); - - return Build<TKqlDeleteRowsIndex>(ctx, write.Pos()) - .Table(BuildTableMeta(tableData, write.Pos(), ctx)) - .Input(keysToDelete) - .Done(); -} - +TExprBase BuildDeleteTableWithIndex(const TKiWriteTable& write, const TKikimrTableDescription& tableData, TExprContext& ctx) { + const auto keysToDelete = ProjectColumns(write.Input(), tableData.Metadata->KeyColumnNames, ctx); + + return Build<TKqlDeleteRowsIndex>(ctx, write.Pos()) + .Table(BuildTableMeta(tableData, write.Pos(), ctx)) + .Input(keysToDelete) + .Done(); +} + TExprBase BuildRowsToDelete(const TKikimrTableDescription& tableData, bool withSystemColumns, const TCoLambda& filter, const TPositionHandle pos, TExprContext& ctx) { @@ -522,7 +522,7 @@ TVector<TExprBase> BuildUpdateTableWithIndex(const TKiUpdateTable& update, const return effects; } -TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, const TKikimrTablesData& tablesData, +TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { TKikimrKey key(ctx); @@ -530,40 +530,40 @@ TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, con YQL_ENSURE(key.GetKeyType() == TKikimrKey::Type::Table); auto& tableData = GetTableData(tablesData, read.DataSource().Cluster(), key.GetTablePath()); - if (key.GetView()) { - const auto& indexName = key.GetView().GetRef(); - if (!ValidateTableHasIndex(tableData.Metadata, ctx, read.Pos())) { - return nullptr; - } - - if (kqpCtx->IsScanQuery()) { - const TString err = "Secondary index is not supported for ScanQuery"; - ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err)); - return nullptr; - } - - auto [metadata, state] = tableData.Metadata->GetIndexMetadata(indexName); - YQL_ENSURE(metadata, "unable to find metadata for index: " << indexName); - YQL_ENSURE(state == TIndexDescription::EIndexState::Ready - || state == TIndexDescription::EIndexState::WriteOnly); - - if (state != TIndexDescription::EIndexState::Ready) { - auto err = TStringBuilder() - << "Requested index: " << indexName - << " is not ready to use"; - ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_INDEX_IS_NOT_READY, err)); - return nullptr; - } - - return BuildReadTableIndex(read, tableData, indexName, withSystemColumns, ctx).Ptr(); - } - + if (key.GetView()) { + const auto& indexName = key.GetView().GetRef(); + if (!ValidateTableHasIndex(tableData.Metadata, ctx, read.Pos())) { + return nullptr; + } + + if (kqpCtx->IsScanQuery()) { + const TString err = "Secondary index is not supported for ScanQuery"; + ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err)); + return nullptr; + } + + auto [metadata, state] = tableData.Metadata->GetIndexMetadata(indexName); + YQL_ENSURE(metadata, "unable to find metadata for index: " << indexName); + YQL_ENSURE(state == TIndexDescription::EIndexState::Ready + || state == TIndexDescription::EIndexState::WriteOnly); + + if (state != TIndexDescription::EIndexState::Ready) { + auto err = TStringBuilder() + << "Requested index: " << indexName + << " is not ready to use"; + ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_INDEX_IS_NOT_READY, err)); + return nullptr; + } + + return BuildReadTableIndex(read, tableData, indexName, withSystemColumns, ctx).Ptr(); + } + return BuildReadTable(read, tableData, withSystemColumns, ctx, kqpCtx).Ptr(); } -TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ +TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ auto op = GetTableOp(write); switch (op) { case TYdbOperation::Upsert: @@ -584,43 +584,43 @@ TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputC } } -TExprBase WriteTableWithIndexUpdate(const TKiWriteTable& write, const TCoAtomList& inputColumns, - const TKikimrTableDescription& tableData, TExprContext& ctx) -{ - auto op = GetTableOp(write); - switch (op) { +TExprBase WriteTableWithIndexUpdate(const TKiWriteTable& write, const TCoAtomList& inputColumns, + const TKikimrTableDescription& tableData, TExprContext& ctx) +{ + auto op = GetTableOp(write); + switch (op) { case TYdbOperation::Upsert: - return BuildUpsertTableWithIndex(write, inputColumns, tableData, ctx); + return BuildUpsertTableWithIndex(write, inputColumns, tableData, ctx); case TYdbOperation::Replace: - return BuildReplaceTableWithIndex(write, inputColumns, tableData, ctx); + return BuildReplaceTableWithIndex(write, inputColumns, tableData, ctx); case TYdbOperation::InsertAbort: case TYdbOperation::InsertRevert: return BuildInsertTableWithIndex(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx); case TYdbOperation::UpdateOn: - return BuildUpdateOnTableWithIndex(write, inputColumns, tableData, ctx); + return BuildUpdateOnTableWithIndex(write, inputColumns, tableData, ctx); case TYdbOperation::DeleteOn: - return BuildDeleteTableWithIndex(write, tableData, ctx); - default: - YQL_ENSURE(false, "Unsupported table operation: " << (ui32)op << ", table: " << tableData.Metadata->Name); - } - - Y_UNREACHABLE(); -} - -TExprBase HandleWriteTable(const TKiWriteTable& write, TExprContext& ctx, const TKikimrTablesData& tablesData) { - auto& tableData = GetTableData(tablesData, write.DataSink().Cluster(), write.Table().Value()); - - auto inputColumnsSetting = GetSetting(write.Settings().Ref(), "input_columns"); - YQL_ENSURE(inputColumnsSetting); - auto inputColumns = TCoNameValueTuple(inputColumnsSetting).Value().Cast<TCoAtomList>(); - + return BuildDeleteTableWithIndex(write, tableData, ctx); + default: + YQL_ENSURE(false, "Unsupported table operation: " << (ui32)op << ", table: " << tableData.Metadata->Name); + } + + Y_UNREACHABLE(); +} + +TExprBase HandleWriteTable(const TKiWriteTable& write, TExprContext& ctx, const TKikimrTablesData& tablesData) { + auto& tableData = GetTableData(tablesData, write.DataSink().Cluster(), write.Table().Value()); + + auto inputColumnsSetting = GetSetting(write.Settings().Ref(), "input_columns"); + YQL_ENSURE(inputColumnsSetting); + auto inputColumns = TCoNameValueTuple(inputColumnsSetting).Value().Cast<TCoAtomList>(); + if (HasIndexesToWrite(tableData)) { - return WriteTableWithIndexUpdate(write, inputColumns, tableData, ctx); - } else { - return WriteTableSimple(write, inputColumns, tableData, ctx); - } -} - + return WriteTableWithIndexUpdate(write, inputColumns, tableData, ctx); + } else { + return WriteTableSimple(write, inputColumns, tableData, ctx); + } +} + TVector<TExprBase> HandleUpdateTable(const TKiUpdateTable& update, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns) { @@ -630,7 +630,7 @@ TVector<TExprBase> HandleUpdateTable(const TKiUpdateTable& update, TExprContext& return BuildUpdateTableWithIndex(update, tableData, withSystemColumns, ctx); } else { return { BuildUpdateTable(update, tableData, withSystemColumns, ctx) }; - } + } } TVector<TExprBase> HandleDeleteTable(const TKiDeleteTable& del, TExprContext& ctx, const TKikimrTablesData& tablesData, @@ -641,7 +641,7 @@ TVector<TExprBase> HandleDeleteTable(const TKiDeleteTable& del, TExprContext& ct return BuildDeleteTableWithIndex(del, tableData, withSystemColumns, ctx); } else { return { BuildDeleteTable(del, tableData, withSystemColumns, ctx) }; - } + } } } // namespace @@ -663,7 +663,7 @@ TIntrusivePtr<TKikimrTableMetadata> GetIndexMetadata(const TKqlReadTableIndex& r return indexMeta; } -TMaybe<TKqlQuery> BuildKqlQuery(TKiDataQuery query, const TKikimrTablesData& tablesData, TExprContext& ctx, +TMaybe<TKqlQuery> BuildKqlQuery(TKiDataQuery query, const TKikimrTablesData& tablesData, TExprContext& ctx, bool withSystemColumns, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { TVector<TExprBase> kqlEffects; @@ -717,11 +717,11 @@ TMaybe<TKqlQuery> BuildKqlQuery(TKiDataQuery query, const TKikimrTablesData& tab return input; }, ctx, optSettings); - - if (status == IGraphTransformer::TStatus::Error) { - return {}; - } - + + if (status == IGraphTransformer::TStatus::Error) { + return {}; + } + YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); YQL_ENSURE(TMaybeNode<TKqlQuery>(optResult)); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 71fc72f864..d951c5ce30 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -128,13 +128,13 @@ protected: DumpAppliedRule("RewriteTopSortOverIndexRead", node.Ptr(), output.Ptr(), ctx); return output; } - + TMaybeNode<TExprBase> RewriteTakeOverIndexRead(TExprBase node, TExprContext& ctx) { TExprBase output = KqpRewriteTakeOverIndexRead(node, ctx, KqpCtx); DumpAppliedRule("RewriteTakeOverIndexRead", node.Ptr(), output.Ptr(), ctx); return output; } - + TMaybeNode<TExprBase> RewriteFlatMapOverExtend(TExprBase node, TExprContext& ctx) { auto output = DqFlatMapOverExtend(node, ctx); DumpAppliedRule("RewriteFlatMapOverExtend", node.Ptr(), output.Ptr(), ctx); @@ -146,7 +146,7 @@ protected: DumpAppliedRule("RewriteIndexRead", node.Ptr(), output.Ptr(), ctx); return output; } - + TMaybeNode<TExprBase> RewriteLookupIndex(TExprBase node, TExprContext& ctx) { TExprBase output = KqpRewriteLookupIndex(node, ctx, KqpCtx); DumpAppliedRule("RewriteLookupIndex", node.Ptr(), output.Ptr(), ctx); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp index 387f79b723..96642998db 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp @@ -30,18 +30,18 @@ TExprBase KqpApplyExtractMembersToReadTable(TExprBase node, TExprContext& ctx) { auto read = extract.Input().Cast<TKqlReadTableBase>(); - if (auto maybeIndexRead = read.Maybe<TKqlReadTableIndex>()) { - auto indexRead = maybeIndexRead.Cast(); - + if (auto maybeIndexRead = read.Maybe<TKqlReadTableIndex>()) { + auto indexRead = maybeIndexRead.Cast(); + return Build<TKqlReadTableIndex>(ctx, extract.Pos()) - .Table(indexRead.Table()) - .Range(indexRead.Range()) - .Columns(extract.Members()) - .Index(indexRead.Index()) - .Settings(indexRead.Settings()) - .Done(); - } - + .Table(indexRead.Table()) + .Range(indexRead.Range()) + .Columns(extract.Members()) + .Index(indexRead.Index()) + .Settings(indexRead.Settings()) + .Done(); + } + return Build<TKqlReadTableBase>(ctx, extract.Pos()) .CallableName(read.CallableName()) .Table(read.Table()) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp index f244859a1d..4bb48475a7 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp @@ -1,238 +1,238 @@ #include <ydb/core/kqp/opt/kqp_opt_impl.h> #include <ydb/core/kqp/common/kqp_yql.h> #include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h> - + #include <ydb/library/yql/dq/opt/dq_opt_phy.h> - + namespace NKikimr::NKqp::NOpt { - -using namespace NYql; -using namespace NYql::NDq; -using namespace NYql::NNodes; - -namespace { - -bool CanPushTopSort(const TCoTopSort& node, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { - return IsKeySelectorPkPrefix(node.KeySelectorLambda(), tableDesc, columns); -} - -template<typename TRead> -bool CheckIndexCovering(const TRead& read, const TIntrusivePtr<TKikimrTableMetadata>& indexMeta) { - for (const auto& col : read.Columns()) { - if (!indexMeta->Columns.contains(col.StringValue())) { - return true; - } - } - return false; -} - -TExprBase DoRewriteIndexRead(const TKqlReadTableIndex& read, TExprContext& ctx, - const TKikimrTableDescription& tableDesc, TIntrusivePtr<TKikimrTableMetadata> indexMeta, - const TVector<TString>& extraColumns, const std::function<TExprBase(const TExprBase&)>& middleFilter = {}) -{ - const bool needDataRead = CheckIndexCovering(read, indexMeta); - + +using namespace NYql; +using namespace NYql::NDq; +using namespace NYql::NNodes; + +namespace { + +bool CanPushTopSort(const TCoTopSort& node, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { + return IsKeySelectorPkPrefix(node.KeySelectorLambda(), tableDesc, columns); +} + +template<typename TRead> +bool CheckIndexCovering(const TRead& read, const TIntrusivePtr<TKikimrTableMetadata>& indexMeta) { + for (const auto& col : read.Columns()) { + if (!indexMeta->Columns.contains(col.StringValue())) { + return true; + } + } + return false; +} + +TExprBase DoRewriteIndexRead(const TKqlReadTableIndex& read, TExprContext& ctx, + const TKikimrTableDescription& tableDesc, TIntrusivePtr<TKikimrTableMetadata> indexMeta, + const TVector<TString>& extraColumns, const std::function<TExprBase(const TExprBase&)>& middleFilter = {}) +{ + const bool needDataRead = CheckIndexCovering(read, indexMeta); + if (read.Range().From().ArgCount() == 0 && read.Range().To().ArgCount() == 0) { - TString indexName = read.Index().StringValue(); - auto issue = TIssue(ctx.GetPosition(read.Pos()), "Given predicate is not suitable for used index: " + indexName); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_KIKIMR_WRONG_INDEX_USAGE, issue); - ctx.AddWarning(issue); - } - - if (!needDataRead) { - // We can read all data from index table. - auto ret = Build<TKqlReadTable>(ctx, read.Pos()) - .Table(BuildTableMeta(*indexMeta, read.Pos(), ctx)) - .Range(read.Range()) - .Columns(read.Columns()) - .Settings(read.Settings()) - .Done(); - - if (middleFilter) { - return middleFilter(ret); - } - return ret; - } - - auto keyColumnsList = BuildKeyColumnsList(tableDesc, read.Pos(), ctx); - auto columns = MergeColumns(keyColumnsList, extraColumns, ctx); - - TExprBase readIndexTable = Build<TKqlReadTable>(ctx, read.Pos()) - .Table(BuildTableMeta(*indexMeta, read.Pos(), ctx)) - .Range(read.Range()) - .Columns(columns) - .Settings(read.Settings()) - .Done(); - - if (middleFilter) { - readIndexTable = middleFilter(readIndexTable); - } - - if (extraColumns) { - TCoArgument arg = Build<TCoArgument>(ctx, read.Pos()) - .Name("Arg") - .Done(); - - TVector<TExprBase> structMembers; - structMembers.reserve(keyColumnsList.Size()); - - for (const auto& c : keyColumnsList) { - auto member = Build<TCoNameValueTuple>(ctx, read.Pos()) - .Name().Build(c.Value()) - .Value<TCoMember>() - .Struct(arg) - .Name().Build(c.Value()) - .Build() - .Done(); - - structMembers.push_back(member); - } - - readIndexTable= Build<TCoMap>(ctx, read.Pos()) - .Input(readIndexTable) - .Lambda() - .Args({arg}) - .Body<TCoAsStruct>() - .Add(structMembers) - .Build() - .Build() - .Done(); - } - - return Build<TKqlLookupTable>(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(readIndexTable.Ptr()) - .Columns(read.Columns()) - .Done(); -} - -} // namespace - -TExprBase KqpRewriteIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - if (!kqpCtx.IsDataQuery()) { - return node; - } - - if (auto maybeIndexRead = node.Maybe<TKqlReadTableIndex>()) { - auto indexRead = maybeIndexRead.Cast(); - - const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, indexRead.Table().Path()); - const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(indexRead.Index().Value())); - - return DoRewriteIndexRead(indexRead, ctx, tableDesc, indexMeta, {}); - } - - return node; -} - -TExprBase KqpRewriteLookupIndex(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - if (!kqpCtx.IsDataQuery()) { - return node; - } - - if (auto maybeLookupIndex = node.Maybe<TKqlLookupIndex>()) { - auto lookupIndex = maybeLookupIndex.Cast(); - - const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, lookupIndex.Table().Path()); - const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(lookupIndex.Index().StringValue()); - - const bool needDataRead = CheckIndexCovering(lookupIndex, indexMeta); - - if (!needDataRead) { - return Build<TKqlLookupTable>(ctx, node.Pos()) - .Table(BuildTableMeta(*indexMeta, node.Pos(), ctx)) - .LookupKeys(lookupIndex.LookupKeys()) - .Columns(lookupIndex.Columns()) - .Done(); - } - - auto keyColumnsList = BuildKeyColumnsList(tableDesc, node.Pos(), ctx); - - TExprBase lookupIndexTable = Build<TKqlLookupTable>(ctx, node.Pos()) - .Table(BuildTableMeta(*indexMeta, node.Pos(), ctx)) - .LookupKeys(lookupIndex.LookupKeys()) - .Columns(keyColumnsList) - .Done(); - - return Build<TKqlLookupTable>(ctx, node.Pos()) - .Table(lookupIndex.Table()) - .LookupKeys(lookupIndexTable.Ptr()) - .Columns(lookupIndex.Columns()) - .Done(); - } - - return node; -} - -// The index and main table have same number of rows, so we can push TCoTopSort or TCoTake through TKqlLookupTable. -// The simplest way is to match TopSort or Take over TKqlReadTableIndex. - -TExprBase KqpRewriteTopSortOverIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - if (!kqpCtx.IsDataQuery()) { - return node; - } - - if (!node.Maybe<TCoTopSort>()) { - return node; - } - - auto topSort = node.Maybe<TCoTopSort>().Cast(); - - if (auto maybeReadTableIndex = topSort.Input().Maybe<TKqlReadTableIndex>()) { - auto readTableIndex = maybeReadTableIndex.Cast(); - - const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, readTableIndex.Table().Path()); - const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(readTableIndex.Index().Value())); - const auto& indexDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, indexMeta->Name); - - TVector<TString> sortByColumns; - - if (!CanPushTopSort(topSort, indexDesc, &sortByColumns)) { - return node; - } - - auto filter = [&ctx, &node, &topSort](const TExprBase& in) mutable { - auto newTopSort = Build<TCoTopSort>(ctx, node.Pos()) - .Input(in) - .KeySelectorLambda(ctx.DeepCopyLambda(topSort.KeySelectorLambda().Ref())) - .SortDirections(topSort.SortDirections()) - .Count(topSort.Count()) - .Done(); - return TExprBase(newTopSort); - }; - - return DoRewriteIndexRead(readTableIndex, ctx, tableDesc, indexMeta, sortByColumns, filter); - } - - return node; -} - -TExprBase KqpRewriteTakeOverIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - if (!kqpCtx.IsDataQuery()) { - return node; - } - - if (!node.Maybe<TCoTake>()) { - return node; - } - - auto take = node.Maybe<TCoTake>().Cast(); - - if (auto maybeReadTableIndex = take.Input().Maybe<TKqlReadTableIndex>()) { - auto readTableIndex = maybeReadTableIndex.Cast(); - - const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, readTableIndex.Table().Path()); - const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(readTableIndex.Index().Value())); - - auto filter = [&ctx, &node](const TExprBase& in) mutable { - // Change input for TCoTake. New input is result of TKqlReadTable. - return TExprBase(ctx.ChangeChild(*node.Ptr(), 0, in.Ptr())); - }; - - return DoRewriteIndexRead(readTableIndex, ctx, tableDesc, indexMeta, {}, filter); - } - - return node; -} - + TString indexName = read.Index().StringValue(); + auto issue = TIssue(ctx.GetPosition(read.Pos()), "Given predicate is not suitable for used index: " + indexName); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_KIKIMR_WRONG_INDEX_USAGE, issue); + ctx.AddWarning(issue); + } + + if (!needDataRead) { + // We can read all data from index table. + auto ret = Build<TKqlReadTable>(ctx, read.Pos()) + .Table(BuildTableMeta(*indexMeta, read.Pos(), ctx)) + .Range(read.Range()) + .Columns(read.Columns()) + .Settings(read.Settings()) + .Done(); + + if (middleFilter) { + return middleFilter(ret); + } + return ret; + } + + auto keyColumnsList = BuildKeyColumnsList(tableDesc, read.Pos(), ctx); + auto columns = MergeColumns(keyColumnsList, extraColumns, ctx); + + TExprBase readIndexTable = Build<TKqlReadTable>(ctx, read.Pos()) + .Table(BuildTableMeta(*indexMeta, read.Pos(), ctx)) + .Range(read.Range()) + .Columns(columns) + .Settings(read.Settings()) + .Done(); + + if (middleFilter) { + readIndexTable = middleFilter(readIndexTable); + } + + if (extraColumns) { + TCoArgument arg = Build<TCoArgument>(ctx, read.Pos()) + .Name("Arg") + .Done(); + + TVector<TExprBase> structMembers; + structMembers.reserve(keyColumnsList.Size()); + + for (const auto& c : keyColumnsList) { + auto member = Build<TCoNameValueTuple>(ctx, read.Pos()) + .Name().Build(c.Value()) + .Value<TCoMember>() + .Struct(arg) + .Name().Build(c.Value()) + .Build() + .Done(); + + structMembers.push_back(member); + } + + readIndexTable= Build<TCoMap>(ctx, read.Pos()) + .Input(readIndexTable) + .Lambda() + .Args({arg}) + .Body<TCoAsStruct>() + .Add(structMembers) + .Build() + .Build() + .Done(); + } + + return Build<TKqlLookupTable>(ctx, read.Pos()) + .Table(read.Table()) + .LookupKeys(readIndexTable.Ptr()) + .Columns(read.Columns()) + .Done(); +} + +} // namespace + +TExprBase KqpRewriteIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + if (!kqpCtx.IsDataQuery()) { + return node; + } + + if (auto maybeIndexRead = node.Maybe<TKqlReadTableIndex>()) { + auto indexRead = maybeIndexRead.Cast(); + + const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, indexRead.Table().Path()); + const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(indexRead.Index().Value())); + + return DoRewriteIndexRead(indexRead, ctx, tableDesc, indexMeta, {}); + } + + return node; +} + +TExprBase KqpRewriteLookupIndex(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + if (!kqpCtx.IsDataQuery()) { + return node; + } + + if (auto maybeLookupIndex = node.Maybe<TKqlLookupIndex>()) { + auto lookupIndex = maybeLookupIndex.Cast(); + + const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, lookupIndex.Table().Path()); + const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(lookupIndex.Index().StringValue()); + + const bool needDataRead = CheckIndexCovering(lookupIndex, indexMeta); + + if (!needDataRead) { + return Build<TKqlLookupTable>(ctx, node.Pos()) + .Table(BuildTableMeta(*indexMeta, node.Pos(), ctx)) + .LookupKeys(lookupIndex.LookupKeys()) + .Columns(lookupIndex.Columns()) + .Done(); + } + + auto keyColumnsList = BuildKeyColumnsList(tableDesc, node.Pos(), ctx); + + TExprBase lookupIndexTable = Build<TKqlLookupTable>(ctx, node.Pos()) + .Table(BuildTableMeta(*indexMeta, node.Pos(), ctx)) + .LookupKeys(lookupIndex.LookupKeys()) + .Columns(keyColumnsList) + .Done(); + + return Build<TKqlLookupTable>(ctx, node.Pos()) + .Table(lookupIndex.Table()) + .LookupKeys(lookupIndexTable.Ptr()) + .Columns(lookupIndex.Columns()) + .Done(); + } + + return node; +} + +// The index and main table have same number of rows, so we can push TCoTopSort or TCoTake through TKqlLookupTable. +// The simplest way is to match TopSort or Take over TKqlReadTableIndex. + +TExprBase KqpRewriteTopSortOverIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + if (!kqpCtx.IsDataQuery()) { + return node; + } + + if (!node.Maybe<TCoTopSort>()) { + return node; + } + + auto topSort = node.Maybe<TCoTopSort>().Cast(); + + if (auto maybeReadTableIndex = topSort.Input().Maybe<TKqlReadTableIndex>()) { + auto readTableIndex = maybeReadTableIndex.Cast(); + + const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, readTableIndex.Table().Path()); + const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(readTableIndex.Index().Value())); + const auto& indexDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, indexMeta->Name); + + TVector<TString> sortByColumns; + + if (!CanPushTopSort(topSort, indexDesc, &sortByColumns)) { + return node; + } + + auto filter = [&ctx, &node, &topSort](const TExprBase& in) mutable { + auto newTopSort = Build<TCoTopSort>(ctx, node.Pos()) + .Input(in) + .KeySelectorLambda(ctx.DeepCopyLambda(topSort.KeySelectorLambda().Ref())) + .SortDirections(topSort.SortDirections()) + .Count(topSort.Count()) + .Done(); + return TExprBase(newTopSort); + }; + + return DoRewriteIndexRead(readTableIndex, ctx, tableDesc, indexMeta, sortByColumns, filter); + } + + return node; +} + +TExprBase KqpRewriteTakeOverIndexRead(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + if (!kqpCtx.IsDataQuery()) { + return node; + } + + if (!node.Maybe<TCoTake>()) { + return node; + } + + auto take = node.Maybe<TCoTake>().Cast(); + + if (auto maybeReadTableIndex = take.Input().Maybe<TKqlReadTableIndex>()) { + auto readTableIndex = maybeReadTableIndex.Cast(); + + const auto& tableDesc = GetTableData(*kqpCtx.Tables, kqpCtx.Cluster, readTableIndex.Table().Path()); + const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(TString(readTableIndex.Index().Value())); + + auto filter = [&ctx, &node](const TExprBase& in) mutable { + // Change input for TCoTake. New input is result of TKqlReadTable. + return TExprBase(ctx.ChangeChild(*node.Ptr(), 0, in.Ptr())); + }; + + return DoRewriteIndexRead(readTableIndex, ctx, tableDesc, indexMeta, {}, filter); + } + + return node; +} + } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp index 54e8d8d841..ac0fe0bc39 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp @@ -69,79 +69,79 @@ TMaybeNode<TKqlKeyInc> GetRightTableKeyPrefix(const TKqlKeyRange& range) { return rangeFrom; } -TExprBase BuildLookupIndex(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read, +TExprBase BuildLookupIndex(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read, const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames, const TString& indexName) -{ - return Build<TKqlLookupIndex>(ctx, pos) - .Table(read.Table()) - .LookupKeys<TCoSkipNullMembers>() - .Input(keysToLookup) - .Members() - .Add(lookupNames) - .Build() - .Build() - .Columns(read.Columns()) - .Index() - .Build(indexName) - .Done(); -} - -TExprBase BuildLookupTable(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read, +{ + return Build<TKqlLookupIndex>(ctx, pos) + .Table(read.Table()) + .LookupKeys<TCoSkipNullMembers>() + .Input(keysToLookup) + .Members() + .Add(lookupNames) + .Build() + .Build() + .Columns(read.Columns()) + .Index() + .Build(indexName) + .Done(); +} + +TExprBase BuildLookupTable(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read, const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames) -{ - return Build<TKqlLookupTable>(ctx, pos) - .Table(read.Table()) - .LookupKeys<TCoSkipNullMembers>() - .Input(keysToLookup) - .Members() - .Add(lookupNames) - .Build() - .Build() - .Columns(read.Columns()) - .Done(); -} - -TVector<TExprBase> CreateRenames(const TMaybeNode<TCoFlatMap>& rightFlatmap, const TCoAtomList& tableColumns, - const TCoArgument& arg, const TStringBuf& rightLabel, TPositionHandle pos, TExprContext& ctx) -{ - TVector<TExprBase> renames; - if (rightFlatmap) { - const auto flatMapType = GetSeqItemType(rightFlatmap.Ref().GetTypeAnn()); - YQL_ENSURE(flatMapType->GetKind() == ETypeAnnotationKind::Struct); - renames.reserve(flatMapType->Cast<TStructExprType>()->GetSize()); - - for (const auto& column : flatMapType->Cast<TStructExprType>()->GetItems()) { - renames.emplace_back( - Build<TCoNameValueTuple>(ctx, pos) - .Name<TCoAtom>() - .Build(Join('.', rightLabel, column->GetName())) - .Value<TCoMember>() - .Struct(arg) - .Name<TCoAtom>() - .Build(column->GetName()) - .Build() - .Done()); - } - } else { - renames.reserve(tableColumns.Size()); - - for (const auto& column : tableColumns) { - renames.emplace_back( - Build<TCoNameValueTuple>(ctx, pos) - .Name<TCoAtom>() - .Build(Join('.', rightLabel, column.Value())) - .Value<TCoMember>() - .Struct(arg) - .Name(column) - .Build() - .Done()); - } - } - return renames; -} - - -//#define DBG(...) YQL_CLOG(DEBUG, ProviderKqp) << __VA_ARGS__ +{ + return Build<TKqlLookupTable>(ctx, pos) + .Table(read.Table()) + .LookupKeys<TCoSkipNullMembers>() + .Input(keysToLookup) + .Members() + .Add(lookupNames) + .Build() + .Build() + .Columns(read.Columns()) + .Done(); +} + +TVector<TExprBase> CreateRenames(const TMaybeNode<TCoFlatMap>& rightFlatmap, const TCoAtomList& tableColumns, + const TCoArgument& arg, const TStringBuf& rightLabel, TPositionHandle pos, TExprContext& ctx) +{ + TVector<TExprBase> renames; + if (rightFlatmap) { + const auto flatMapType = GetSeqItemType(rightFlatmap.Ref().GetTypeAnn()); + YQL_ENSURE(flatMapType->GetKind() == ETypeAnnotationKind::Struct); + renames.reserve(flatMapType->Cast<TStructExprType>()->GetSize()); + + for (const auto& column : flatMapType->Cast<TStructExprType>()->GetItems()) { + renames.emplace_back( + Build<TCoNameValueTuple>(ctx, pos) + .Name<TCoAtom>() + .Build(Join('.', rightLabel, column->GetName())) + .Value<TCoMember>() + .Struct(arg) + .Name<TCoAtom>() + .Build(column->GetName()) + .Build() + .Done()); + } + } else { + renames.reserve(tableColumns.Size()); + + for (const auto& column : tableColumns) { + renames.emplace_back( + Build<TCoNameValueTuple>(ctx, pos) + .Name<TCoAtom>() + .Build(Join('.', rightLabel, column.Value())) + .Value<TCoMember>() + .Struct(arg) + .Name(column) + .Build() + .Done()); + } + } + return renames; +} + + +//#define DBG(...) YQL_CLOG(DEBUG, ProviderKqp) << __VA_ARGS__ #define DBG(...) TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { @@ -155,27 +155,27 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext return {}; } - TMaybeNode<TKqlReadTableBase> rightRead; + TMaybeNode<TKqlReadTableBase> rightRead; TMaybeNode<TCoFlatMap> rightFlatmap; TMaybeNode<TCoFilterNullMembers> rightFilterNull; TMaybeNode<TCoSkipNullMembers> rightSkipNull; - if (auto readTable = join.RightInput().Maybe<TKqlReadTableBase>()) { + if (auto readTable = join.RightInput().Maybe<TKqlReadTableBase>()) { rightRead = readTable; } - if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TKqlReadTableBase>()) { + if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TKqlReadTableBase>()) { rightRead = readTable; rightFlatmap = join.RightInput().Maybe<TCoFlatMap>(); } - if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTableBase>()) { + if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTableBase>()) { rightRead = readTable; rightFlatmap = join.RightInput().Maybe<TCoFlatMap>(); rightFilterNull = rightFlatmap.Input().Cast<TCoFilterNullMembers>(); } - if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTableBase>()) { + if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTableBase>()) { rightRead = readTable; rightFlatmap = join.RightInput().Maybe<TCoFlatMap>(); rightSkipNull = rightFlatmap.Input().Cast<TCoSkipNullMembers>(); @@ -185,9 +185,9 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext return {}; } - Y_ENSURE(rightRead.Maybe<TKqlReadTable>() || rightRead.Maybe<TKqlReadTableIndex>()); - - const TKqlReadTableBase read = rightRead.Cast(); + Y_ENSURE(rightRead.Maybe<TKqlReadTable>() || rightRead.Maybe<TKqlReadTableIndex>()); + + const TKqlReadTableBase read = rightRead.Cast(); if (!read.Table().SysView().Value().empty()) { // Can't lookup in system views return {}; @@ -204,18 +204,18 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext } auto rightTableKeyPrefix = maybeRightTableKeyPrefix.Cast(); - TString lookupTable; - TString indexName; - - if (auto indexRead = rightRead.Maybe<TKqlReadTableIndex>()) { - indexName = indexRead.Cast().Index().StringValue(); - lookupTable = GetIndexMetadata(indexRead.Cast(), *kqpCtx.Tables, kqpCtx.Cluster)->Name; - } else { - lookupTable = read.Table().Path().StringValue(); - } - - const auto& rightTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable); - + TString lookupTable; + TString indexName; + + if (auto indexRead = rightRead.Maybe<TKqlReadTableIndex>()) { + indexName = indexRead.Cast().Index().StringValue(); + lookupTable = GetIndexMetadata(indexRead.Cast(), *kqpCtx.Tables, kqpCtx.Cluster)->Name; + } else { + lookupTable = read.Table().Path().StringValue(); + } + + const auto& rightTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable); + TMap<std::string_view, TString> rightJoinKeyToLeft; TVector<TCoAtom> rightKeyColumns; rightKeyColumns.reserve(join.JoinKeys().Size()); @@ -360,9 +360,9 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext .Build() .Done(); - TExprBase lookup = indexName - ? BuildLookupIndex(ctx, join.Pos(), read, keysToLookup, lookupNames, indexName) - : BuildLookupTable(ctx, join.Pos(), read, keysToLookup, lookupNames); + TExprBase lookup = indexName + ? BuildLookupIndex(ctx, join.Pos(), read, keysToLookup, lookupNames, indexName) + : BuildLookupTable(ctx, join.Pos(), read, keysToLookup, lookupNames); // Skip null keys in lookup part as for equijoin semantics null != null, // so we can't have nulls in lookup part @@ -398,7 +398,7 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext auto arg = TCoArgument(ctx.NewArgument(join.Pos(), "row")); auto rightLabel = join.RightLabel().Cast<TCoAtom>().Value(); - TVector<TExprBase> renames = CreateRenames(rightFlatmap, read.Columns(), arg, rightLabel, join.Pos(), ctx); + TVector<TExprBase> renames = CreateRenames(rightFlatmap, read.Columns(), arg, rightLabel, join.Pos(), ctx); lookup = Build<TCoMap>(ctx, join.Pos()) .Input(lookup) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp index ed46f3e2f7..38933b19fb 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp @@ -125,43 +125,43 @@ TExprBase KqpPushPredicateToReadTable(TExprBase node, TExprContext& ctx, const T return node; } - TMaybeNode<TKqlReadTableBase> readTable; + TMaybeNode<TKqlReadTableBase> readTable; TMaybeNode<TCoFilterNullMembers> filterNull; TMaybeNode<TCoSkipNullMembers> skipNull; - TMaybeNode<TCoAtom> indexName; - + TMaybeNode<TCoAtom> indexName; + if (auto maybeRead = flatmap.Input().Maybe<TKqlReadTable>()) { readTable = maybeRead.Cast(); } - if (auto maybeRead = flatmap.Input().Maybe<TKqlReadTableIndex>()) { - readTable = maybeRead.Cast(); - indexName = maybeRead.Cast().Index(); - } - + if (auto maybeRead = flatmap.Input().Maybe<TKqlReadTableIndex>()) { + readTable = maybeRead.Cast(); + indexName = maybeRead.Cast().Index(); + } + if (auto maybeRead = flatmap.Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTable>()) { readTable = maybeRead.Cast(); filterNull = flatmap.Input().Cast<TCoFilterNullMembers>(); } - if (auto maybeRead = flatmap.Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTableIndex>()) { - readTable = maybeRead.Cast(); - filterNull = flatmap.Input().Cast<TCoFilterNullMembers>(); - indexName = maybeRead.Cast().Index(); - } - + if (auto maybeRead = flatmap.Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTableIndex>()) { + readTable = maybeRead.Cast(); + filterNull = flatmap.Input().Cast<TCoFilterNullMembers>(); + indexName = maybeRead.Cast().Index(); + } + if (auto maybeRead = flatmap.Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTable>()) { readTable = maybeRead.Cast(); skipNull = flatmap.Input().Cast<TCoSkipNullMembers>(); } - if (auto maybeRead = flatmap.Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTableIndex>()) { - readTable = maybeRead.Cast(); - skipNull = flatmap.Input().Cast<TCoSkipNullMembers>(); - indexName = maybeRead.Cast().Index(); - } - + if (auto maybeRead = flatmap.Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTableIndex>()) { + readTable = maybeRead.Cast(); + skipNull = flatmap.Input().Cast<TCoSkipNullMembers>(); + indexName = maybeRead.Cast().Index(); + } + if (!readTable) { return node; } @@ -172,10 +172,10 @@ TExprBase KqpPushPredicateToReadTable(TExprBase node, TExprContext& ctx, const T return node; } - auto& mainTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); + auto& mainTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); + + auto& tableDesc = indexName ? kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) : mainTableDesc; - auto& tableDesc = indexName ? kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) : mainTableDesc; - YQL_ENSURE(tableDesc.Metadata->Kind != EKikimrTableKind::Olap); auto row = flatmap.Lambda().Args().Arg(0); @@ -208,20 +208,20 @@ TExprBase KqpPushPredicateToReadTable(TExprBase node, TExprContext& ctx, const T if (useLookup) { auto lookupKeys = BuildEquiRangeLookup(keyRange, tableDesc, read.Pos(), ctx); - if (indexName) { - readInput = Build<TKqlLookupIndex>(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .Index(indexName.Cast()) - .Done(); - } else { - readInput = Build<TKqlLookupTable>(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .Done(); - } + if (indexName) { + readInput = Build<TKqlLookupIndex>(ctx, read.Pos()) + .Table(read.Table()) + .LookupKeys(lookupKeys) + .Columns(read.Columns()) + .Index(indexName.Cast()) + .Done(); + } else { + readInput = Build<TKqlLookupTable>(ctx, read.Pos()) + .Table(read.Table()) + .LookupKeys(lookupKeys) + .Columns(read.Columns()) + .Done(); + } } else { auto keyRangeExpr = BuildKeyRangeExpr(keyRange, tableDesc, node.Pos(), ctx); @@ -234,22 +234,22 @@ TExprBase KqpPushPredicateToReadTable(TExprBase node, TExprContext& ctx, const T } } - if (indexName) { - readInput = Build<TKqlReadTableIndex>(ctx, read.Pos()) - .Table(read.Table()) - .Range(keyRangeExpr) - .Columns(read.Columns()) - .Index(indexName.Cast()) - .Settings(settings.BuildNode(ctx, read.Pos())) - .Done(); - } else { - readInput = Build<TKqlReadTable>(ctx, read.Pos()) - .Table(read.Table()) - .Range(keyRangeExpr) - .Columns(read.Columns()) - .Settings(settings.BuildNode(ctx, read.Pos())) - .Done(); - } + if (indexName) { + readInput = Build<TKqlReadTableIndex>(ctx, read.Pos()) + .Table(read.Table()) + .Range(keyRangeExpr) + .Columns(read.Columns()) + .Index(indexName.Cast()) + .Settings(settings.BuildNode(ctx, read.Pos())) + .Done(); + } else { + readInput = Build<TKqlReadTable>(ctx, read.Pos()) + .Table(read.Table()) + .Range(keyRangeExpr) + .Columns(read.Columns()) + .Settings(settings.BuildNode(ctx, read.Pos())) + .Done(); + } } auto input = readInput.Cast(); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h index 8b03010165..9e84d83d82 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h @@ -31,18 +31,18 @@ NYql::NNodes::TExprBase KqpRewriteSqlInToEquiJoin(const NYql::NNodes::TExprBase& NYql::NNodes::TExprBase KqpRewriteSqlInCompactToJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx); -NYql::NNodes::TExprBase KqpRewriteIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx); - -NYql::NNodes::TExprBase KqpRewriteLookupIndex(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx); - -NYql::NNodes::TExprBase KqpRewriteTopSortOverIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext&, - const TKqpOptimizeContext& kqpCtx); - -NYql::NNodes::TExprBase KqpRewriteTakeOverIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext&, - const TKqpOptimizeContext& kqpCtx); - +NYql::NNodes::TExprBase KqpRewriteIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, + const TKqpOptimizeContext& kqpCtx); + +NYql::NNodes::TExprBase KqpRewriteLookupIndex(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, + const TKqpOptimizeContext& kqpCtx); + +NYql::NNodes::TExprBase KqpRewriteTopSortOverIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext&, + const TKqpOptimizeContext& kqpCtx); + +NYql::NNodes::TExprBase KqpRewriteTakeOverIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext&, + const TKqpOptimizeContext& kqpCtx); + NYql::NNodes::TExprBase KqpDeleteOverLookup(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx, const TKqpOptimizeContext &kqpCtx); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp index eee05e274b..fd5d213001 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp @@ -40,24 +40,24 @@ TExprBase KqpRewriteSqlInToEquiJoin(const TExprBase& node, TExprContext& ctx, co return node; } - if (!flatMap.Input().Maybe<TKqlReadTable>() && !flatMap.Input().Maybe<TKqlReadTableIndex>()) { + if (!flatMap.Input().Maybe<TKqlReadTable>() && !flatMap.Input().Maybe<TKqlReadTableIndex>()) { return node; } - const auto readTable = flatMap.Input().Cast<TKqlReadTableBase>(); - + const auto readTable = flatMap.Input().Cast<TKqlReadTableBase>(); + if (!readTable.Table().SysView().Value().empty()) { return node; } - TString lookupTable; - - if (auto indexRead = flatMap.Input().Maybe<TKqlReadTableIndex>()) { - lookupTable = GetIndexMetadata(indexRead.Cast(), *kqpCtx.Tables, kqpCtx.Cluster)->Name; - } else { - lookupTable = readTable.Table().Path().StringValue(); - } - + TString lookupTable; + + if (auto indexRead = flatMap.Input().Maybe<TKqlReadTableIndex>()) { + lookupTable = GetIndexMetadata(indexRead.Cast(), *kqpCtx.Tables, kqpCtx.Cluster)->Name; + } else { + lookupTable = readTable.Table().Path().StringValue(); + } + const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable); const auto& rangeFrom = readTable.Range().From(); const auto& rangeTo = readTable.Range().To(); diff --git a/ydb/core/kqp/opt/logical/ya.make b/ydb/core/kqp/opt/logical/ya.make index 9a3730ff19..b9811606c1 100644 --- a/ydb/core/kqp/opt/logical/ya.make +++ b/ydb/core/kqp/opt/logical/ya.make @@ -9,7 +9,7 @@ SRCS( kqp_opt_log_effects.cpp kqp_opt_log_extract.cpp kqp_opt_log_join.cpp - kqp_opt_log_indexes.cpp + kqp_opt_log_indexes.cpp kqp_opt_log_ranges.cpp kqp_opt_log_ranges_predext.cpp kqp_opt_log_sqlin.cpp diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp index 314c65ded6..11e1af7ae8 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp @@ -53,10 +53,10 @@ public: AddHandler(0, &TCoOrderedLMap::Match, HNDL(PushOrderedLMapToStage<false>)); AddHandler(0, &TKqlInsertRows::Match, HNDL(BuildInsertStages)); AddHandler(0, &TKqlUpdateRows::Match, HNDL(BuildUpdateStages)); - AddHandler(0, &TKqlUpdateRowsIndex::Match, HNDL(BuildUpdateIndexStages)); - AddHandler(0, &TKqlUpsertRowsIndex::Match, HNDL(BuildUpsertIndexStages)); - AddHandler(0, &TKqlInsertRowsIndex::Match, HNDL(BuildInsertIndexStages)); - AddHandler(0, &TKqlDeleteRowsIndex::Match, HNDL(BuildDeleteIndexStages)); + AddHandler(0, &TKqlUpdateRowsIndex::Match, HNDL(BuildUpdateIndexStages)); + AddHandler(0, &TKqlUpsertRowsIndex::Match, HNDL(BuildUpsertIndexStages)); + AddHandler(0, &TKqlInsertRowsIndex::Match, HNDL(BuildInsertIndexStages)); + AddHandler(0, &TKqlDeleteRowsIndex::Match, HNDL(BuildDeleteIndexStages)); AddHandler(0, &TDqStage::Match, HNDL(FloatUpStage)); AddHandler(0, &TCoHasItems::Match, HNDL(BuildHasItems)); AddHandler(0, &TCoToOptional::Match, HNDL(BuildScalarPrecompute)); @@ -270,30 +270,30 @@ protected: return output; } - TMaybeNode<TExprBase> BuildUpdateIndexStages(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpBuildUpdateIndexStages(node, ctx, KqpCtx); - DumpAppliedRule("BuildUpdateIndexStages", node.Ptr(), output.Ptr(), ctx); - return output; - } - - TMaybeNode<TExprBase> BuildUpsertIndexStages(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpBuildUpsertIndexStages(node, ctx, KqpCtx); - DumpAppliedRule("BuildUpsertIndexStages", node.Ptr(), output.Ptr(), ctx); - return output; - } - - TMaybeNode<TExprBase> BuildInsertIndexStages(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpBuildInsertIndexStages(node, ctx, KqpCtx); - DumpAppliedRule("BuildInsertIndexStages", node.Ptr(), output.Ptr(), ctx); - return output; - } - - TMaybeNode<TExprBase> BuildDeleteIndexStages(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpBuildDeleteIndexStages(node, ctx, KqpCtx); - DumpAppliedRule("BuildDeleteIndexStages", node.Ptr(), output.Ptr(), ctx); - return output; - } - + TMaybeNode<TExprBase> BuildUpdateIndexStages(TExprBase node, TExprContext& ctx) { + TExprBase output = KqpBuildUpdateIndexStages(node, ctx, KqpCtx); + DumpAppliedRule("BuildUpdateIndexStages", node.Ptr(), output.Ptr(), ctx); + return output; + } + + TMaybeNode<TExprBase> BuildUpsertIndexStages(TExprBase node, TExprContext& ctx) { + TExprBase output = KqpBuildUpsertIndexStages(node, ctx, KqpCtx); + DumpAppliedRule("BuildUpsertIndexStages", node.Ptr(), output.Ptr(), ctx); + return output; + } + + TMaybeNode<TExprBase> BuildInsertIndexStages(TExprBase node, TExprContext& ctx) { + TExprBase output = KqpBuildInsertIndexStages(node, ctx, KqpCtx); + DumpAppliedRule("BuildInsertIndexStages", node.Ptr(), output.Ptr(), ctx); + return output; + } + + TMaybeNode<TExprBase> BuildDeleteIndexStages(TExprBase node, TExprContext& ctx) { + TExprBase output = KqpBuildDeleteIndexStages(node, ctx, KqpCtx); + DumpAppliedRule("BuildDeleteIndexStages", node.Ptr(), output.Ptr(), ctx); + return output; + } + TMaybeNode<TExprBase> FloatUpStage(TExprBase node, TExprContext& ctx) { TExprBase output = KqpFloatUpStage(node, ctx); DumpAppliedRule("FloatUpStage", node.Ptr(), output.Ptr(), ctx); diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp index b3efda2f69..f49aff031f 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp @@ -316,9 +316,9 @@ TExprBase KqpBuildLookupTableStage(TExprBase node, TExprContext& ctx) { return node; } const TKqlLookupTable& lookup = node.Cast<TKqlLookupTable>(); - + YQL_ENSURE(lookup.CallableName() == TKqlLookupTable::CallableName()); - + TMaybeNode<TDqStage> stage; if (!RequireLookupPrecomputeStage(lookup)) { diff --git a/ydb/core/kqp/prepare/kqp_query_finalize.cpp b/ydb/core/kqp/prepare/kqp_query_finalize.cpp index c8954d2de3..5e442780f7 100644 --- a/ydb/core/kqp/prepare/kqp_query_finalize.cpp +++ b/ydb/core/kqp/prepare/kqp_query_finalize.cpp @@ -2,7 +2,7 @@ #include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h> #include <ydb/core/tx/datashard/sys_tables.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/issue/yql_issue.h> @@ -18,9 +18,9 @@ namespace { const TStringBuf LocksInvalidatedResultName = "tx_locks_invalidated"; const TStringBuf LocksInvalidatedListName = "tx_locks_invalidated_list"; -const TStringBuf LocksTableName = "/sys/locks2"; -const TStringBuf LocksTableVersion = "0"; -const TString LocksTablePathId = TKikimrPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2).ToString(); +const TStringBuf LocksTableName = "/sys/locks2"; +const TStringBuf LocksTableVersion = "0"; +const TString LocksTablePathId = TKikimrPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2).ToString(); const ui64 LocksInvalidatedCount = 1; TExprBase GetDeferredEffectsList(const TDeferredEffects& effects, TPositionHandle pos, TExprContext& ctx) { @@ -48,11 +48,11 @@ TExprBase GetEraseLocksEffects(const TString& cluster, TPositionHandle pos, TCoP .Args({"lockItem"}) .Body<TKiEraseRow>() .Cluster().Build(cluster) - .Table<TKiVersionedTable>() - .Path<TCoAtom>().Build(LocksTableName) - .SchemaVersion<TCoAtom>().Build(LocksTableVersion) - .PathId<TCoAtom>().Build(LocksTablePathId) - .Build() + .Table<TKiVersionedTable>() + .Path<TCoAtom>().Build(LocksTableName) + .SchemaVersion<TCoAtom>().Build(LocksTableVersion) + .PathId<TCoAtom>().Build(LocksTablePathId) + .Build() .Key() .Add() .Name().Build("LockId") @@ -455,11 +455,11 @@ private: auto selectLock = Build<TKiSelectRow>(ctx, pos) .Cluster().Build(Cluster) - .Table<TKiVersionedTable>() - .Path<TCoAtom>().Build(LocksTableName) - .SchemaVersion<TCoAtom>().Build(LocksTableVersion) - .PathId<TCoAtom>().Build(LocksTablePathId) - .Build() + .Table<TKiVersionedTable>() + .Path<TCoAtom>().Build(LocksTableName) + .SchemaVersion<TCoAtom>().Build(LocksTableVersion) + .PathId<TCoAtom>().Build(LocksTablePathId) + .Build() .Key() .Add() .Name().Build("LockId") diff --git a/ydb/core/kqp/prepare/kqp_query_simplify.cpp b/ydb/core/kqp/prepare/kqp_query_simplify.cpp index 76c1975c91..d8fefeb939 100644 --- a/ydb/core/kqp/prepare/kqp_query_simplify.cpp +++ b/ydb/core/kqp/prepare/kqp_query_simplify.cpp @@ -44,7 +44,7 @@ TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) { auto blacklistedNode = FindNode(conditional.Predicate().Ptr(), [](const TExprNode::TPtr& exprNode) { auto node = TExprBase(exprNode); - if (node.Maybe<TKiSelectRow>() || node.Maybe<TKiSelectRangeBase>()) { + if (node.Maybe<TKiSelectRow>() || node.Maybe<TKiSelectRangeBase>()) { return true; } @@ -86,7 +86,7 @@ TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) { auto blacklistedNode = FindNode(conditional.Predicate().Ptr(), [](const TExprNode::TPtr& exprNode) { auto node = TExprBase(exprNode); - if (node.Maybe<TKiSelectRow>() || node.Maybe<TKiSelectRangeBase>()) { + if (node.Maybe<TKiSelectRow>() || node.Maybe<TKiSelectRangeBase>()) { return true; } @@ -173,10 +173,10 @@ TExprNode::TPtr MergeMapsWithSameLambda(TExprBase node, TExprContext& ctx) { if (!node.Maybe<TCoExtend>()) { return node.Ptr(); } - + bool hasInputsToMerge = false; TVector<std::pair<TMaybeNode<TCoLambda>, TVector<TExprBase>>> inputs; - + auto extend = node.Cast<TCoExtend>(); for (const auto& list : extend) { TMaybeNode<TExprBase> input; @@ -193,14 +193,14 @@ TExprNode::TPtr MergeMapsWithSameLambda(TExprBase node, TExprContext& ctx) { lambda = maybeMap.Cast().Lambda(); input = maybeMap.Cast().Input(); buildList = true; - } - + } + if (buildList) { input = Build<TCoToList>(ctx, node.Pos()) .Optional(input.Cast()) - .Done(); - } - + .Done(); + } + if (lambda && !IsKqlPureLambda(lambda.Cast())) { if (!inputs.empty() && inputs.back().first && inputs.back().first.Cast().Raw() == lambda.Cast().Raw()) { inputs.back().second.push_back(input.Cast()); @@ -208,10 +208,10 @@ TExprNode::TPtr MergeMapsWithSameLambda(TExprBase node, TExprContext& ctx) { } else { inputs.emplace_back(lambda, TVector<TExprBase>{input.Cast()}); } - } else { + } else { inputs.emplace_back(TMaybeNode<TCoLambda>(), TVector<TExprBase>{list}); - } - } + } + } if (!hasInputsToMerge) { return node.Ptr(); @@ -259,8 +259,8 @@ TExprNode::TPtr MergeMapsWithSameLambda(TExprBase node, TExprContext& ctx) { } return ret.Ptr(); -} - +} + TExprNode::TPtr RewritePresentIfToFlatMap(TExprBase node, TExprContext& ctx) { if (!node.Maybe<TCoIfPresent>()) { return node.Ptr(); @@ -306,10 +306,10 @@ public: TExprBase node(input); ret = MergeMapsWithSameLambda<TCoMap>(node, ctx); - if (ret != input) { - return ret; - } - + if (ret != input) { + return ret; + } + ret = MergeMapsWithSameLambda<TCoFlatMap>(node, ctx); if (ret != input) { return ret; diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index cecf0b9515..ee51c7349c 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -23,17 +23,17 @@ const TTypeAnnotationNode* MakeKqpEffectType(TExprContext& ctx) { return ctx.MakeType<TResourceExprType>(KqpEffectTag); } -bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tableDesc, - const TKikimrTableMetadataPtr meta, TExprContext& ctx) -{ - YQL_ENSURE(meta); +bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tableDesc, + const TKikimrTableMetadataPtr meta, TExprContext& ctx) +{ + YQL_ENSURE(meta); for (ui32 i = 0; i < tuple.ArgCount(); ++i) { auto actualType = tuple.Arg(i).Ref().GetTypeAnn(); YQL_ENSURE(actualType); - YQL_ENSURE(i < meta->KeyColumnNames.size()); - auto expectedType = tableDesc.GetColumnType(meta->KeyColumnNames[i]); + YQL_ENSURE(i < meta->KeyColumnNames.size()); + auto expectedType = tableDesc.GetColumnType(meta->KeyColumnNames[i]); YQL_ENSURE(expectedType); if (IsSameAnnotation(*expectedType, *actualType)) { @@ -48,8 +48,8 @@ bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tab ctx.AddError(TIssue(ctx.GetPosition(tuple.Pos()), TStringBuilder() << "Table key type mismatch" - << ", column: " << meta->KeyColumnNames[i] - << ", table: " << meta->Name + << ", column: " << meta->KeyColumnNames[i] + << ", table: " << meta->Name << ", expected: " << *expectedType << ", actual: " << *actualType)); @@ -141,61 +141,61 @@ const TFlowExprType* GetWideRowsType(TExprContext& ctx, const TStructExprType* r return ctx.MakeType<TFlowExprType>(wideRowType); } -bool CalcKeyColumnsCount(TExprContext& ctx, const TPositionHandle pos, const TStructExprType& structType, - const TKikimrTableDescription& tableDesc, const TKikimrTableMetadata& metadata, ui32& keyColumnsCount) -{ - for (auto& keyColumnName : metadata.KeyColumnNames) { - auto itemIndex = structType.FindItem(keyColumnName); - if (!itemIndex) { - break; - } - - auto itemType = structType.GetItems()[*itemIndex]->GetItemType(); - auto keyColumnType = tableDesc.GetColumnType(keyColumnName); - - if (CanCompare<true>(itemType, keyColumnType) == ECompareOptions::Uncomparable) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() - << "Invalid column type in table lookup, column: " << keyColumnName - << ", expected: " << FormatType(keyColumnType) - << ", actual: " << FormatType(itemType))); - return false; - } - - ++keyColumnsCount; - } - return true; -} - +bool CalcKeyColumnsCount(TExprContext& ctx, const TPositionHandle pos, const TStructExprType& structType, + const TKikimrTableDescription& tableDesc, const TKikimrTableMetadata& metadata, ui32& keyColumnsCount) +{ + for (auto& keyColumnName : metadata.KeyColumnNames) { + auto itemIndex = structType.FindItem(keyColumnName); + if (!itemIndex) { + break; + } + + auto itemType = structType.GetItems()[*itemIndex]->GetItemType(); + auto keyColumnType = tableDesc.GetColumnType(keyColumnName); + + if (CanCompare<true>(itemType, keyColumnType) == ECompareOptions::Uncomparable) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() + << "Invalid column type in table lookup, column: " << keyColumnName + << ", expected: " << FormatType(keyColumnType) + << ", actual: " << FormatType(itemType))); + return false; + } + + ++keyColumnsCount; + } + return true; +} + TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster, const TKikimrTablesData& tablesData, bool withSystemColumns) { - const bool readIndex = TKqlReadTableIndex::Match(node.Get()); - if (readIndex && !EnsureArgsCount(*node, 5, ctx)) { + const bool readIndex = TKqlReadTableIndex::Match(node.Get()); + if (readIndex && !EnsureArgsCount(*node, 5, ctx)) { return TStatus::Error; } if (!readIndex && !EnsureArgsCount(*node, 4, ctx)) { - return TStatus::Error; - } - + return TStatus::Error; + } + auto table = ResolveTable(node->Child(TKqlReadTableBase::idx_Table), ctx, cluster, tablesData); if (!table.second) { return TStatus::Error; } - YQL_ENSURE(table.second->Metadata, "Expected loaded metadata"); - - TKikimrTableMetadataPtr meta; - - if (readIndex) { - meta = table.second->Metadata->GetIndexMetadata(TString(node->Child(TKqlReadTableIndex::idx_Index)->Content())).first; - if (!meta) { - return TStatus::Error; - } - } else { - meta = table.second->Metadata; - } - + YQL_ENSURE(table.second->Metadata, "Expected loaded metadata"); + + TKikimrTableMetadataPtr meta; + + if (readIndex) { + meta = table.second->Metadata->GetIndexMetadata(TString(node->Child(TKqlReadTableIndex::idx_Index)->Content())).first; + if (!meta) { + return TStatus::Error; + } + } else { + meta = table.second->Metadata; + } + const auto& columns = node->ChildPtr(TKqlReadTableBase::idx_Columns); if (!EnsureTupleOfAtoms(*columns, ctx)) { return TStatus::Error; @@ -213,15 +213,15 @@ TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const TKqlKeyRange range{node->ChildPtr(TKqlReadTableBase::idx_Range)}; - if (!CheckKeyTuple(range.From(), *table.second, meta, ctx)) { + if (!CheckKeyTuple(range.From(), *table.second, meta, ctx)) { return TStatus::Error; } - if (!CheckKeyTuple(range.To(), *table.second, meta, ctx)) { + if (!CheckKeyTuple(range.To(), *table.second, meta, ctx)) { return TStatus::Error; } - if (TKqlReadTable::Match(node.Get()) || TKqlReadTableIndex::Match(node.Get())) { + if (TKqlReadTable::Match(node.Get()) || TKqlReadTableIndex::Match(node.Get())) { node->SetTypeAnn(ctx.MakeType<TListExprType>(rowType)); } else if (TKqpReadTable::Match(node.Get())) { node->SetTypeAnn(ctx.MakeType<TFlowExprType>(rowType)); @@ -320,7 +320,7 @@ TStatus AnnotateReadTableRanges(const TExprNode::TPtr& node, TExprContext& ctx, TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster, const TKikimrTablesData& tablesData, bool withSystemColumns) { - if (!EnsureArgsCount(*node, TKqlLookupIndex::Match(node.Get()) ? 4 : 3, ctx)) { + if (!EnsureArgsCount(*node, TKqlLookupIndex::Match(node.Get()) ? 4 : 3, ctx)) { return TStatus::Error; } @@ -368,19 +368,19 @@ TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, cons auto structType = lookupType->Cast<TStructExprType>(); ui32 keyColumnsCount = 0; - if (TKqlLookupIndex::Match(node.Get())) { - auto index = node->Child(TKqlLookupIndex::idx_Index); - if (!EnsureAtom(*index, ctx)) { - return TStatus::Error; + if (TKqlLookupIndex::Match(node.Get())) { + auto index = node->Child(TKqlLookupIndex::idx_Index); + if (!EnsureAtom(*index, ctx)) { + return TStatus::Error; } - auto indexMeta = table.second->Metadata->GetIndexMetadata(TString(index->Content())).first; + auto indexMeta = table.second->Metadata->GetIndexMetadata(TString(index->Content())).first; - if (!CalcKeyColumnsCount(ctx, node->Pos(), *structType, *table.second, *indexMeta, keyColumnsCount)) { - return TStatus::Error; - } + if (!CalcKeyColumnsCount(ctx, node->Pos(), *structType, *table.second, *indexMeta, keyColumnsCount)) { + return TStatus::Error; + } - } else { - if (!CalcKeyColumnsCount(ctx, node->Pos(), *structType, *table.second, *table.second->Metadata, keyColumnsCount)) { + } else { + if (!CalcKeyColumnsCount(ctx, node->Pos(), *structType, *table.second, *table.second->Metadata, keyColumnsCount)) { return TStatus::Error; } } @@ -496,9 +496,9 @@ TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const } if (TKqlUpsertRowsIndex::Match(node.Get())) { - Y_ENSURE(!table.second->Metadata->SecondaryGlobalIndexMetadata.empty()); - } - + Y_ENSURE(!table.second->Metadata->SecondaryGlobalIndexMetadata.empty()); + } + auto effectType = MakeKqpEffectType(ctx); if (isStream) { node->SetTypeAnn(ctx.MakeType<TStreamExprType>(effectType)); @@ -661,7 +661,7 @@ TStatus AnnotateDeleteRows(const TExprNode::TPtr& node, TExprContext& ctx, const itemType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); isStream = true; } else { - YQL_ENSURE(TKqlDeleteRows::Match(node.Get()) || TKqlDeleteRowsIndex::Match(node.Get())); + YQL_ENSURE(TKqlDeleteRows::Match(node.Get()) || TKqlDeleteRowsIndex::Match(node.Get())); if (!EnsureListType(*input, ctx)) { return TStatus::Error; } @@ -1066,11 +1066,11 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl return AnnotateUpsertRows(input, ctx, cluster, *tablesData); } - if (TKqlInsertRowsBase::Match(input.Get())) { + if (TKqlInsertRowsBase::Match(input.Get())) { return AnnotateInsertRows(input, ctx, cluster, *tablesData); } - if (TKqlUpdateRowsBase::Match(input.Get())) { + if (TKqlUpdateRowsBase::Match(input.Get())) { return AnnotateUpdateRows(input, ctx, cluster, *tablesData); } diff --git a/ydb/core/kqp/provider/kqp_opt_helpers.cpp b/ydb/core/kqp/provider/kqp_opt_helpers.cpp index 0874077169..0f9d77214f 100644 --- a/ydb/core/kqp/provider/kqp_opt_helpers.cpp +++ b/ydb/core/kqp/provider/kqp_opt_helpers.cpp @@ -1,241 +1,241 @@ -#include "kqp_opt_helpers.h" - +#include "kqp_opt_helpers.h" + #include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYql; -using namespace NYql::NNodes; - -TExprBase ExtractKeys(TCoArgument itemArg, const TKikimrTableDescription& tableDesc, - TExprContext& ctx) -{ - TVector<TExprBase> keys; - for (TString& keyColumnName : tableDesc.Metadata->KeyColumnNames) { - auto key = Build<TCoMember>(ctx, itemArg.Pos()) - .Struct(itemArg) - .Name().Build(keyColumnName) - .Done(); - - keys.emplace_back(std::move(key)); - } - - if (keys.size() == 1) { - return keys[0]; - } - - return Build<TExprList>(ctx, itemArg.Pos()) - .Add(keys) - .Done(); -} - -TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> BuildSecondaryIndexVector( - const TKikimrTableDescription& table, - TPositionHandle pos, - TExprContext& ctx, - const THashSet<TStringBuf>* filter, - const std::function<TExprBase (const TKikimrTableMetadata&, TPositionHandle, TExprContext&)>& tableBuilder) -{ - TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> secondaryIndexes; - secondaryIndexes.reserve(table.Metadata->Indexes.size()); - YQL_ENSURE(table.Metadata->Indexes.size() == table.Metadata->SecondaryGlobalIndexMetadata.size()); - for (size_t i = 0; i < table.Metadata->Indexes.size(); i++) { - const auto& indexMeta = table.Metadata->Indexes[i]; - - if (!indexMeta.ItUsedForWrite()) { - continue; - } - - // Add index if filter absent - bool addIndex = filter ? false : true; - - for (const auto& col : indexMeta.KeyColumns) { - - if (filter) { - // Add index if filter and at least one column present in the filter - addIndex |= filter->contains(TStringBuf(col)); - } - } - - for (const auto& col : indexMeta.DataColumns) { - - if (filter) { - // Add index if filter and at least one column present in the filter - addIndex |= filter->contains(TStringBuf(col)); - } - } - - if (indexMeta.KeyColumns && addIndex) { - auto indexTable = tableBuilder(*table.Metadata->SecondaryGlobalIndexMetadata[i], pos, ctx).Ptr(); - secondaryIndexes.emplace_back(std::make_pair(indexTable, &indexMeta)); - } - } - return secondaryIndexes; -} - -TVector<TExprBase> CreateColumnsToSelectToUpdateIndex( - const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> indexes, - const TVector<TString>& pk, - const THashSet<TString>& dataColumns, - TPositionHandle pos, - TExprContext& ctx) -{ - TVector<TExprBase> columnsToSelect; - TSet<TString> columns; - - for (const auto& pair : indexes) { - for (const auto& col : pair.second->KeyColumns) { - if (columns.insert(col).second) { - auto atom = Build<TCoAtom>(ctx, pos) - .Value(col) - .Done(); - columnsToSelect.emplace_back(std::move(atom)); - } - } - - for (const auto& col : dataColumns) { - if (columns.insert(col).second) { - auto atom = Build<TCoAtom>(ctx, pos) - .Value(col) - .Done(); - columnsToSelect.emplace_back(std::move(atom)); - } - } - } - - for (const auto& p : pk) { - const auto& atom = Build<TCoAtom>(ctx, pos) - .Value(p) - .Done(); - if (columns.insert(p).second) { - columnsToSelect.push_back(atom); - } - } - - return columnsToSelect; -} - -// Return set of data columns need to be save during index update -THashSet<TString> CreateDataColumnSetToRead( - const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>>& indexes, - const THashSet<TStringBuf>& inputColumns) -{ - THashSet<TString> res; - - for (const auto& index : indexes) { - for (const auto& col : index.second->DataColumns) { - if (!inputColumns.contains(col)) { - res.emplace(col); - } - } - } - - return res; -} - -TExprBase RemoveDuplicateKeyFromInput(const TExprBase& input, const TKikimrTableDescription& tableDesc, - TPositionHandle pos, TExprContext& ctx) -{ - const auto& keySelectorArg = Build<TCoArgument>(ctx, pos) - .Name("item") - .Done(); - - const auto& streamArg = Build<TCoArgument>(ctx, pos) - .Name("streamArg") - .Done(); - - return Build<TCoPartitionByKey>(ctx, pos) - .Input(input) - .KeySelectorLambda() - .Args(keySelectorArg) - .Body(ExtractKeys(keySelectorArg, tableDesc, ctx)) - .Build() - .SortDirections<TCoVoid>().Build() - .SortKeySelectorLambda<TCoVoid>().Build() - .ListHandlerLambda() - .Args({TStringBuf("stream")}) - .Body<TCoFlatMap>() - .Input(TStringBuf("stream")) - .Lambda<TCoLambda>() - .Args(streamArg) - .Body<TCoLast>() - .Input<TCoForwardList>() - .Stream<TCoNth>() - .Tuple(streamArg) - .Index().Value(ToString(1)) - .Build() - .Build() - .Build() - .Build() - .Build() - .Build() - .Build() - .Done(); -} - -// Replace absent input columns to NULL to perform REPLACE via UPSERT -std::pair<TExprBase, TCoAtomList> CreateRowsToReplace(const TExprBase& input, - const TCoAtomList& inputColumns, const TKikimrTableDescription& tableDesc, - TPositionHandle pos, TExprContext& ctx) -{ - THashSet<TStringBuf> inputColumnsSet; - for (const auto& name : inputColumns) { - inputColumnsSet.insert(name.Value()); - } - - auto rowArg = Build<TCoArgument>(ctx, pos) - .Name("row") - .Done(); - - TVector<TCoAtom> writeColumns; - TVector<TExprBase> writeMembers; - - for (const auto& [name, _] : tableDesc.Metadata->Columns) { - TMaybeNode<TExprBase> memberValue; - if (tableDesc.GetKeyColumnIndex(name) || inputColumnsSet.contains(name)) { - memberValue = Build<TCoMember>(ctx, pos) - .Struct(rowArg) - .Name().Build(name) - .Done(); - } else { - auto type = tableDesc.GetColumnType(name); - YQL_ENSURE(type); - - memberValue = Build<TCoNothing>(ctx, pos) - .OptionalType(NCommon::BuildTypeExpr(pos, *type, ctx)) - .Done(); - } - - auto nameAtom = TCoAtom(ctx.NewAtom(pos, name)); - - YQL_ENSURE(memberValue); - auto memberTuple = Build<TCoNameValueTuple>(ctx, pos) - .Name(nameAtom) - .Value(memberValue.Cast()) - .Done(); - - writeColumns.emplace_back(std::move(nameAtom)); - writeMembers.emplace_back(std::move(memberTuple)); - } - - auto writeData = Build<TCoMap>(ctx, pos) - .Input(input) - .Lambda() - .Args({rowArg}) - .Body<TCoAsStruct>() - .Add(writeMembers) - .Build() - .Build() - .Done(); - - auto columnList = Build<TCoAtomList>(ctx, pos) - .Add(writeColumns) - .Done(); - - return {writeData, columnList}; -} - -} // namespace NKqp -} // namespace NKikimr - + +namespace NKikimr { +namespace NKqp { + +using namespace NYql; +using namespace NYql::NNodes; + +TExprBase ExtractKeys(TCoArgument itemArg, const TKikimrTableDescription& tableDesc, + TExprContext& ctx) +{ + TVector<TExprBase> keys; + for (TString& keyColumnName : tableDesc.Metadata->KeyColumnNames) { + auto key = Build<TCoMember>(ctx, itemArg.Pos()) + .Struct(itemArg) + .Name().Build(keyColumnName) + .Done(); + + keys.emplace_back(std::move(key)); + } + + if (keys.size() == 1) { + return keys[0]; + } + + return Build<TExprList>(ctx, itemArg.Pos()) + .Add(keys) + .Done(); +} + +TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> BuildSecondaryIndexVector( + const TKikimrTableDescription& table, + TPositionHandle pos, + TExprContext& ctx, + const THashSet<TStringBuf>* filter, + const std::function<TExprBase (const TKikimrTableMetadata&, TPositionHandle, TExprContext&)>& tableBuilder) +{ + TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> secondaryIndexes; + secondaryIndexes.reserve(table.Metadata->Indexes.size()); + YQL_ENSURE(table.Metadata->Indexes.size() == table.Metadata->SecondaryGlobalIndexMetadata.size()); + for (size_t i = 0; i < table.Metadata->Indexes.size(); i++) { + const auto& indexMeta = table.Metadata->Indexes[i]; + + if (!indexMeta.ItUsedForWrite()) { + continue; + } + + // Add index if filter absent + bool addIndex = filter ? false : true; + + for (const auto& col : indexMeta.KeyColumns) { + + if (filter) { + // Add index if filter and at least one column present in the filter + addIndex |= filter->contains(TStringBuf(col)); + } + } + + for (const auto& col : indexMeta.DataColumns) { + + if (filter) { + // Add index if filter and at least one column present in the filter + addIndex |= filter->contains(TStringBuf(col)); + } + } + + if (indexMeta.KeyColumns && addIndex) { + auto indexTable = tableBuilder(*table.Metadata->SecondaryGlobalIndexMetadata[i], pos, ctx).Ptr(); + secondaryIndexes.emplace_back(std::make_pair(indexTable, &indexMeta)); + } + } + return secondaryIndexes; +} + +TVector<TExprBase> CreateColumnsToSelectToUpdateIndex( + const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> indexes, + const TVector<TString>& pk, + const THashSet<TString>& dataColumns, + TPositionHandle pos, + TExprContext& ctx) +{ + TVector<TExprBase> columnsToSelect; + TSet<TString> columns; + + for (const auto& pair : indexes) { + for (const auto& col : pair.second->KeyColumns) { + if (columns.insert(col).second) { + auto atom = Build<TCoAtom>(ctx, pos) + .Value(col) + .Done(); + columnsToSelect.emplace_back(std::move(atom)); + } + } + + for (const auto& col : dataColumns) { + if (columns.insert(col).second) { + auto atom = Build<TCoAtom>(ctx, pos) + .Value(col) + .Done(); + columnsToSelect.emplace_back(std::move(atom)); + } + } + } + + for (const auto& p : pk) { + const auto& atom = Build<TCoAtom>(ctx, pos) + .Value(p) + .Done(); + if (columns.insert(p).second) { + columnsToSelect.push_back(atom); + } + } + + return columnsToSelect; +} + +// Return set of data columns need to be save during index update +THashSet<TString> CreateDataColumnSetToRead( + const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>>& indexes, + const THashSet<TStringBuf>& inputColumns) +{ + THashSet<TString> res; + + for (const auto& index : indexes) { + for (const auto& col : index.second->DataColumns) { + if (!inputColumns.contains(col)) { + res.emplace(col); + } + } + } + + return res; +} + +TExprBase RemoveDuplicateKeyFromInput(const TExprBase& input, const TKikimrTableDescription& tableDesc, + TPositionHandle pos, TExprContext& ctx) +{ + const auto& keySelectorArg = Build<TCoArgument>(ctx, pos) + .Name("item") + .Done(); + + const auto& streamArg = Build<TCoArgument>(ctx, pos) + .Name("streamArg") + .Done(); + + return Build<TCoPartitionByKey>(ctx, pos) + .Input(input) + .KeySelectorLambda() + .Args(keySelectorArg) + .Body(ExtractKeys(keySelectorArg, tableDesc, ctx)) + .Build() + .SortDirections<TCoVoid>().Build() + .SortKeySelectorLambda<TCoVoid>().Build() + .ListHandlerLambda() + .Args({TStringBuf("stream")}) + .Body<TCoFlatMap>() + .Input(TStringBuf("stream")) + .Lambda<TCoLambda>() + .Args(streamArg) + .Body<TCoLast>() + .Input<TCoForwardList>() + .Stream<TCoNth>() + .Tuple(streamArg) + .Index().Value(ToString(1)) + .Build() + .Build() + .Build() + .Build() + .Build() + .Build() + .Build() + .Done(); +} + +// Replace absent input columns to NULL to perform REPLACE via UPSERT +std::pair<TExprBase, TCoAtomList> CreateRowsToReplace(const TExprBase& input, + const TCoAtomList& inputColumns, const TKikimrTableDescription& tableDesc, + TPositionHandle pos, TExprContext& ctx) +{ + THashSet<TStringBuf> inputColumnsSet; + for (const auto& name : inputColumns) { + inputColumnsSet.insert(name.Value()); + } + + auto rowArg = Build<TCoArgument>(ctx, pos) + .Name("row") + .Done(); + + TVector<TCoAtom> writeColumns; + TVector<TExprBase> writeMembers; + + for (const auto& [name, _] : tableDesc.Metadata->Columns) { + TMaybeNode<TExprBase> memberValue; + if (tableDesc.GetKeyColumnIndex(name) || inputColumnsSet.contains(name)) { + memberValue = Build<TCoMember>(ctx, pos) + .Struct(rowArg) + .Name().Build(name) + .Done(); + } else { + auto type = tableDesc.GetColumnType(name); + YQL_ENSURE(type); + + memberValue = Build<TCoNothing>(ctx, pos) + .OptionalType(NCommon::BuildTypeExpr(pos, *type, ctx)) + .Done(); + } + + auto nameAtom = TCoAtom(ctx.NewAtom(pos, name)); + + YQL_ENSURE(memberValue); + auto memberTuple = Build<TCoNameValueTuple>(ctx, pos) + .Name(nameAtom) + .Value(memberValue.Cast()) + .Done(); + + writeColumns.emplace_back(std::move(nameAtom)); + writeMembers.emplace_back(std::move(memberTuple)); + } + + auto writeData = Build<TCoMap>(ctx, pos) + .Input(input) + .Lambda() + .Args({rowArg}) + .Body<TCoAsStruct>() + .Add(writeMembers) + .Build() + .Build() + .Done(); + + auto columnList = Build<TCoAtomList>(ctx, pos) + .Add(writeColumns) + .Done(); + + return {writeData, columnList}; +} + +} // namespace NKqp +} // namespace NKikimr + diff --git a/ydb/core/kqp/provider/kqp_opt_helpers.h b/ydb/core/kqp/provider/kqp_opt_helpers.h index 6d28f5f3a5..db7507cf55 100644 --- a/ydb/core/kqp/provider/kqp_opt_helpers.h +++ b/ydb/core/kqp/provider/kqp_opt_helpers.h @@ -1,45 +1,45 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/provider/yql_kikimr_gateway.h> - -// Common opt helpers for "Old" and "New" engine. -// Should be in core/kqp/opt directory, but also use by Old engine from core/kqp/provider. - -namespace NYql { - class TKikimrTableDescription; -} - -namespace NKikimr { -namespace NKqp { - -NYql::NNodes::TExprBase ExtractKeys(NYql::NNodes::TCoArgument itemArg, const NYql::TKikimrTableDescription& tableDesc, - NYql::TExprContext& ctx); - -TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>> BuildSecondaryIndexVector( - const NYql::TKikimrTableDescription& table, - NYql::TPositionHandle pos, - NYql::TExprContext& ctx, - const THashSet<TStringBuf>* filter, - const std::function<NYql::NNodes::TExprBase (const NYql::TKikimrTableMetadata&, NYql::TPositionHandle, NYql::TExprContext&)>& tableBuilder); - -TVector<NYql::NNodes::TExprBase> CreateColumnsToSelectToUpdateIndex( - const TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>> indexes, - const TVector<TString>& pk, - const THashSet<TString>& dataColumns, - NYql::TPositionHandle pos, - NYql::TExprContext& ctx); - -THashSet<TString> CreateDataColumnSetToRead( - const TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>>& indexes, - const THashSet<TStringBuf>& inputColumns); - -NYql::NNodes::TExprBase RemoveDuplicateKeyFromInput( - const NYql::NNodes::TExprBase& input, const NYql::TKikimrTableDescription& tableDesc, - NYql::TPositionHandle pos, NYql::TExprContext& ctx); - -std::pair<NYql::NNodes::TExprBase, NYql::NNodes::TCoAtomList> CreateRowsToReplace(const NYql::NNodes::TExprBase& input, - const NYql::NNodes::TCoAtomList& inputColumns, const NYql::TKikimrTableDescription& tableDesc, - NYql::TPositionHandle pos, NYql::TExprContext& ctx); - -} // namespace NKqp -} // namespace NKikimr + +// Common opt helpers for "Old" and "New" engine. +// Should be in core/kqp/opt directory, but also use by Old engine from core/kqp/provider. + +namespace NYql { + class TKikimrTableDescription; +} + +namespace NKikimr { +namespace NKqp { + +NYql::NNodes::TExprBase ExtractKeys(NYql::NNodes::TCoArgument itemArg, const NYql::TKikimrTableDescription& tableDesc, + NYql::TExprContext& ctx); + +TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>> BuildSecondaryIndexVector( + const NYql::TKikimrTableDescription& table, + NYql::TPositionHandle pos, + NYql::TExprContext& ctx, + const THashSet<TStringBuf>* filter, + const std::function<NYql::NNodes::TExprBase (const NYql::TKikimrTableMetadata&, NYql::TPositionHandle, NYql::TExprContext&)>& tableBuilder); + +TVector<NYql::NNodes::TExprBase> CreateColumnsToSelectToUpdateIndex( + const TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>> indexes, + const TVector<TString>& pk, + const THashSet<TString>& dataColumns, + NYql::TPositionHandle pos, + NYql::TExprContext& ctx); + +THashSet<TString> CreateDataColumnSetToRead( + const TVector<std::pair<NYql::TExprNode::TPtr, const NYql::TIndexDescription*>>& indexes, + const THashSet<TStringBuf>& inputColumns); + +NYql::NNodes::TExprBase RemoveDuplicateKeyFromInput( + const NYql::NNodes::TExprBase& input, const NYql::TKikimrTableDescription& tableDesc, + NYql::TPositionHandle pos, NYql::TExprContext& ctx); + +std::pair<NYql::NNodes::TExprBase, NYql::NNodes::TCoAtomList> CreateRowsToReplace(const NYql::NNodes::TExprBase& input, + const NYql::NNodes::TCoAtomList& inputColumns, const NYql::TKikimrTableDescription& tableDesc, + NYql::TPositionHandle pos, NYql::TExprContext& ctx); + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/provider/mkql/ya.make b/ydb/core/kqp/provider/mkql/ya.make index bf80cc8c22..021b843562 100644 --- a/ydb/core/kqp/provider/mkql/ya.make +++ b/ydb/core/kqp/provider/mkql/ya.make @@ -1,33 +1,33 @@ -LIBRARY() - -OWNER(spuchin) - -SRCS( -) - -PEERDIR( -) - -SRCDIR( +LIBRARY() + +OWNER(spuchin) + +SRCS( +) + +PEERDIR( +) + +SRCDIR( ydb/library/yql/core/expr_nodes_gen -) - -RUN_PROGRAM( +) + +RUN_PROGRAM( ydb/library/yql/core/expr_nodes_gen/gen - yql_expr_nodes_gen.jnj - yql_kikimr_mkql_expr_nodes.json - yql_kikimr_mkql_expr_nodes.gen.h - yql_kikimr_mkql_expr_nodes.decl.inl.h - yql_kikimr_mkql_expr_nodes.defs.inl.h - IN yql_expr_nodes_gen.jnj - IN yql_kikimr_mkql_expr_nodes.json - OUT yql_kikimr_mkql_expr_nodes.gen.h - OUT yql_kikimr_mkql_expr_nodes.decl.inl.h - OUT yql_kikimr_mkql_expr_nodes.defs.inl.h - OUTPUT_INCLUDES + yql_expr_nodes_gen.jnj + yql_kikimr_mkql_expr_nodes.json + yql_kikimr_mkql_expr_nodes.gen.h + yql_kikimr_mkql_expr_nodes.decl.inl.h + yql_kikimr_mkql_expr_nodes.defs.inl.h + IN yql_expr_nodes_gen.jnj + IN yql_kikimr_mkql_expr_nodes.json + OUT yql_kikimr_mkql_expr_nodes.gen.h + OUT yql_kikimr_mkql_expr_nodes.decl.inl.h + OUT yql_kikimr_mkql_expr_nodes.defs.inl.h + OUTPUT_INCLUDES ${ARCADIA_ROOT}/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h - ${ARCADIA_ROOT}/util/generic/hash_set.h -) - -END() - + ${ARCADIA_ROOT}/util/generic/hash_set.h +) + +END() + diff --git a/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.h b/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.h index 3fb4cd589b..83c1637d19 100644 --- a/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.h +++ b/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.h @@ -1,15 +1,15 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.gen.h> - + #include <ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.gen.h> - -namespace NYql { -namespace NNodes { - + +namespace NYql { +namespace NNodes { + #include <ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.decl.inl.h> - + #include <ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.defs.inl.h> - -} // namespace NNodes -} // namespace NYql + +} // namespace NNodes +} // namespace NYql diff --git a/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.json b/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.json index 52a62a2ba5..1051c124b8 100644 --- a/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.json +++ b/ydb/core/kqp/provider/mkql/yql_kikimr_mkql_expr_nodes.json @@ -1,38 +1,38 @@ -{ - "NodeRootType": "TExprBase", - "NodeBuilderBase": "TNodeBuilderBase", - "ListBuilderBase": "TListBuilderBase", - "FreeArgCallableBase": "TFreeArgCallable", - "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", - "Nodes": [ - { - "Name": "TMkqlVersionedTable", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 1, "Name": "SchemaVersion", "Type": "TCoAtom"}, - {"Index": 2, "Name": "PathId", "Type": "TCoAtom"} - ] - }, - { - "Name": "TMkqlUpdateRow", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "UpdateRow"}, - "Children": [ - {"Index": 0, "Name": "Table", "Type": "TMkqlVersionedTable"}, - {"Index": 1, "Name": "Key", "Type": "TCoNameValueTupleList"}, - {"Index": 2, "Name": "Update", "Type": "TCoNameValueTupleList"} - ] - }, - { - "Name": "TMkqlEraseRow", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "EraseRow"}, - "Children": [ - {"Index": 0, "Name": "Table", "Type": "TMkqlVersionedTable"}, - {"Index": 1, "Name": "Key", "Type": "TCoNameValueTupleList"} - ] - } - ] -} +{ + "NodeRootType": "TExprBase", + "NodeBuilderBase": "TNodeBuilderBase", + "ListBuilderBase": "TListBuilderBase", + "FreeArgCallableBase": "TFreeArgCallable", + "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", + "Nodes": [ + { + "Name": "TMkqlVersionedTable", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 1, "Name": "SchemaVersion", "Type": "TCoAtom"}, + {"Index": 2, "Name": "PathId", "Type": "TCoAtom"} + ] + }, + { + "Name": "TMkqlUpdateRow", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "UpdateRow"}, + "Children": [ + {"Index": 0, "Name": "Table", "Type": "TMkqlVersionedTable"}, + {"Index": 1, "Name": "Key", "Type": "TCoNameValueTupleList"}, + {"Index": 2, "Name": "Update", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TMkqlEraseRow", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "EraseRow"}, + "Children": [ + {"Index": 0, "Name": "Table", "Type": "TMkqlVersionedTable"}, + {"Index": 1, "Name": "Key", "Type": "TCoNameValueTupleList"} + ] + } + ] +} diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 59f896f262..035d6b7701 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -64,16 +64,16 @@ private: return TStatus::Ok; } - TStatus HandleAlterTable(TKiAlterTable node, TExprContext& ctx) override { - Y_UNUSED(ctx); - - auto cluster = node.DataSink().Cluster(); - auto table = node.Table(); - + TStatus HandleAlterTable(TKiAlterTable node, TExprContext& ctx) override { + Y_UNUSED(ctx); + + auto cluster = node.DataSink().Cluster(); + auto table = node.Table(); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; - } - + } + TStatus HandleDropTable(TKiDropTable node, TExprContext& ctx) override { Y_UNUSED(ctx); @@ -190,22 +190,22 @@ private: } auto mode = settings.Mode.Cast(); - if (mode == "create") { + if (mode == "create") { if (!settings.Columns) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "No columns provided for create mode.")); - return TStatus::Error; - } - + << "No columns provided for create mode.")); + return TStatus::Error; + } + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; - } else if (mode == "alter") { + } else if (mode == "alter") { if (!settings.AlterActions) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "No actions provided for alter mode.")); - return TStatus::Error; - } - + return TStatus::Error; + } + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } @@ -235,9 +235,9 @@ private: TStatus HandleDataQuery(TKiDataQuery node, TExprContext& ctx) override { Y_UNUSED(ctx); - for (const auto& op : node.Operations()) { + for (const auto& op : node.Operations()) { SessionCtx->Tables().GetOrAddTable(TString(op.Cluster()), SessionCtx->GetDatabase(), TString(op.Table())); - } + } return TStatus::Ok; } @@ -275,7 +275,7 @@ private: } if (auto call = node.Maybe<TKiUpdateRow>()) { - + auto cluster = call.Cast().Cluster().Value(); auto table = call.Cast().Table(); @@ -422,9 +422,9 @@ public: return true; } - if (node.IsCallable(TKiCreateTable::CallableName()) - || node.IsCallable(TKiDropTable::CallableName()) - || node.IsCallable(TKiAlterTable::CallableName())) { + if (node.IsCallable(TKiCreateTable::CallableName()) + || node.IsCallable(TKiDropTable::CallableName()) + || node.IsCallable(TKiAlterTable::CallableName())) { return true; } @@ -534,54 +534,54 @@ public: NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); YQL_ENSURE(settings.Mode); auto mode = settings.Mode.Cast(); - if (mode == "create") { + if (mode == "create") { YQL_ENSURE(settings.Columns); YQL_ENSURE(!settings.Columns.Cast().Empty()); if (!settings.PrimaryKey) { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Primary key is required for ydb tables.")); - return nullptr; - } - + return nullptr; + } + if (!settings.PartitionBy.IsValid()) { settings.PartitionBy = Build<TCoAtomList>(ctx, node->Pos()).Done(); } - return Build<TKiCreateTable>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .Table().Build(key.GetTablePath()) + return Build<TKiCreateTable>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .Table().Build(key.GetTablePath()) .Columns(settings.Columns.Cast()) .PrimaryKey(settings.PrimaryKey.Cast()) .Settings(settings.Other) - .Indexes(settings.Indexes.Cast()) + .Indexes(settings.Indexes.Cast()) .Changefeeds(settings.Changefeeds.Cast()) .PartitionBy(settings.PartitionBy.Cast()) .ColumnFamilies(settings.ColumnFamilies.Cast()) .TableSettings(settings.TableSettings.Cast()) - .Done() - .Ptr(); - } else if (mode == "alter") { + .Done() + .Ptr(); + } else if (mode == "alter") { for (auto setting : settings.Other) { if (setting.Name().Value() == "intent") { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Old AST format for AlterTable")); return nullptr; - } - } + } + } YQL_ENSURE(settings.AlterActions); YQL_ENSURE(!settings.AlterActions.Cast().Empty()); - return Build<TKiAlterTable>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .Table().Build(key.GetTablePath()) + return Build<TKiAlterTable>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .Table().Build(key.GetTablePath()) .Actions(settings.AlterActions.Cast()) - .Done() - .Ptr(); - } else { + .Done() + .Ptr(); + } else { YQL_ENSURE(false, "unknown TableScheme mode \"" << TString(mode) << "\""); - } + } } case TKikimrKey::Type::TableList: @@ -718,10 +718,10 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt return HandleCreateTable(node.Cast(), ctx); } - if (auto node = TMaybeNode<TKiAlterTable>(input)) { - return HandleAlterTable(node.Cast(), ctx); - } - + if (auto node = TMaybeNode<TKiAlterTable>(input)) { + return HandleAlterTable(node.Cast(), ctx); + } + if (auto node = TMaybeNode<TKiDropTable>(input)) { return HandleDropTable(node.Cast(), ctx); } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index a3a1b8d99e..65ef7eb932 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -110,32 +110,32 @@ public: std::make_shared<IKikimrGateway::TTableMetadataResult>()); YQL_ENSURE(emplaceResult.second); - auto queryType = SessionCtx->Query().Type; + auto queryType = SessionCtx->Query().Type; auto& result = emplaceResult.first->second; - auto future = Gateway->LoadTableMetadata(clusterName, tableName, - IKikimrGateway::TLoadTableMetadataSettings().WithTableStats(table.GetNeedsStats())); - - futures.push_back(future.Apply([result, queryType] + auto future = Gateway->LoadTableMetadata(clusterName, tableName, + IKikimrGateway::TLoadTableMetadataSettings().WithTableStats(table.GetNeedsStats())); + + futures.push_back(future.Apply([result, queryType] (const NThreading::TFuture<IKikimrGateway::TTableMetadataResult>& future) { YQL_ENSURE(!future.HasException()); - const auto& value = future.GetValue(); - switch (queryType) { + const auto& value = future.GetValue(); + switch (queryType) { case EKikimrQueryType::Unspecified: { - if (value.Metadata) { - if (!value.Metadata->Indexes.empty()) { + if (value.Metadata) { + if (!value.Metadata->Indexes.empty()) { result->AddIssue(TIssue({}, TStringBuilder() - << "Using index tables unsupported for legacy or unspecified request type")); - result->SetStatus(TIssuesIds::KIKIMR_INDEX_METADATA_LOAD_FAILED); - return; - } - } - } - break; - default: - break; - } - *result = value; + << "Using index tables unsupported for legacy or unspecified request type")); + result->SetStatus(TIssuesIds::KIKIMR_INDEX_METADATA_LOAD_FAILED); + return; + } + } + } + break; + default: + break; + } + *result = value; })); } @@ -168,14 +168,14 @@ public: tableDesc.Metadata = res.Metadata; bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); - YQL_ENSURE(res.Metadata->Indexes.size() == res.Metadata->SecondaryGlobalIndexMetadata.size()); - for (const auto& indexMeta : res.Metadata->SecondaryGlobalIndexMetadata) { - YQL_ENSURE(indexMeta); + YQL_ENSURE(res.Metadata->Indexes.size() == res.Metadata->SecondaryGlobalIndexMetadata.size()); + for (const auto& indexMeta : res.Metadata->SecondaryGlobalIndexMetadata) { + YQL_ENSURE(indexMeta); auto& desc = SessionCtx->Tables().GetOrAddTable(indexMeta->Cluster, SessionCtx->GetDatabase(), indexMeta->Name); - desc.Metadata = indexMeta; + desc.Metadata = indexMeta; desc.Load(ctx, sysColumnsEnabled); - } - + } + if (!tableDesc.Load(ctx, sysColumnsEnabled)) { LoadResults.clear(); return TStatus::Error; diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 24b1d8d3f6..ad360fcaeb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -387,10 +387,10 @@ private: auto resultType = executeRightType->Cast<TTupleExprType>()->GetItems()[resultIndex]; YQL_ENSURE(resultIndex < runResult->Results.size()); - auto resultValue = runResult->Results[resultIndex]; - YQL_ENSURE(resultValue); + auto resultValue = runResult->Results[resultIndex]; + YQL_ENSURE(resultValue); - auto resResultNode = GetResOrPullResult(res.Ref(), fillSettings, resultType, *resultValue, ctx); + auto resResultNode = GetResOrPullResult(res.Ref(), fillSettings, resultType, *resultValue, ctx); res.Ptr()->SetResult(std::move(resResultNode)); return SyncOk(); @@ -508,26 +508,26 @@ public: return SyncOk(); } - if (auto maybeAlter = TMaybeNode<TKiAlterTable>(input)) { + if (auto maybeAlter = TMaybeNode<TKiAlterTable>(input)) { if (!EnsureNotPrepare("ALTER TABLE", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + auto cluster = TString(maybeAlter.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeAlter.Cast().Table())); - + if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::AlterTable, ctx)) { return SyncError(); } - Ydb::Table::AlterTableRequest alterTableRequest; - alterTableRequest.set_path(table.Metadata->Name); - + Ydb::Table::AlterTableRequest alterTableRequest; + alterTableRequest.set_path(table.Metadata->Name); + for (auto action : maybeAlter.Cast().Actions()) { auto name = action.Name().Value(); if (name == "renameTo") { @@ -542,12 +542,12 @@ public: } else if (name == "addColumns") { auto listNode = action.Value().Cast<TExprList>(); for (size_t i = 0; i < listNode.Size(); ++i) { - auto add_column = alterTableRequest.add_add_columns(); + auto add_column = alterTableRequest.add_add_columns(); auto item = listNode.Item(i); auto columnTuple = item.Cast<TExprList>(); auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); - add_column->set_name(TString(columnName)); - + add_column->set_name(TString(columnName)); + auto typeNode = columnTuple.Item(1); auto columnType = typeNode.Ref().GetTypeAnn(); auto type = columnType->Cast<TTypeExprType>()->GetType(); @@ -557,38 +557,38 @@ public: SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull); if (columnTuple.Size() > 2) { auto families = columnTuple.Item(2).Cast<TCoAtomList>(); - if (families.Size() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), - "Unsupported number of families")); - return SyncError(); - } + if (families.Size() > 1) { + ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), + "Unsupported number of families")); + return SyncError(); + } for (auto family : families) { - add_column->set_family(TString(family.Value())); + add_column->set_family(TString(family.Value())); } } } } else if (name == "dropColumns") { auto listNode = action.Value().Cast<TCoAtomList>(); for (auto dropColumn : listNode) { - alterTableRequest.add_drop_columns(TString(dropColumn.Value())); + alterTableRequest.add_drop_columns(TString(dropColumn.Value())); } } else if (name == "alterColumns") { auto listNode = action.Value().Cast<TExprList>(); for (size_t i = 0; i < listNode.Size(); ++i) { - auto alter_columns = alterTableRequest.add_alter_columns(); + auto alter_columns = alterTableRequest.add_alter_columns(); auto item = listNode.Item(i); auto columnTuple = item.Cast<TExprList>(); auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); - alter_columns->set_name(TString(columnName)); - + alter_columns->set_name(TString(columnName)); + auto families = columnTuple.Item(1).Cast<TCoAtomList>(); - if (families.Size() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), - "Unsupported number of families")); - return SyncError(); - } + if (families.Size() > 1) { + ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), + "Unsupported number of families")); + return SyncError(); + } for (auto family : families) { - alter_columns->set_family(TString(family.Value())); + alter_columns->set_family(TString(family.Value())); } } } else if (name == "addColumnFamilies" || name == "alterColumnFamilies") { @@ -596,36 +596,36 @@ public: for (size_t i = 0; i < listNode.Size(); ++i) { auto item = listNode.Item(i); if (auto maybeTupleList = item.Maybe<TCoNameValueTupleList>()) { - auto f = (name == "addColumnFamilies") ? - alterTableRequest.add_add_column_families() : - alterTableRequest.add_alter_column_families(); - + auto f = (name == "addColumnFamilies") ? + alterTableRequest.add_add_column_families() : + alterTableRequest.add_alter_column_families(); + for (auto familySetting : maybeTupleList.Cast()) { auto name = familySetting.Name().Value(); if (name == "name") { - f->set_name(TString(familySetting.Value().Cast<TCoAtom>().Value())); + f->set_name(TString(familySetting.Value().Cast<TCoAtom>().Value())); } else if (name == "data") { - auto data = TString( - familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() - .Value()); - f->mutable_data()->set_media(data); + auto data = TString( + familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() + .Value()); + f->mutable_data()->set_media(data); } else if (name == "compression") { - auto comp = TString( + auto comp = TString( familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() .Value() ); - if (to_lower(comp) == "off") { - f->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); - } else if (to_lower(comp) == "lz4") { - f->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_LZ4); - } else { - auto errText = TStringBuilder() << "Unknown compression '" << comp - << "' for a column family"; - ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), - errText)); - return SyncError(); - } - + if (to_lower(comp) == "off") { + f->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); + } else if (to_lower(comp) == "lz4") { + f->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_LZ4); + } else { + auto errText = TStringBuilder() << "Unknown compression '" << comp + << "' for a column family"; + ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), + errText)); + return SyncError(); + } + } else { ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), TStringBuilder() << "Unknown column family setting name: " << name)); @@ -640,31 +640,31 @@ public: for (const auto& setting : listNode) { auto name = setting.Name().Value(); if (name == "compactionPolicy") { - alterTableRequest.set_set_compaction_policy(TString( + alterTableRequest.set_set_compaction_policy(TString( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - )); + )); } else if (name == "autoPartitioningBySize") { - auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); - auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); - if (val == "enabled") { - partitioningSettings->set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); - } else if (val == "disabled") { - partitioningSettings->set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); - } else { - auto errText = TStringBuilder() << "Unknown feature flag '" - << val - << "' for auto partitioning by size"; - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), - errText)); - return SyncError(); - } + auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); + auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); + if (val == "enabled") { + partitioningSettings->set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); + } else if (val == "disabled") { + partitioningSettings->set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); + } else { + auto errText = TStringBuilder() << "Unknown feature flag '" + << val + << "' for auto partitioning by size"; + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), + errText)); + return SyncError(); + } } else if (name == "partitionSizeMb") { ui64 value = FromString<ui64>( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() ); if (value) { - auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); - partitioningSettings->set_partition_size_mb(value); + auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); + partitioningSettings->set_partition_size_mb(value); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), "Can't set preferred partition size to 0. " @@ -672,27 +672,27 @@ public: return SyncError(); } } else if (name == "autoPartitioningByLoad") { - auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); - TString val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); - if (val == "enabled") { - partitioningSettings->set_partitioning_by_load(Ydb::FeatureFlag::ENABLED); - } else if (val == "disabled") { - partitioningSettings->set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); - } else { - auto errText = TStringBuilder() << "Unknown feature flag '" - << val - << "' for auto partitioning by load"; - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), - errText)); - return SyncError(); - } + auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); + TString val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); + if (val == "enabled") { + partitioningSettings->set_partitioning_by_load(Ydb::FeatureFlag::ENABLED); + } else if (val == "disabled") { + partitioningSettings->set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); + } else { + auto errText = TStringBuilder() << "Unknown feature flag '" + << val + << "' for auto partitioning by load"; + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), + errText)); + return SyncError(); + } } else if (name == "minPartitions") { ui64 value = FromString<ui64>( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() ); if (value) { - auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); - partitioningSettings->set_min_partitions_count(value); + auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); + partitioningSettings->set_min_partitions_count(value); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), "Can't set min partition count to 0")); @@ -703,42 +703,42 @@ public: setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() ); if (value) { - auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); - partitioningSettings->set_max_partitions_count(value); + auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); + partitioningSettings->set_max_partitions_count(value); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), "Can't set max partition count to 0")); return SyncError(); } } else if (name == "keyBloomFilter") { - auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); - if (val == "enabled") { - alterTableRequest.set_set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); - } else if (val == "disabled") { - alterTableRequest.set_set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); - } else { - auto errText = TStringBuilder() << "Unknown feature flag '" - << val - << "' for key bloom filter"; - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), - errText)); - return SyncError(); - } - + auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); + if (val == "enabled") { + alterTableRequest.set_set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); + } else if (val == "disabled") { + alterTableRequest.set_set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); + } else { + auto errText = TStringBuilder() << "Unknown feature flag '" + << val + << "' for key bloom filter"; + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Cast<TCoAtom>().Pos()), + errText)); + return SyncError(); + } + } else if (name == "readReplicasSettings") { - const auto replicasSettings = TString( + const auto replicasSettings = TString( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() ); - Ydb::StatusIds::StatusCode code; - TString errText; - if (!ConvertReadReplicasSettingsToProto(replicasSettings, - *alterTableRequest.mutable_set_read_replicas_settings(), code, errText)) { - - ctx.AddError(YqlIssue(ctx.GetPosition(setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Pos()), - NYql::YqlStatusFromYdbStatus(code), - errText)); - return SyncError(); - } + Ydb::StatusIds::StatusCode code; + TString errText; + if (!ConvertReadReplicasSettingsToProto(replicasSettings, + *alterTableRequest.mutable_set_read_replicas_settings(), code, errText)) { + + ctx.AddError(YqlIssue(ctx.GetPosition(setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Pos()), + NYql::YqlStatusFromYdbStatus(code), + errText)); + return SyncError(); + } } else if (name == "setTtlSettings") { TTtlSettings ttlSettings; TString error; @@ -750,85 +750,85 @@ public: return SyncError(); } - ConvertTtlSettingsToProto(ttlSettings, *alterTableRequest.mutable_set_ttl_settings()); + ConvertTtlSettingsToProto(ttlSettings, *alterTableRequest.mutable_set_ttl_settings()); } else if (name == "resetTtlSettings") { - alterTableRequest.mutable_drop_ttl_settings(); + alterTableRequest.mutable_drop_ttl_settings(); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), TStringBuilder() << "Unknown table profile setting: " << name)); return SyncError(); } } - } else if (name == "addIndex") { - auto listNode = action.Value().Cast<TExprList>(); - auto add_index = alterTableRequest.add_add_indexes(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); - auto name = TString(nameNode.Value()); - if (name == "indexName") { - add_index->set_name(TString(columnTuple.Item(1).Cast<TCoAtom>().Value())); - } else if (name == "indexType") { - const auto type = TString(columnTuple.Item(1).Cast<TCoAtom>().Value()); + } else if (name == "addIndex") { + auto listNode = action.Value().Cast<TExprList>(); + auto add_index = alterTableRequest.add_add_indexes(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto name = TString(nameNode.Value()); + if (name == "indexName") { + add_index->set_name(TString(columnTuple.Item(1).Cast<TCoAtom>().Value())); + } else if (name == "indexType") { + const auto type = TString(columnTuple.Item(1).Cast<TCoAtom>().Value()); if (type == "syncGlobal") { - add_index->mutable_global_index(); + add_index->mutable_global_index(); } else if (type == "asyncGlobal") { add_index->mutable_global_async_index(); - } else { - ctx.AddError(TIssue(ctx.GetPosition(columnTuple.Item(1).Cast<TCoAtom>().Pos()), - TStringBuilder() << "Unknown index type: " << type)); - return SyncError(); - } - } else if (name == "indexColumns") { - auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); - for (auto column : columnList) { - TString columnName(column.Value()); - add_index->add_index_columns(columnName); - } - } else if (name == "dataColumns") { - auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); - for (auto column : columnList) { - TString columnName(column.Value()); - add_index->add_data_columns(columnName); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), - TStringBuilder() << "Unknown add index setting: " << name)); - return SyncError(); - } - } + } else { + ctx.AddError(TIssue(ctx.GetPosition(columnTuple.Item(1).Cast<TCoAtom>().Pos()), + TStringBuilder() << "Unknown index type: " << type)); + return SyncError(); + } + } else if (name == "indexColumns") { + auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); + for (auto column : columnList) { + TString columnName(column.Value()); + add_index->add_index_columns(columnName); + } + } else if (name == "dataColumns") { + auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); + for (auto column : columnList) { + TString columnName(column.Value()); + add_index->add_data_columns(columnName); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), + TStringBuilder() << "Unknown add index setting: " << name)); + return SyncError(); + } + } switch (add_index->type_case()) { case Ydb::Table::TableIndex::kGlobalIndex: - *add_index->mutable_global_index() = Ydb::Table::GlobalIndex(); + *add_index->mutable_global_index() = Ydb::Table::GlobalIndex(); break; case Ydb::Table::TableIndex::kGlobalAsyncIndex: - *add_index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + *add_index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); break; default: YQL_ENSURE(false, "Unknown index type: " << (ui32)add_index->type_case()); } - } else if (name == "dropIndex") { - auto nameNode = action.Value().Cast<TCoAtom>(); - alterTableRequest.add_drop_indexes(TString(nameNode.Value())); + } else if (name == "dropIndex") { + auto nameNode = action.Value().Cast<TCoAtom>(); + alterTableRequest.add_drop_indexes(TString(nameNode.Value())); } else { ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), TStringBuilder() << "Unknown alter table action: " << name)); return SyncError(); - } - } - - auto future = Gateway->AlterTable(std::move(alterTableRequest), cluster); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }); - - } - + } + } + + auto future = Gateway->AlterTable(std::move(alterTableRequest), cluster); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }); + + } + if (auto maybeCreateUser = TMaybeNode<TKiCreateUser>(input)) { if (!EnsureNotPrepare("CREATE USER", input->Pos(), SessionCtx->Query(), ctx)) { return SyncError(); @@ -1099,20 +1099,20 @@ private: auto queryType = SessionCtx->Query().Type; TVector<NKqpProto::TKqpTableInfo> tableInfo; - for (const auto& op : tableOps) { - auto table = op.GetTable(); - const auto& desc = SessionCtx->Tables().GetTable(cluster, table); - YQL_ENSURE(desc.Metadata); + for (const auto& op : tableOps) { + auto table = op.GetTable(); + const auto& desc = SessionCtx->Tables().GetTable(cluster, table); + YQL_ENSURE(desc.Metadata); tableInfo.push_back(NKqpProto::TKqpTableInfo()); - TableDescriptionToTableInfo(desc, &tableInfo.back()); - } - + TableDescriptionToTableInfo(desc, &tableInfo.back()); + } + if (!SessionCtx->HasTx()) { TKikimrTransactionContextBase emptyCtx; - return emptyCtx.ApplyTableOperations(tableOps, tableInfo, isolationLevel, strictDml, queryType, ctx); + return emptyCtx.ApplyTableOperations(tableOps, tableInfo, isolationLevel, strictDml, queryType, ctx); } - return SessionCtx->Tx().ApplyTableOperations(tableOps, tableInfo, isolationLevel, strictDml, queryType, ctx); + return SessionCtx->Tx().ApplyTableOperations(tableOps, tableInfo, isolationLevel, strictDml, queryType, ctx); } bool ApplyDdlOperation(const TString& cluster, TPositionHandle pos, const TString& table, diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp index 996a133b8f..cbe44c3a9d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp @@ -5,7 +5,7 @@ namespace NYql { namespace NNodes { -TString TKiReadTable::GetTable(TExprContext& ctx) const { +TString TKiReadTable::GetTable(TExprContext& ctx) const { TKikimrKey key(ctx); YQL_ENSURE(key.Extract(TableKey().Ref())); YQL_ENSURE(key.GetKeyType() == TKikimrKey::Type::Table); diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h index 73a6b53f7b..e91fd7635a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h @@ -61,7 +61,7 @@ public: explicit TKiReadTable(const TExprNode::TPtr& node) : TKiReadTableStub(node) {} - TString GetTable(TExprContext& ctx) const; + TString GetTable(TExprContext& ctx) const; TCoAtomList GetSelectColumns(TExprContext& ctx, const TKikimrTablesData& tablesData, bool withVirtualColumns = false) const; TCoAtomList GetSelectColumns(TExprContext& ctx, const TKikimrTableDescription& tableData, diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index 31c4a307de..c54d82a2ca 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -26,16 +26,16 @@ ] }, { - "Name": "TKiVersionedTable", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Path", "Type": "TCoAtom"}, - {"Index": 1, "Name": "SchemaVersion", "Type": "TCoAtom"}, - {"Index": 2, "Name": "PathId", "Type": "TCoAtom"} - ] - }, - { + "Name": "TKiVersionedTable", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Path", "Type": "TCoAtom"}, + {"Index": 1, "Name": "SchemaVersion", "Type": "TCoAtom"}, + {"Index": 2, "Name": "PathId", "Type": "TCoAtom"} + ] + }, + { "Name": "TKiReadBase", "Base": "TCallable", "Match": {"Type": "CallableBase"}, @@ -112,7 +112,7 @@ {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, {"Index": 3, "Name": "Columns", "Type": "TExprList"}, {"Index": 4, "Name": "PrimaryKey", "Type": "TCoAtomList"}, - {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"}, {"Index": 6, "Name": "Indexes", "Type": "TCoIndexList"}, {"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"}, {"Index": 8, "Name": "ColumnFamilies", "Type": "TExprList"}, @@ -121,17 +121,17 @@ ] }, { - "Name": "TKiAlterTable", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiAlterTable!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + "Name": "TKiAlterTable", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiAlterTable!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, {"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"} - ] - }, - { + ] + }, + { "Name": "TKiDropTable", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "KiDropTable!"}, @@ -209,20 +209,20 @@ ] }, { - "Name": "TKiOperation", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 2, "Name": "Operation", "Type": "TCoAtom"} - ] - }, - { + "Name": "TKiOperation", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Operation", "Type": "TCoAtom"} + ] + }, + { "Name": "TKiOperationList", - "ListBase": "TKiOperation" - }, - { + "ListBase": "TKiOperation" + }, + { "Name": "TKiResult", "Base": "TExprBase", "Match": {"Type": "Tuple"}, @@ -280,7 +280,7 @@ "Match": {"Type": "Callable", "Name": "KiSelectRow"}, "Children": [ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, + {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"}, {"Index": 3, "Name": "Select", "Type": "TCoAtomList"} ] @@ -291,7 +291,7 @@ "Match": {"Type": "Callable", "Name": "KiUpdateRow"}, "Children": [ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, + {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"}, {"Index": 3, "Name": "Update", "Type": "TCoNameValueTupleList"} ] @@ -302,7 +302,7 @@ "Match": {"Type": "Callable", "Name": "KiEraseRow"}, "Children": [ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, + {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"} ] }, @@ -313,7 +313,7 @@ "Builder": {"Generate": "None"}, "Children": [ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, + {"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"}, {"Index": 2, "Name": "Range", "Type": "TExprList"}, {"Index": 3, "Name": "Select", "Type": "TCoAtomList"}, {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"} @@ -325,12 +325,12 @@ "Match": {"Type": "Callable", "Name": "KiSelectRange"} }, { - "Name": "TKiSelectIndexRange", + "Name": "TKiSelectIndexRange", "Base": "TKiSelectRangeBase", - "Match": {"Type": "Callable", "Name": "KiSelectIndexRange"}, - "Children": [ - {"Index": 5, "Name": "IndexName", "Type": "TCoAtom"} - ] + "Match": {"Type": "Callable", "Name": "KiSelectIndexRange"}, + "Children": [ + {"Index": 5, "Name": "IndexName", "Type": "TCoAtom"} + ] }, { "Name": "TKiSetResult", @@ -359,15 +359,15 @@ ] }, { - "Name": "TMkqlMapParameter", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "MapParameter"}, - "Children": [ - {"Index": 0, "Name": "Input", "Type": "TExprBase"}, - {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"} - ] - }, - { + "Name": "TMkqlMapParameter", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "MapParameter"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"}, + {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"} + ] + }, + { "Name": "TKiFlatMapParameter", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "KiFlatMapParameter"}, diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index add60c21bb..e49f84b5e1 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -163,11 +163,11 @@ TFuture<IKikimrGateway::TGenericResult> IKikimrGateway::CreatePath(const TString return pathPromise.GetFuture(); } -TString IKikimrGateway::CreateIndexTablePath(const TString& tableName, const TString& indexName) { - return tableName + "/" + indexName + "/indexImplTable"; -} - - +TString IKikimrGateway::CreateIndexTablePath(const TString& tableName, const TString& indexName) { + return tableName + "/" + indexName + "/indexImplTable"; +} + + void IKikimrGateway::BuildIndexMetadata(TTableMetadataResult& loadTableMetadataResult) { auto tableMetadata = loadTableMetadataResult.Metadata; YQL_ENSURE(tableMetadata); @@ -176,25 +176,25 @@ void IKikimrGateway::BuildIndexMetadata(TTableMetadataResult& loadTableMetadataR return; } - const auto& cluster = tableMetadata->Cluster; - const auto& tableName = tableMetadata->Name; - const size_t indexesCount = tableMetadata->Indexes.size(); - + const auto& cluster = tableMetadata->Cluster; + const auto& tableName = tableMetadata->Name; + const size_t indexesCount = tableMetadata->Indexes.size(); + NKikimr::NTableIndex::TTableColumns tableColumns; tableColumns.Columns.reserve(tableMetadata->Columns.size()); for (auto& column: tableMetadata->Columns) { tableColumns.Columns.insert_noresize(column.first); } tableColumns.Keys = tableMetadata->KeyColumnNames; - + tableMetadata->SecondaryGlobalIndexMetadata.resize(indexesCount); - for (size_t i = 0; i < indexesCount; i++) { - const auto& index = tableMetadata->Indexes[i]; + for (size_t i = 0; i < indexesCount; i++) { + const auto& index = tableMetadata->Indexes[i]; auto indexTablePath = CreateIndexTablePath(tableName, index.Name); NKikimr::NTableIndex::TTableColumns indexTableColumns = NKikimr::NTableIndex::CalcTableImplDescription( - tableColumns, - NKikimr::NTableIndex::TIndexColumns{index.KeyColumns, {}}); - + tableColumns, + NKikimr::NTableIndex::TIndexColumns{index.KeyColumns, {}}); + TKikimrTableMetadataPtr indexTableMetadata = new TKikimrTableMetadata(cluster, indexTablePath); indexTableMetadata->DoesExist = true; indexTableMetadata->KeyColumnNames = indexTableColumns.Keys; @@ -203,9 +203,9 @@ void IKikimrGateway::BuildIndexMetadata(TTableMetadataResult& loadTableMetadataR } tableMetadata->SecondaryGlobalIndexMetadata[i] = indexTableMetadata; - } -} - + } +} + bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error) { using namespace NNodes; @@ -238,119 +238,119 @@ bool TTableSettings::IsSet() const { || ReadReplicasSettings || TtlSettings; } -EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { - switch (ydbStatus) { - case Ydb::StatusIds::SUCCESS: - return TIssuesIds::SUCCESS; - case Ydb::StatusIds::BAD_REQUEST: - return TIssuesIds::KIKIMR_BAD_REQUEST; - case Ydb::StatusIds::UNAUTHORIZED: - return TIssuesIds::KIKIMR_ACCESS_DENIED; - case Ydb::StatusIds::INTERNAL_ERROR: - return TIssuesIds::DEFAULT_ERROR; - case Ydb::StatusIds::ABORTED: - return TIssuesIds::KIKIMR_OPERATION_ABORTED; - case Ydb::StatusIds::UNAVAILABLE: - return TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE; - case Ydb::StatusIds::OVERLOADED: - return TIssuesIds::KIKIMR_OVERLOADED; - case Ydb::StatusIds::SCHEME_ERROR: - return TIssuesIds::KIKIMR_SCHEME_ERROR; - case Ydb::StatusIds::GENERIC_ERROR: - return TIssuesIds::DEFAULT_ERROR; +EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { + switch (ydbStatus) { + case Ydb::StatusIds::SUCCESS: + return TIssuesIds::SUCCESS; + case Ydb::StatusIds::BAD_REQUEST: + return TIssuesIds::KIKIMR_BAD_REQUEST; + case Ydb::StatusIds::UNAUTHORIZED: + return TIssuesIds::KIKIMR_ACCESS_DENIED; + case Ydb::StatusIds::INTERNAL_ERROR: + return TIssuesIds::DEFAULT_ERROR; + case Ydb::StatusIds::ABORTED: + return TIssuesIds::KIKIMR_OPERATION_ABORTED; + case Ydb::StatusIds::UNAVAILABLE: + return TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE; + case Ydb::StatusIds::OVERLOADED: + return TIssuesIds::KIKIMR_OVERLOADED; + case Ydb::StatusIds::SCHEME_ERROR: + return TIssuesIds::KIKIMR_SCHEME_ERROR; + case Ydb::StatusIds::GENERIC_ERROR: + return TIssuesIds::DEFAULT_ERROR; case Ydb::StatusIds::TIMEOUT: return TIssuesIds::KIKIMR_TIMEOUT; - case Ydb::StatusIds::BAD_SESSION: - return TIssuesIds::KIKIMR_TOO_MANY_TRANSACTIONS; - case Ydb::StatusIds::PRECONDITION_FAILED: - return TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION; + case Ydb::StatusIds::BAD_SESSION: + return TIssuesIds::KIKIMR_TOO_MANY_TRANSACTIONS; + case Ydb::StatusIds::PRECONDITION_FAILED: + return TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION; case Ydb::StatusIds::CANCELLED: return TIssuesIds::KIKIMR_OPERATION_CANCELLED; case Ydb::StatusIds::UNSUPPORTED: return TIssuesIds::KIKIMR_UNSUPPORTED; - default: - return TIssuesIds::DEFAULT_ERROR; - } -} - + default: + return TIssuesIds::DEFAULT_ERROR; + } +} + void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull) { - NUdf::EDataSlot dataSlot = NUdf::GetDataSlot(typeName); - if (dataSlot == NUdf::EDataSlot::Decimal) { + NUdf::EDataSlot dataSlot = NUdf::GetDataSlot(typeName); + if (dataSlot == NUdf::EDataSlot::Decimal) { auto decimal = notNull ? protoType.mutable_decimal_type() : protoType.mutable_optional_type()->mutable_item()->mutable_decimal_type(); - // We have no params right now - // TODO: Fix decimal params support for kikimr - decimal->set_precision(22); - decimal->set_scale(9); - } else { + // We have no params right now + // TODO: Fix decimal params support for kikimr + decimal->set_precision(22); + decimal->set_scale(9); + } else { auto& primitive = notNull ? protoType : *protoType.mutable_optional_type()->mutable_item(); - auto id = NUdf::GetDataTypeInfo(dataSlot).TypeId; + auto id = NUdf::GetDataTypeInfo(dataSlot).TypeId; primitive.set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(id)); - } -} - -bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::ReadReplicasSettings& proto, - Ydb::StatusIds::StatusCode& code, TString& error) -{ - TVector<TString> azs = StringSplitter(to_lower(settings)).Split(',').SkipEmpty(); - bool wrongFormat = false; - if (azs) { - for (const auto& az : azs) { - TVector<TString> azSettings = StringSplitter(az).Split(':').SkipEmpty(); - if (azSettings.size() != 2) { - wrongFormat = true; - break; - } - TVector<TString> valueString = StringSplitter(azSettings[1]).Split(' ').SkipEmpty(); - TVector<TString> nameString = StringSplitter(azSettings[0]).Split(' ').SkipEmpty(); - ui64 value; - if (valueString.size() != 1 || !TryFromString<ui64>(valueString[0], value) || nameString.size() != 1) { - wrongFormat = true; - break; - } - if ("per_az" == nameString[0]) { - if (azs.size() != 1) { - wrongFormat = true; - break; - } - proto.set_per_az_read_replicas_count(value); - } else if ("any_az" == nameString[0]) { - if (azs.size() != 1) { - wrongFormat = true; - break; - } - proto.set_any_az_read_replicas_count(value); - } else { - code = Ydb::StatusIds::UNSUPPORTED; - error = "Specifying read replicas count for each AZ in cluster is not supported yet"; - return false; - //auto& clusterReplicasSettings = *proto.mutable_cluster_replicas_settings(); - //auto& azDesc = *clusterReplicasSettings.add_az_read_replicas_settings(); - //azDesc.set_name(nameString[0]); - //azDesc.set_read_replicas_count(value); - } - } - } else { - wrongFormat = true; - } - if (wrongFormat) { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Wrong format for read replicas settings '" << settings - << "'. It should be one of: " - << "1) 'PER_AZ:<read_replicas_count>' to set equal read replicas count for every AZ; " - << "2) 'ANY_AZ:<read_replicas_count>' to set total read replicas count between all AZs; " - << "3) '<az1_name>:<read_replicas_count1>, <az2_name>:<read_replicas_count2>, ...' " - << "to specify read replicas count for each AZ in cluster."; - return false; - } - return true; -} - -void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto) { - proto.mutable_date_type_column()->set_column_name(settings.ColumnName); - proto.mutable_date_type_column()->set_expire_after_seconds(settings.ExpireAfter.Seconds()); -} - + } +} + +bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::ReadReplicasSettings& proto, + Ydb::StatusIds::StatusCode& code, TString& error) +{ + TVector<TString> azs = StringSplitter(to_lower(settings)).Split(',').SkipEmpty(); + bool wrongFormat = false; + if (azs) { + for (const auto& az : azs) { + TVector<TString> azSettings = StringSplitter(az).Split(':').SkipEmpty(); + if (azSettings.size() != 2) { + wrongFormat = true; + break; + } + TVector<TString> valueString = StringSplitter(azSettings[1]).Split(' ').SkipEmpty(); + TVector<TString> nameString = StringSplitter(azSettings[0]).Split(' ').SkipEmpty(); + ui64 value; + if (valueString.size() != 1 || !TryFromString<ui64>(valueString[0], value) || nameString.size() != 1) { + wrongFormat = true; + break; + } + if ("per_az" == nameString[0]) { + if (azs.size() != 1) { + wrongFormat = true; + break; + } + proto.set_per_az_read_replicas_count(value); + } else if ("any_az" == nameString[0]) { + if (azs.size() != 1) { + wrongFormat = true; + break; + } + proto.set_any_az_read_replicas_count(value); + } else { + code = Ydb::StatusIds::UNSUPPORTED; + error = "Specifying read replicas count for each AZ in cluster is not supported yet"; + return false; + //auto& clusterReplicasSettings = *proto.mutable_cluster_replicas_settings(); + //auto& azDesc = *clusterReplicasSettings.add_az_read_replicas_settings(); + //azDesc.set_name(nameString[0]); + //azDesc.set_read_replicas_count(value); + } + } + } else { + wrongFormat = true; + } + if (wrongFormat) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Wrong format for read replicas settings '" << settings + << "'. It should be one of: " + << "1) 'PER_AZ:<read_replicas_count>' to set equal read replicas count for every AZ; " + << "2) 'ANY_AZ:<read_replicas_count>' to set total read replicas count between all AZs; " + << "3) '<az1_name>:<read_replicas_count1>, <az2_name>:<read_replicas_count2>, ...' " + << "to specify read replicas count for each AZ in cluster."; + return false; + } + return true; +} + +void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto) { + proto.mutable_date_type_column()->set_column_name(settings.ColumnName); + proto.mutable_date_type_column()->set_expire_after_seconds(settings.ExpireAfter.Seconds()); +} + Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) { if (!value) { return Ydb::FeatureFlag::STATUS_UNSPECIFIED; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 22b1656eb7..7598494a2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -46,30 +46,30 @@ struct TKikimrQueryLimits { TKikimrQueryPhaseLimits PhaseLimits; }; -struct TIndexDescription { +struct TIndexDescription { enum class EType : ui32 { GlobalSync = 0, GlobalAsync = 1, - }; - + }; + // Index states here must be in sync with NKikimrSchemeOp::EIndexState protobuf - enum class EIndexState : ui32 { - Invalid = 0, // this state should not be used - Ready = 1, // index is ready to use + enum class EIndexState : ui32 { + Invalid = 0, // this state should not be used + Ready = 1, // index is ready to use NotReady = 2, // index is visible but not ready to use WriteOnly = 3 // index is visible only write operations to index are allowed - }; - - const TString Name; - const TVector<TString> KeyColumns; - const TVector<TString> DataColumns; - const EType Type; - const EIndexState State; - const ui64 SchemaVersion; - const ui64 LocalPathId; - const ui64 PathOwnerId; - + }; + + const TString Name; + const TVector<TString> KeyColumns; + const TVector<TString> DataColumns; + const EType Type; + const EIndexState State; + const ui64 SchemaVersion; + const ui64 LocalPathId; + const ui64 PathOwnerId; + TIndexDescription(const TString& name, const TVector<TString>& keyColumns, const TVector<TString>& dataColumns, EType type, EIndexState state, ui64 schemaVersion, ui64 localPathId, ui64 pathOwnerId) : Name(name) @@ -130,12 +130,12 @@ struct TIndexDescription { } } - bool IsSameIndex(const TIndexDescription& other) const { - return Name == other.Name && - KeyColumns == other.KeyColumns && - DataColumns == other.DataColumns && - Type == other.Type; - } + bool IsSameIndex(const TIndexDescription& other) const { + return Name == other.Name && + KeyColumns == other.KeyColumns && + DataColumns == other.DataColumns && + Type == other.Type; + } bool ItUsedForWrite() const { switch (Type) { @@ -145,8 +145,8 @@ struct TIndexDescription { return false; } } -}; - +}; + struct TColumnFamily { TString Name; TMaybe<TString> Data; @@ -292,10 +292,10 @@ struct TKikimrTableMetadata : public TThrRefBase { TVector<TString> KeyColumnNames; TVector<TString> ColumnOrder; - // Indexes and SecondaryGlobalIndexMetadata must be in same order - TVector<TIndexDescription> Indexes; - TVector<TIntrusivePtr<TKikimrTableMetadata>> SecondaryGlobalIndexMetadata; - + // Indexes and SecondaryGlobalIndexMetadata must be in same order + TVector<TIndexDescription> Indexes; + TVector<TIntrusivePtr<TKikimrTableMetadata>> SecondaryGlobalIndexMetadata; + TVector<TColumnFamily> ColumnFamilies; TTableSettings TableSettings; @@ -347,10 +347,10 @@ struct TKikimrTableMetadata : public TThrRefBase { } if (Cluster != other.Cluster || Name != other.Name || Columns.size() != other.Columns.size() || - KeyColumnNames != other.KeyColumnNames || Indexes.size() != other.Indexes.size()) { - return false; - } - + KeyColumnNames != other.KeyColumnNames || Indexes.size() != other.Indexes.size()) { + return false; + } + for (auto& [name, column]: Columns) { auto otherColumn = other.Columns.FindPtr(name); if (!otherColumn) { @@ -362,15 +362,15 @@ struct TKikimrTableMetadata : public TThrRefBase { } } - for (size_t i = 0; i < Indexes.size(); i++) { - if (!Indexes[i].IsSameIndex(other.Indexes[i])) { - return false; - } - } - - return true; + for (size_t i = 0; i < Indexes.size(); i++) { + if (!Indexes[i].IsSameIndex(other.Indexes[i])) { + return false; + } + } + + return true; } - + void ToMessage(NKikimrKqp::TKqpTableMetadataProto* message) const { message->SetDoesExist(DoesExist); message->SetCluster(Cluster); @@ -407,18 +407,18 @@ struct TKikimrTableMetadata : public TThrRefBase { return proto.SerializeAsString(); } - std::pair<TIntrusivePtr<TKikimrTableMetadata>, TIndexDescription::EIndexState> GetIndexMetadata(const TString& indexName) const { - YQL_ENSURE(Indexes.size(), "GetIndexMetadata called for table without indexes"); - YQL_ENSURE(Indexes.size() == SecondaryGlobalIndexMetadata.size(), "index metadata has not been loaded yet"); - for (size_t i = 0; i < Indexes.size(); i++) { - if (Indexes[i].Name == indexName) { - auto metadata = SecondaryGlobalIndexMetadata[i]; - YQL_ENSURE(metadata, "unexpected empty metadata for index " << indexName); - return {metadata, Indexes[i].State}; - } - } - return {nullptr, TIndexDescription::EIndexState::Invalid}; - } + std::pair<TIntrusivePtr<TKikimrTableMetadata>, TIndexDescription::EIndexState> GetIndexMetadata(const TString& indexName) const { + YQL_ENSURE(Indexes.size(), "GetIndexMetadata called for table without indexes"); + YQL_ENSURE(Indexes.size() == SecondaryGlobalIndexMetadata.size(), "index metadata has not been loaded yet"); + for (size_t i = 0; i < Indexes.size(); i++) { + if (Indexes[i].Name == indexName) { + auto metadata = SecondaryGlobalIndexMetadata[i]; + YQL_ENSURE(metadata, "unexpected empty metadata for index " << indexName); + return {metadata, Indexes[i].State}; + } + } + return {nullptr, TIndexDescription::EIndexState::Invalid}; + } }; struct TCreateUserSettings { @@ -527,8 +527,8 @@ static TIntrusivePtr<TKikimrResultHolder<TResult>> MakeKikimrResultHolder(TResul class IKikimrGateway : public TThrRefBase { public: - using TPtr = TIntrusivePtr<IKikimrGateway>; - + using TPtr = TIntrusivePtr<IKikimrGateway>; + struct TGenericResult : public NCommon::TOperationResult { }; @@ -543,7 +543,7 @@ public: struct TQueryResult : public TGenericResult { TString SessionId; - TVector<NKikimrMiniKQL::TResult*> Results; + TVector<NKikimrMiniKQL::TResult*> Results; TMaybe<NKikimrKqp::TQueryProfile> Profile; // TODO: Deprecate. NKqpProto::TKqpStatsQuery QueryStats; std::unique_ptr<NKikimrKqp::TPreparedQuery> PreparingQuery; @@ -551,25 +551,25 @@ public: std::optional<NKikimr::NKqp::TQueryTraits> QueryTraits; TString QueryAst; TString QueryPlan; - std::shared_ptr<google::protobuf::Arena> ProtobufArenaPtr; + std::shared_ptr<google::protobuf::Arena> ProtobufArenaPtr; TMaybe<ui16> SqlVersion; }; - struct TLoadTableMetadataSettings { - TLoadTableMetadataSettings& WithTableStats(bool enable) { - RequestStats_ = enable; - return *this; - } - - TLoadTableMetadataSettings& WithPrivateTables(bool enable) { - WithPrivateTables_ = enable; - return *this; - } - - bool RequestStats_ = false; - bool WithPrivateTables_ = false; - }; - + struct TLoadTableMetadataSettings { + TLoadTableMetadataSettings& WithTableStats(bool enable) { + RequestStats_ = enable; + return *this; + } + + TLoadTableMetadataSettings& WithPrivateTables(bool enable) { + WithPrivateTables_ = enable; + return *this; + } + + bool RequestStats_ = false; + bool WithPrivateTables_ = false; + }; + class IKqpTableMetadataLoader : public std::enable_shared_from_this<IKqpTableMetadataLoader> { public: virtual NThreading::TFuture<TTableMetadataResult> LoadTableMetadata( @@ -593,12 +593,12 @@ public: virtual NThreading::TFuture<TListPathResult> ListPath(const TString& cluster, const TString& path) = 0; virtual NThreading::TFuture<TTableMetadataResult> LoadTableMetadata( - const TString& cluster, const TString& table, TLoadTableMetadataSettings settings) = 0; + const TString& cluster, const TString& table, TLoadTableMetadataSettings settings) = 0; virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir) = 0; - virtual NThreading::TFuture<TGenericResult> AlterTable(Ydb::Table::AlterTableRequest&& req, const TString& cluster) = 0; - + virtual NThreading::TFuture<TGenericResult> AlterTable(Ydb::Table::AlterTableRequest&& req, const TString& cluster) = 0; + virtual NThreading::TFuture<TGenericResult> RenameTable(const TString& src, const TString& dst, const TString& cluster) = 0; virtual NThreading::TFuture<TGenericResult> DropTable(const TString& cluster, const TString& table) = 0; @@ -635,20 +635,20 @@ public: static bool TrySplitTablePath(const TString& path, std::pair<TString, TString>& result, TString& error); static NThreading::TFuture<TGenericResult> CreatePath(const TString& path, TCreateDirFunc createDir); - - static TString CreateIndexTablePath(const TString& tableName, const TString& indexName); - + + static TString CreateIndexTablePath(const TString& tableName, const TString& indexName); + static void BuildIndexMetadata(TTableMetadataResult& loadTableMetadataResult); }; -EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus); +EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus); Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value); void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull); -bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::ReadReplicasSettings& proto, - Ydb::StatusIds::StatusCode& code, TString& error); -void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto); - +bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::ReadReplicasSettings& proto, + Ydb::StatusIds::StatusCode& code, TString& error); +void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto); + } // namespace NYql template<> diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index 63c4197374..791a617c08 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -85,22 +85,22 @@ void TestListPathCommon(TIntrusivePtr<IKikimrGateway> gateway) { UNIT_ASSERT(response.Success()); UNIT_ASSERT_VALUES_EQUAL(response.Path, "/Root/Test"); - UNIT_ASSERT_VALUES_EQUAL(response.Items.size(), 5); + UNIT_ASSERT_VALUES_EQUAL(response.Items.size(), 5); UNIT_ASSERT_VALUES_EQUAL(response.Items[0].Name, "TestTable2"); UNIT_ASSERT_VALUES_EQUAL(response.Items[0].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(response.Items[1].Name, "TestTable3"); + UNIT_ASSERT_VALUES_EQUAL(response.Items[1].Name, "TestTable3"); UNIT_ASSERT_VALUES_EQUAL(response.Items[1].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(response.Items[2].Name, "TestTableKsv"); - UNIT_ASSERT_VALUES_EQUAL(response.Items[2].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(response.Items[3].Name, "UserDir"); - UNIT_ASSERT_VALUES_EQUAL(response.Items[3].IsDirectory, true); - UNIT_ASSERT_VALUES_EQUAL(response.Items[4].Name, "UserTable"); - UNIT_ASSERT_VALUES_EQUAL(response.Items[4].IsDirectory, false); + UNIT_ASSERT_VALUES_EQUAL(response.Items[2].Name, "TestTableKsv"); + UNIT_ASSERT_VALUES_EQUAL(response.Items[2].IsDirectory, false); + UNIT_ASSERT_VALUES_EQUAL(response.Items[3].Name, "UserDir"); + UNIT_ASSERT_VALUES_EQUAL(response.Items[3].IsDirectory, true); + UNIT_ASSERT_VALUES_EQUAL(response.Items[4].Name, "UserTable"); + UNIT_ASSERT_VALUES_EQUAL(response.Items[4].IsDirectory, false); } void TestLoadTableMetadataCommon(TIntrusivePtr<IKikimrGateway> gateway) { auto responseFuture = gateway->LoadTableMetadata(TestCluster, "/Root/Test/UserTable", - IKikimrGateway::TLoadTableMetadataSettings()); + IKikimrGateway::TLoadTableMetadataSettings()); responseFuture.Wait(); auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); @@ -130,12 +130,12 @@ void TestRunSimpleCommon(TIntrusivePtr<IKqpGateway> gateway) { 'Name 'Amount )) - (let table '( + (let table '( '"/Root/Test/TestTable2" - '"0" - '"" - )) - (let data (SelectRow table key row)) + '"0" + '"" + )) + (let data (SelectRow table key row)) (let r (SetResult 'Result data)) (let pgmReturn (AsList r)) (return pgmReturn) @@ -181,12 +181,12 @@ void CheckPolicies(Tests::TClient& client, const TString& tableName) { } } -struct TTestIndexSettings { - const bool WithDataColumns; -}; - +struct TTestIndexSettings { + const bool WithDataColumns; +}; + void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient& client, - bool createFolders = true, const TMaybe<TTestIndexSettings> withIndex = Nothing(), bool withExtendedDdl = false, + bool createFolders = true, const TMaybe<TTestIndexSettings> withIndex = Nothing(), bool withExtendedDdl = false, const TMaybe<bool>& shouldCreate = Nothing()) { auto metadata = MakeIntrusive<TKikimrTableMetadata>(); @@ -207,26 +207,26 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient metadata->KeyColumnNames.push_back("Column1"); - if (withIndex) { - TVector<TString> dataColumns; - if (withIndex->WithDataColumns) { + if (withIndex) { + TVector<TString> dataColumns; + if (withIndex->WithDataColumns) { metadata->Columns.insert(std::make_pair("Column3", TKikimrColumnMetadata{"Column3", 0, "String", false})); metadata->ColumnOrder.push_back("Column3"); - dataColumns.push_back("Column3"); - } - TIndexDescription indexDesc{ - TString("Column2Index"), - TVector<TString>{"Column2"}, - dataColumns, + dataColumns.push_back("Column3"); + } + TIndexDescription indexDesc{ + TString("Column2Index"), + TVector<TString>{"Column2"}, + dataColumns, TIndexDescription::EType::GlobalSync, - TIndexDescription::EIndexState::Ready, - 0, - 0, - 0 - }; - metadata->Indexes.push_back(indexDesc); - } - + TIndexDescription::EIndexState::Ready, + 0, + 0, + 0 + }; + metadata->Indexes.push_back(indexDesc); + } + if (withExtendedDdl) { metadata->TableSettings.AutoPartitioningBySize = "disabled"; metadata->TableSettings.PartitionAtKeys = { @@ -251,37 +251,37 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient UNIT_ASSERT_C(response.Success(), response.Issues().ToString()); auto loadFuture = gateway->LoadTableMetadata(TestCluster, "/Root/f1/f2/table", - IKikimrGateway::TLoadTableMetadataSettings()); + IKikimrGateway::TLoadTableMetadataSettings()); loadFuture.Wait(); auto loadResponse = loadFuture.GetValue(); UNIT_ASSERT(loadResponse.Success()); - UNIT_ASSERT_VALUES_EQUAL(metadata->Name, loadResponse.Metadata->Name); - UNIT_ASSERT_VALUES_EQUAL(metadata->Indexes.size(), loadResponse.Metadata->Indexes.size()); - - THashMap<TString, TIndexDescription> expected; - for (const auto& indexDesc : metadata->Indexes) { - expected.insert(std::make_pair(indexDesc.Name, indexDesc)); - } - - THashMap<TString, TIndexDescription> indexResult; - for (const auto& indexDesc : loadResponse.Metadata->Indexes) { - indexResult.insert(std::make_pair(indexDesc.Name, indexDesc)); - } - - UNIT_ASSERT_VALUES_EQUAL(indexResult.size(), expected.size()); - for (const auto& indexDescResult : indexResult) { - const auto expectedDesc = expected.find(indexDescResult.first); - UNIT_ASSERT(expectedDesc != expected.end()); - UNIT_ASSERT_VALUES_EQUAL(expectedDesc->second.KeyColumns.size(), indexDescResult.second.KeyColumns.size()); - UNIT_ASSERT_EQUAL(expectedDesc->second.Type, indexDescResult.second.Type); - for (size_t i = 0; i < indexDescResult.second.KeyColumns.size(); i++) { - UNIT_ASSERT_VALUES_EQUAL(indexDescResult.second.KeyColumns[i], expectedDesc->second.KeyColumns[i]); - } - UNIT_ASSERT_VALUES_EQUAL(expectedDesc->second.DataColumns.size(), indexDescResult.second.DataColumns.size()); - for (size_t i = 0; i < indexDescResult.second.DataColumns.size(); i++) { - UNIT_ASSERT_VALUES_EQUAL(indexDescResult.second.DataColumns[i], expectedDesc->second.DataColumns[i]); - } - } + UNIT_ASSERT_VALUES_EQUAL(metadata->Name, loadResponse.Metadata->Name); + UNIT_ASSERT_VALUES_EQUAL(metadata->Indexes.size(), loadResponse.Metadata->Indexes.size()); + + THashMap<TString, TIndexDescription> expected; + for (const auto& indexDesc : metadata->Indexes) { + expected.insert(std::make_pair(indexDesc.Name, indexDesc)); + } + + THashMap<TString, TIndexDescription> indexResult; + for (const auto& indexDesc : loadResponse.Metadata->Indexes) { + indexResult.insert(std::make_pair(indexDesc.Name, indexDesc)); + } + + UNIT_ASSERT_VALUES_EQUAL(indexResult.size(), expected.size()); + for (const auto& indexDescResult : indexResult) { + const auto expectedDesc = expected.find(indexDescResult.first); + UNIT_ASSERT(expectedDesc != expected.end()); + UNIT_ASSERT_VALUES_EQUAL(expectedDesc->second.KeyColumns.size(), indexDescResult.second.KeyColumns.size()); + UNIT_ASSERT_EQUAL(expectedDesc->second.Type, indexDescResult.second.Type); + for (size_t i = 0; i < indexDescResult.second.KeyColumns.size(); i++) { + UNIT_ASSERT_VALUES_EQUAL(indexDescResult.second.KeyColumns[i], expectedDesc->second.KeyColumns[i]); + } + UNIT_ASSERT_VALUES_EQUAL(expectedDesc->second.DataColumns.size(), indexDescResult.second.DataColumns.size()); + for (size_t i = 0; i < indexDescResult.second.DataColumns.size(); i++) { + UNIT_ASSERT_VALUES_EQUAL(indexDescResult.second.DataColumns[i], expectedDesc->second.DataColumns[i]); + } + } if (withExtendedDdl) { CheckPolicies(client, metadata->Name); @@ -297,7 +297,7 @@ void TestDropTableCommon(TIntrusivePtr<IKikimrGateway> gateway) { UNIT_ASSERT(response.Success()); auto loadFuture = gateway->LoadTableMetadata(TestCluster, "/Root/Test/UserTable", - IKikimrGateway::TLoadTableMetadataSettings()); + IKikimrGateway::TLoadTableMetadataSettings()); loadFuture.Wait(); auto loadResponse = loadFuture.GetValue(); UNIT_ASSERT(loadResponse.Success()); @@ -332,20 +332,20 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient()); } - Y_UNIT_TEST(TestCreateTableWithIndex) { + Y_UNIT_TEST(TestCreateTableWithIndex) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient(), true, TTestIndexSettings{false}); - } - - Y_UNIT_TEST(TestCreateTableWithCoverIndex) { + } + + Y_UNIT_TEST(TestCreateTableWithCoverIndex) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient(), true, TTestIndexSettings{true}); - } - + } + Y_UNIT_TEST(TestCreateTableNoFolder) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); @@ -359,13 +359,13 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient()); } - Y_UNIT_TEST(TestCreateSameTableWithIndex) { + Y_UNIT_TEST(TestCreateSameTableWithIndex) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient(), true, TTestIndexSettings{false}); - } - + } + Y_UNIT_TEST(TestDropTable) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); diff --git a/ydb/core/kqp/provider/yql_kikimr_kql.cpp b/ydb/core/kqp/provider/yql_kikimr_kql.cpp index 5c218b449c..95f5915a57 100644 --- a/ydb/core/kqp/provider/yql_kikimr_kql.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_kql.cpp @@ -1,5 +1,5 @@ #include "yql_kikimr_provider_impl.h" -#include "kqp_opt_helpers.h" +#include "kqp_opt_helpers.h" #include <ydb/library/yql/core/yql_opt_utils.h> @@ -7,235 +7,235 @@ namespace NYql { namespace { using namespace NNodes; -using namespace NKikimr::NKqp; - -TCoNameValueTupleList CreateSecondaryIndexKeyTuples(TCoArgument itemArg, const TVector<TString>& keyColumnNames, - const THashSet<TStringBuf>& inputColumns, const TKikimrTableDescription& table, - const TExprBase& fetch, bool nullExtension, TExprContext& ctx) -{ - TVector<TExprBase> keyTuples; - THashSet<TString> uniqColumns; - for (const TString& name : keyColumnNames) { - // skip already added columns - if (!uniqColumns.insert(name).second) { - continue; - } - if (inputColumns.contains(name)) { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(name) - .Build() - .Done(); - - keyTuples.push_back(tuple); - } else { - if (nullExtension) { - const auto& type = table.GetColumnType(name); - const auto& member = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .template Value<TCoNothing>() - .OptionalType(NCommon::BuildTypeExpr(itemArg.Pos(), *type, ctx)) - .Build() - .Done(); - keyTuples.emplace_back(TExprBase(member)); - } else { - const auto& member = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(name) - .Build() - .Done(); - keyTuples.emplace_back(TExprBase(member)); - } - } - } - return Build<TCoNameValueTupleList>(ctx, itemArg.Pos()) - .Add(keyTuples) - .Done(); -} - -TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> BuildSecondaryIndexVector( - const TKikimrTableDescription& table, +using namespace NKikimr::NKqp; + +TCoNameValueTupleList CreateSecondaryIndexKeyTuples(TCoArgument itemArg, const TVector<TString>& keyColumnNames, + const THashSet<TStringBuf>& inputColumns, const TKikimrTableDescription& table, + const TExprBase& fetch, bool nullExtension, TExprContext& ctx) +{ + TVector<TExprBase> keyTuples; + THashSet<TString> uniqColumns; + for (const TString& name : keyColumnNames) { + // skip already added columns + if (!uniqColumns.insert(name).second) { + continue; + } + if (inputColumns.contains(name)) { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(name) + .Build() + .Done(); + + keyTuples.push_back(tuple); + } else { + if (nullExtension) { + const auto& type = table.GetColumnType(name); + const auto& member = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .template Value<TCoNothing>() + .OptionalType(NCommon::BuildTypeExpr(itemArg.Pos(), *type, ctx)) + .Build() + .Done(); + keyTuples.emplace_back(TExprBase(member)); + } else { + const auto& member = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(name) + .Build() + .Done(); + keyTuples.emplace_back(TExprBase(member)); + } + } + } + return Build<TCoNameValueTupleList>(ctx, itemArg.Pos()) + .Add(keyTuples) + .Done(); +} + +TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> BuildSecondaryIndexVector( + const TKikimrTableDescription& table, TPositionHandle pos, - TExprContext& ctx, - const THashSet<TStringBuf>* filter = nullptr) -{ - return NKikimr::NKqp::BuildSecondaryIndexVector(table, pos, ctx, filter, &BuildVersionedTable); -} - -TExprBase BuildConditionalErase( - const TExprBase& condition, - const TCoArgument& itemArg, - const TVector<TExprBase>& keyToErase, - TExprNode::TPtr table, - const TKiWriteTable& node, - TExprContext& ctx) -{ - const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(table) - .Key<TCoNameValueTupleList>() - .Add(keyToErase) - .Build() - .Done(); - - return Build<TCoIfPresent>(ctx, node.Pos()) - .Optional(condition) - .PresentHandler<TCoLambda>() - .Args(itemArg) - .Body(erase) - .Build() - .MissingValue<TCoVoid>().Build() - .Done(); -} - -TExprBase CreateUpdateRowWithSecondaryIndex( - const TKiUpdateRow& updateTable, - const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> indexes, - const TKikimrTableDescription& table, - const TCoNameValueTupleList& tablePk, - const THashSet<TStringBuf>& inputColumns, - const TCoArgument& itemArg, - const TKiWriteTable& node, TExprContext& ctx, bool replace, bool skipErase) -{ - TVector<TExprBase> updates; - updates.push_back(updateTable); - - const TVector<TString>& pk = table.Metadata->KeyColumnNames; - // in case of replace mode no need to read old data columns - const auto dataColumnSet = replace ? THashSet<TString>() : CreateDataColumnSetToRead(indexes, inputColumns); - const auto columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dataColumnSet, node.Pos(), ctx); - - const TExprBase& fetch = Build<TKiSelectRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(updateTable.Table()) - .Key(tablePk) - .template Select<TCoAtomList>() - .Add(columnsToSelect) - .Build() - .Done(); - - for (const auto& pair : indexes) { - TVector<TString> indexTablePk; - TVector<TCoNameValueTuple> dataColumnUpdates; - - // skip update of index table if indexed column not present in request - // and upserted rows already present - bool mayBeSkipIndexUpdate = !replace; - indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); - for (const auto& col : pair.second->KeyColumns) { - indexTablePk.push_back(col); - if (inputColumns.contains(col)) { - mayBeSkipIndexUpdate = false; - } - } - indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); - - // We can`t skip index update if data column present in query - // but in this case we may be need to skip erace - bool mustUpdateDataColumn = false; - - for (const auto& col : pair.second->DataColumns) { - if (inputColumns.contains(col)) { - mustUpdateDataColumn = true; - auto tuple = Build<TCoNameValueTuple>(ctx, node.Pos()) - .Name().Build(col) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(col) - .Build() - .Done(); - dataColumnUpdates.push_back(tuple); - } else if (!replace) { - // Index table has data column, but data column has not been - // specifyed in request - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(col) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(col) - .Build() - .Done(); - dataColumnUpdates.push_back(tuple); - } - } - - TExprBase updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(pair.first) - .Key(CreateSecondaryIndexKeyTuples( - itemArg, - indexTablePk, - inputColumns, - table, - fetch, - replace || skipErase, - ctx)) - .Update<TCoNameValueTupleList>() - .Add(dataColumnUpdates) - .Build() - .Done(); - - if (!mustUpdateDataColumn && mayBeSkipIndexUpdate) { - updateRowSecondaryIndex = Build<TCoIf>(ctx, node.Pos()) - .Predicate<TCoHasItems>() - .List<TCoToList>() - .Optional(fetch) - .Build() - .Build() - .ThenValue<TCoVoid>() - .Build() - .ElseValue(updateRowSecondaryIndex) - .Done(); - } - - TVector<TExprBase> keyToErase; - keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); - if (!skipErase && !mayBeSkipIndexUpdate) { - THashSet<TString> uniqColumns; - for (const auto& col : pair.second->KeyColumns) { - if (uniqColumns.insert(col).second) { - const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) - .Name().Build(col) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(col) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - } - - for (const auto& k : pk) { - if (uniqColumns.insert(k).second) { - const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) - .Name().Build(k) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(k) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - } - - const auto& conditionalErase = BuildConditionalErase( - fetch, itemArg, keyToErase, pair.first, node, ctx); - updates.push_back(conditionalErase); - } - updates.push_back(updateRowSecondaryIndex); - } - - return Build<TCoAsList>(ctx, node.Pos()) - .Add(updates) - .Done(); -} - + TExprContext& ctx, + const THashSet<TStringBuf>* filter = nullptr) +{ + return NKikimr::NKqp::BuildSecondaryIndexVector(table, pos, ctx, filter, &BuildVersionedTable); +} + +TExprBase BuildConditionalErase( + const TExprBase& condition, + const TCoArgument& itemArg, + const TVector<TExprBase>& keyToErase, + TExprNode::TPtr table, + const TKiWriteTable& node, + TExprContext& ctx) +{ + const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(table) + .Key<TCoNameValueTupleList>() + .Add(keyToErase) + .Build() + .Done(); + + return Build<TCoIfPresent>(ctx, node.Pos()) + .Optional(condition) + .PresentHandler<TCoLambda>() + .Args(itemArg) + .Body(erase) + .Build() + .MissingValue<TCoVoid>().Build() + .Done(); +} + +TExprBase CreateUpdateRowWithSecondaryIndex( + const TKiUpdateRow& updateTable, + const TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> indexes, + const TKikimrTableDescription& table, + const TCoNameValueTupleList& tablePk, + const THashSet<TStringBuf>& inputColumns, + const TCoArgument& itemArg, + const TKiWriteTable& node, TExprContext& ctx, bool replace, bool skipErase) +{ + TVector<TExprBase> updates; + updates.push_back(updateTable); + + const TVector<TString>& pk = table.Metadata->KeyColumnNames; + // in case of replace mode no need to read old data columns + const auto dataColumnSet = replace ? THashSet<TString>() : CreateDataColumnSetToRead(indexes, inputColumns); + const auto columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dataColumnSet, node.Pos(), ctx); + + const TExprBase& fetch = Build<TKiSelectRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(updateTable.Table()) + .Key(tablePk) + .template Select<TCoAtomList>() + .Add(columnsToSelect) + .Build() + .Done(); + + for (const auto& pair : indexes) { + TVector<TString> indexTablePk; + TVector<TCoNameValueTuple> dataColumnUpdates; + + // skip update of index table if indexed column not present in request + // and upserted rows already present + bool mayBeSkipIndexUpdate = !replace; + indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); + for (const auto& col : pair.second->KeyColumns) { + indexTablePk.push_back(col); + if (inputColumns.contains(col)) { + mayBeSkipIndexUpdate = false; + } + } + indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); + + // We can`t skip index update if data column present in query + // but in this case we may be need to skip erace + bool mustUpdateDataColumn = false; + + for (const auto& col : pair.second->DataColumns) { + if (inputColumns.contains(col)) { + mustUpdateDataColumn = true; + auto tuple = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build(col) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(col) + .Build() + .Done(); + dataColumnUpdates.push_back(tuple); + } else if (!replace) { + // Index table has data column, but data column has not been + // specifyed in request + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(col) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(col) + .Build() + .Done(); + dataColumnUpdates.push_back(tuple); + } + } + + TExprBase updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(pair.first) + .Key(CreateSecondaryIndexKeyTuples( + itemArg, + indexTablePk, + inputColumns, + table, + fetch, + replace || skipErase, + ctx)) + .Update<TCoNameValueTupleList>() + .Add(dataColumnUpdates) + .Build() + .Done(); + + if (!mustUpdateDataColumn && mayBeSkipIndexUpdate) { + updateRowSecondaryIndex = Build<TCoIf>(ctx, node.Pos()) + .Predicate<TCoHasItems>() + .List<TCoToList>() + .Optional(fetch) + .Build() + .Build() + .ThenValue<TCoVoid>() + .Build() + .ElseValue(updateRowSecondaryIndex) + .Done(); + } + + TVector<TExprBase> keyToErase; + keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); + if (!skipErase && !mayBeSkipIndexUpdate) { + THashSet<TString> uniqColumns; + for (const auto& col : pair.second->KeyColumns) { + if (uniqColumns.insert(col).second) { + const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build(col) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(col) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + } + + for (const auto& k : pk) { + if (uniqColumns.insert(k).second) { + const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build(k) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(k) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + } + + const auto& conditionalErase = BuildConditionalErase( + fetch, itemArg, keyToErase, pair.first, node, ctx); + updates.push_back(conditionalErase); + } + updates.push_back(updateRowSecondaryIndex); + } + + return Build<TCoAsList>(ctx, node.Pos()) + .Add(updates) + .Done(); +} + TExprNode::TPtr KiUpsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, const TKikimrTableDescription& table, - bool replace, bool skipErase, TExprNode::TPtr& effect) + bool replace, bool skipErase, TExprNode::TPtr& effect) { auto itemArg = Build<TCoArgument>(ctx, node.Pos()) .Name("item") @@ -249,13 +249,13 @@ TExprNode::TPtr KiUpsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, inputColumns.insert(atom.Value()); } - const auto& secondaryIndexes = BuildSecondaryIndexVector(table, node.Pos(), ctx); - + const auto& secondaryIndexes = BuildSecondaryIndexVector(table, node.Pos(), ctx); + TVector<TCoNameValueTuple> valueTuples; - - const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == node.Table()); - + + const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == node.Table()); + for (auto& pair : table.Metadata->Columns) { const TString& name = pair.first; @@ -285,39 +285,39 @@ TExprNode::TPtr KiUpsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, } } - const auto& tablePk = ExtractNamedKeyTuples(itemArg, table, ctx); - - // Update for main table - const auto& tableUpdate = Build<TKiUpdateRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(versionedTable) - .Key(tablePk) - .Update<TCoNameValueTupleList>() - .Add(valueTuples) + const auto& tablePk = ExtractNamedKeyTuples(itemArg, table, ctx); + + // Update for main table + const auto& tableUpdate = Build<TKiUpdateRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(versionedTable) + .Key(tablePk) + .Update<TCoNameValueTupleList>() + .Add(valueTuples) .Build() .Done(); - const auto& input = (skipErase || secondaryIndexes.empty()) ? node.Input() : RemoveDuplicateKeyFromInput(node.Input(), table, node.Pos(), ctx); - - effect = Build<TCoFlatMap>(ctx, node.Pos()) - .Input(input) - .Lambda<TCoLambda>() - .Args({itemArg}) - .Body(CreateUpdateRowWithSecondaryIndex( - tableUpdate, - secondaryIndexes, - table, - tablePk, - inputColumns, - itemArg, - node, - ctx, - replace, - skipErase) - ) - .Build() - .Done() - .Ptr(); + const auto& input = (skipErase || secondaryIndexes.empty()) ? node.Input() : RemoveDuplicateKeyFromInput(node.Input(), table, node.Pos(), ctx); + + effect = Build<TCoFlatMap>(ctx, node.Pos()) + .Input(input) + .Lambda<TCoLambda>() + .Args({itemArg}) + .Body(CreateUpdateRowWithSecondaryIndex( + tableUpdate, + secondaryIndexes, + table, + tablePk, + inputColumns, + itemArg, + node, + ctx, + replace, + skipErase) + ) + .Build() + .Done() + .Ptr(); return Build<TCoWorld>(ctx, node.Pos()) .Done() @@ -331,14 +331,14 @@ TExprNode::TPtr KiInsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, .Name("fetchItem") .Done(); - const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == node.Table()); - + const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == node.Table()); + auto fetchLambda = Build<TCoLambda>(ctx, node.Pos()) .Args(fetchItemArg) .Body<TKiSelectRow>() .Cluster(node.DataSink().Cluster()) - .Table(versionedTable) + .Table(versionedTable) .Key(ExtractNamedKeyTuples(fetchItemArg, table, ctx)) .Select() .Build() @@ -392,7 +392,7 @@ TExprNode::TPtr KiInsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, .Done(); TExprNode::TPtr insertEffect; - auto insertKql = KiUpsertTableToKql(node, ctx, table, false, true, insertEffect); + auto insertKql = KiUpsertTableToKql(node, ctx, table, false, true, insertEffect); if (op == TYdbOperation::InsertAbort) { effect = Build<TKiAbortIf>(ctx, node.Pos()) @@ -422,91 +422,91 @@ TExprNode::TPtr KiDeleteOnTableToKql(const TKiWriteTable& node, TExprContext& ct .Name("item") .Done(); - TVector<TExprBase> updates; - - const auto& tablePk = ExtractNamedKeyTuples(itemArg, table, ctx); - const TVector<TString>& pk = table.Metadata->KeyColumnNames; - - const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == node.Table()); - - updates.emplace_back(Build<TKiEraseRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(versionedTable) - .Key(tablePk) - .Done() - ); - - const auto& indexes = BuildSecondaryIndexVector(table, node.Pos(), ctx); - - if (indexes) { - // No need to read dataColumn - we just going to remove row - const THashSet<TString> dummyDataColumns; - const auto& columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dummyDataColumns, node.Pos(), ctx); - const TExprBase& fetch = Build<TKiSelectRow>(ctx, node.Pos()) + TVector<TExprBase> updates; + + const auto& tablePk = ExtractNamedKeyTuples(itemArg, table, ctx); + const TVector<TString>& pk = table.Metadata->KeyColumnNames; + + const auto versionedTable = BuildVersionedTable(*table.Metadata, node.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == node.Table()); + + updates.emplace_back(Build<TKiEraseRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(versionedTable) + .Key(tablePk) + .Done() + ); + + const auto& indexes = BuildSecondaryIndexVector(table, node.Pos(), ctx); + + if (indexes) { + // No need to read dataColumn - we just going to remove row + const THashSet<TString> dummyDataColumns; + const auto& columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dummyDataColumns, node.Pos(), ctx); + const TExprBase& fetch = Build<TKiSelectRow>(ctx, node.Pos()) .Cluster(node.DataSink().Cluster()) - .Table(versionedTable) - .Key(tablePk) - .template Select<TCoAtomList>() - .Add(columnsToSelect) - .Build() - .Done(); - - for (const auto& pair : indexes) { - - TVector<TExprBase> keyToErase; - keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); - - THashSet<TString> uniqColumns; - for (const auto& col : pair.second->KeyColumns) { - if (!uniqColumns.insert(col).second) { - continue; - } - const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) - .Name().Build(col) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(col) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - - for (const auto& k : pk) { - if (!uniqColumns.insert(k).second) { - continue; - } - const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) - .Name().Build(k) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(k) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - - const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) - .Cluster(node.DataSink().Cluster()) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToErase) - .Build() - .Done(); - - updates.push_back(erase); - } - - } - - effect = Build<TCoFlatMap>(ctx, node.Pos()) + .Table(versionedTable) + .Key(tablePk) + .template Select<TCoAtomList>() + .Add(columnsToSelect) + .Build() + .Done(); + + for (const auto& pair : indexes) { + + TVector<TExprBase> keyToErase; + keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); + + THashSet<TString> uniqColumns; + for (const auto& col : pair.second->KeyColumns) { + if (!uniqColumns.insert(col).second) { + continue; + } + const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build(col) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(col) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + + for (const auto& k : pk) { + if (!uniqColumns.insert(k).second) { + continue; + } + const auto& member = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build(k) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(k) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + + const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) + .Cluster(node.DataSink().Cluster()) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToErase) + .Build() + .Done(); + + updates.push_back(erase); + } + + } + + effect = Build<TCoFlatMap>(ctx, node.Pos()) .Input(node.Input()) - .Lambda<TCoLambda>() - .Args({itemArg}) - .Body<TCoAsList>() - .Add(updates) - .Build() - .Build() + .Lambda<TCoLambda>() + .Args({itemArg}) + .Body<TCoAsList>() + .Add(updates) + .Build() + .Build() .Done() .Ptr(); @@ -516,102 +516,102 @@ TExprNode::TPtr KiDeleteOnTableToKql(const TKiWriteTable& node, TExprContext& ct } TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns) { - const auto& read = right.Input().Cast<TKiReadTable>(); + const auto& read = right.Input().Cast<TKiReadTable>(); bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); TKikimrKey key(ctx); YQL_ENSURE(key.Extract(read.TableKey().Ref())); YQL_ENSURE(key.GetKeyType() == TKikimrKey::Type::Table); - const auto& cluster = read.DataSource().Cluster(); - const auto& table = key.GetTablePath(); - - const auto& tableDesc = tablesData.ExistingTable(TString(cluster), TString(table)); - - const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, read.Pos(), ctx); - YQL_ENSURE(versionedTable.Path().Value() == table); - - TMaybe<TString> secondaryIndex; - - if (const auto& view = key.GetView()) { - YQL_ENSURE(tableDesc.Metadata); - if (!ValidateTableHasIndex(tableDesc.Metadata, ctx, read.Pos())) { - return nullptr; - } - auto [metadata, state] = tableDesc.Metadata->GetIndexMetadata(view.GetRef()); - YQL_ENSURE(metadata, "unable to find metadta for index: " << view.GetRef()); - YQL_ENSURE(state == TIndexDescription::EIndexState::Ready - || state == TIndexDescription::EIndexState::WriteOnly); - if (state != TIndexDescription::EIndexState::Ready) { - auto err = TStringBuilder() - << "Requested index: " << view.GetRef() - << " is not ready to use"; - ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_INDEX_IS_NOT_READY, err)); - return nullptr; - } - secondaryIndex = metadata->Name; - } - - if (secondaryIndex) { - const auto& indexTableName = secondaryIndex.GetRef(); - const auto& keyTableDesc = tablesData.ExistingTable(TString(cluster), TString(indexTableName)); + const auto& cluster = read.DataSource().Cluster(); + const auto& table = key.GetTablePath(); + + const auto& tableDesc = tablesData.ExistingTable(TString(cluster), TString(table)); + + const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, read.Pos(), ctx); + YQL_ENSURE(versionedTable.Path().Value() == table); + + TMaybe<TString> secondaryIndex; + + if (const auto& view = key.GetView()) { + YQL_ENSURE(tableDesc.Metadata); + if (!ValidateTableHasIndex(tableDesc.Metadata, ctx, read.Pos())) { + return nullptr; + } + auto [metadata, state] = tableDesc.Metadata->GetIndexMetadata(view.GetRef()); + YQL_ENSURE(metadata, "unable to find metadta for index: " << view.GetRef()); + YQL_ENSURE(state == TIndexDescription::EIndexState::Ready + || state == TIndexDescription::EIndexState::WriteOnly); + if (state != TIndexDescription::EIndexState::Ready) { + auto err = TStringBuilder() + << "Requested index: " << view.GetRef() + << " is not ready to use"; + ctx.AddError(YqlIssue(ctx.GetPosition(read.Pos()), TIssuesIds::KIKIMR_INDEX_IS_NOT_READY, err)); + return nullptr; + } + secondaryIndex = metadata->Name; + } + + if (secondaryIndex) { + const auto& indexTableName = secondaryIndex.GetRef(); + const auto& keyTableDesc = tablesData.ExistingTable(TString(cluster), TString(indexTableName)); TKikimrKeyRange range(ctx, keyTableDesc); - - const auto& selectRange = Build<TKiSelectIndexRange>(ctx, read.Pos()) - .Cluster(cluster) - .Table(versionedTable) - .Range(range.ToRangeExpr(read, ctx)) + + const auto& selectRange = Build<TKiSelectIndexRange>(ctx, read.Pos()) + .Cluster(cluster) + .Table(versionedTable) + .Range(range.ToRangeExpr(read, ctx)) .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) - .Settings() - .Build() - .IndexName() - .Value(secondaryIndex.GetRef()) - .Build() - .Done(); - - if (unwrapValues) { - return UnwrapKiReadTableValues(selectRange, tableDesc, selectRange.Select(), ctx).Ptr(); - } else { - return TExprBase(selectRange).Ptr(); - } - } else { + .Settings() + .Build() + .IndexName() + .Value(secondaryIndex.GetRef()) + .Build() + .Done(); + + if (unwrapValues) { + return UnwrapKiReadTableValues(selectRange, tableDesc, selectRange.Select(), ctx).Ptr(); + } else { + return TExprBase(selectRange).Ptr(); + } + } else { TKikimrKeyRange range(ctx, tableDesc); - - const auto& selectRange = Build<TKiSelectRange>(ctx, read.Pos()) - .Cluster(cluster) - .Table(versionedTable) - .Range(range.ToRangeExpr(read, ctx)) + + const auto& selectRange = Build<TKiSelectRange>(ctx, read.Pos()) + .Cluster(cluster) + .Table(versionedTable) + .Range(range.ToRangeExpr(read, ctx)) .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) - .Settings() - .Build() - .Done(); - - if (unwrapValues) { - return UnwrapKiReadTableValues(selectRange, tableDesc, selectRange.Select(), ctx).Ptr(); - } else { - return TExprBase(selectRange).Ptr(); - } + .Settings() + .Build() + .Done(); + + if (unwrapValues) { + return UnwrapKiReadTableValues(selectRange, tableDesc, selectRange.Select(), ctx).Ptr(); + } else { + return TExprBase(selectRange).Ptr(); + } } } TExprNode::TPtr KiUpdateOnTableToKql(const TKiWriteTable& node, TExprContext& ctx, - const TKikimrTableDescription& tableDesc, TExprNode::TPtr& effect) + const TKikimrTableDescription& tableDesc, TExprNode::TPtr& effect) { // TODO: KIKIMR-3206 - // This function should be rewriten + // This function should be rewriten + + const auto& cluster = node.DataSink().Cluster(); - const auto& cluster = node.DataSink().Cluster(); - - const auto& itemArg = Build<TCoArgument>(ctx, node.Pos()) + const auto& itemArg = Build<TCoArgument>(ctx, node.Pos()) .Name("item") .Done(); - const auto& inputColumnsSetting = GetSetting(node.Settings().Ref(), "input_columns"); + const auto& inputColumnsSetting = GetSetting(node.Settings().Ref(), "input_columns"); YQL_ENSURE(inputColumnsSetting); TVector<TCoNameValueTuple> valueTuples; - THashSet<TStringBuf> updatedColumns; + THashSet<TStringBuf> updatedColumns; for (const auto& atom : TCoNameValueTuple(inputColumnsSetting).Value().Cast<TCoAtomList>()) { - if (tableDesc.GetKeyColumnIndex(TString(atom.Value()))) { + if (tableDesc.GetKeyColumnIndex(TString(atom.Value()))) { continue; } @@ -623,155 +623,155 @@ TExprNode::TPtr KiUpdateOnTableToKql(const TKiWriteTable& node, TExprContext& ct .Build() .Done(); valueTuples.push_back(tuple); - updatedColumns.insert(atom.Value()); + updatedColumns.insert(atom.Value()); } - // Returns index only if at least one of indexed columns for coresponding index has been updated. - const auto& indexes = BuildSecondaryIndexVector(tableDesc, node.Pos(), ctx, &updatedColumns); - const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; - - const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, node.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == node.Table()); - - const auto dataColumnSet = CreateDataColumnSetToRead(indexes, updatedColumns); - const TVector<TExprBase> columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dataColumnSet, node.Pos(), ctx); - const auto& fetch = Build<TKiSelectRow>(ctx, node.Pos()) - .Cluster(cluster) - .Table(versionedTable) - .Key(ExtractNamedKeyTuples(itemArg, tableDesc, ctx)) - .Select<TCoAtomList>() - .Add(columnsToSelect) - .Build() - .Done(); - - TVector<TExprBase> updates; - - updates.emplace_back(Build<TKiUpdateRow>(ctx, node.Pos()) - .Cluster(cluster) - .Table(versionedTable) - .Key(ExtractNamedKeyTuples(itemArg, tableDesc, ctx)) + // Returns index only if at least one of indexed columns for coresponding index has been updated. + const auto& indexes = BuildSecondaryIndexVector(tableDesc, node.Pos(), ctx, &updatedColumns); + const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; + + const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, node.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == node.Table()); + + const auto dataColumnSet = CreateDataColumnSetToRead(indexes, updatedColumns); + const TVector<TExprBase> columnsToSelect = NKikimr::NKqp::CreateColumnsToSelectToUpdateIndex(indexes, pk, dataColumnSet, node.Pos(), ctx); + const auto& fetch = Build<TKiSelectRow>(ctx, node.Pos()) + .Cluster(cluster) + .Table(versionedTable) + .Key(ExtractNamedKeyTuples(itemArg, tableDesc, ctx)) + .Select<TCoAtomList>() + .Add(columnsToSelect) + .Build() + .Done(); + + TVector<TExprBase> updates; + + updates.emplace_back(Build<TKiUpdateRow>(ctx, node.Pos()) + .Cluster(cluster) + .Table(versionedTable) + .Key(ExtractNamedKeyTuples(itemArg, tableDesc, ctx)) .Update<TCoNameValueTupleList>() .Add(valueTuples) .Build() - .Done() - ); - - if (indexes) { - for (const auto& pair : indexes) { - TVector<TString> indexTablePk; - indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); - for (const auto& col : pair.second->KeyColumns) { - indexTablePk.push_back(col); - } - indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); - - TVector<TExprBase> keyToAdd; - TVector<TExprBase> keyToErase; - TVector<TCoNameValueTuple> dataColumnUpdates; - - keyToAdd.reserve(indexTablePk.size()); - keyToErase.reserve(indexTablePk.size()); - dataColumnUpdates.reserve(pair.second->DataColumns.size()); - - for (const TString& name : pair.second->DataColumns) { - // Data column was specified in request - update it - if (updatedColumns.contains(name)) { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(name) - .Build() - .Done(); - dataColumnUpdates.push_back(tuple); - } else { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(name) - .Build() - .Done(); - dataColumnUpdates.push_back(tuple); - } - } - - THashSet<TString> uniqColumns; - for (const TString& name : indexTablePk) { - if (!uniqColumns.insert(name).second) { - continue; - } - const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(fetch) - .Name().Build(name) - .Build() - .Done(); - keyToErase.push_back(oldTuple); - if (updatedColumns.contains(name)) { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(name) - .Build() - .Done(); - keyToAdd.push_back(tuple); - } else { - keyToAdd.push_back(oldTuple); - } - } - - const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) - .Cluster(cluster) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToErase) - .Build() - .Done(); - - updates.push_back(erase); - - const auto& updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, node.Pos()) - .Cluster(cluster) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToAdd) - .Build() - .Update<TCoNameValueTupleList>() - .Add(dataColumnUpdates) - .Build() - .Done(); - - updates.push_back(updateRowSecondaryIndex); - - } - } - - const auto& updateLambda = Build<TCoLambda>(ctx, node.Pos()) - .Args(itemArg) - .Body<TCoIf>() - .Predicate<TCoHasItems>() - .List<TCoToList>() - .Optional(fetch) - .Build() - .Build() - .ThenValue<TCoAsList>() - .Add(updates) - .Build() - .ElseValue<TCoList>() - .ListType<TCoListType>() - .ItemType<TCoVoidType>() - .Build() - .Build() - .Build() + .Done() + ); + + if (indexes) { + for (const auto& pair : indexes) { + TVector<TString> indexTablePk; + indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); + for (const auto& col : pair.second->KeyColumns) { + indexTablePk.push_back(col); + } + indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); + + TVector<TExprBase> keyToAdd; + TVector<TExprBase> keyToErase; + TVector<TCoNameValueTuple> dataColumnUpdates; + + keyToAdd.reserve(indexTablePk.size()); + keyToErase.reserve(indexTablePk.size()); + dataColumnUpdates.reserve(pair.second->DataColumns.size()); + + for (const TString& name : pair.second->DataColumns) { + // Data column was specified in request - update it + if (updatedColumns.contains(name)) { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(name) + .Build() + .Done(); + dataColumnUpdates.push_back(tuple); + } else { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(name) + .Build() + .Done(); + dataColumnUpdates.push_back(tuple); + } + } + + THashSet<TString> uniqColumns; + for (const TString& name : indexTablePk) { + if (!uniqColumns.insert(name).second) { + continue; + } + const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(fetch) + .Name().Build(name) + .Build() + .Done(); + keyToErase.push_back(oldTuple); + if (updatedColumns.contains(name)) { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(name) + .Build() + .Done(); + keyToAdd.push_back(tuple); + } else { + keyToAdd.push_back(oldTuple); + } + } + + const auto& erase = Build<TKiEraseRow>(ctx, node.Pos()) + .Cluster(cluster) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToErase) + .Build() + .Done(); + + updates.push_back(erase); + + const auto& updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, node.Pos()) + .Cluster(cluster) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToAdd) + .Build() + .Update<TCoNameValueTupleList>() + .Add(dataColumnUpdates) + .Build() + .Done(); + + updates.push_back(updateRowSecondaryIndex); + + } + } + + const auto& updateLambda = Build<TCoLambda>(ctx, node.Pos()) + .Args(itemArg) + .Body<TCoIf>() + .Predicate<TCoHasItems>() + .List<TCoToList>() + .Optional(fetch) + .Build() + .Build() + .ThenValue<TCoAsList>() + .Add(updates) + .Build() + .ElseValue<TCoList>() + .ListType<TCoListType>() + .ItemType<TCoVoidType>() + .Build() + .Build() + .Build() .Build() .Done(); - effect = Build<TCoFlatMap>(ctx, node.Pos()) - .Input(node.Input()) - .Lambda(updateLambda) + effect = Build<TCoFlatMap>(ctx, node.Pos()) + .Input(node.Input()) + .Lambda(updateLambda) .Done() .Ptr(); @@ -810,43 +810,43 @@ TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, { YQL_ENSURE(update.Update().Ref().GetTypeAnn()); - const auto& cluster = update.DataSink().Cluster(); - const auto& table = update.Table(); - const auto& tableDesc = tablesData.ExistingTable(TString(cluster.Value()), TString(table.Value())); - const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; + const auto& cluster = update.DataSink().Cluster(); + const auto& table = update.Table(); + const auto& tableDesc = tablesData.ExistingTable(TString(cluster.Value()), TString(table.Value())); + const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; + + const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, update.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == update.Table()); + + auto schemaVersion = TStringBuilder() << tableDesc.Metadata->SchemaVersion; - const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, update.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == update.Table()); - - auto schemaVersion = TStringBuilder() << tableDesc.Metadata->SchemaVersion; - TKikimrKeyRange range(ctx, tableDesc); - const auto& selectRange = Build<TKiSelectRange>(ctx, update.Pos()) - .Cluster(cluster) - .Table(versionedTable) + const auto& selectRange = Build<TKiSelectRange>(ctx, update.Pos()) + .Cluster(cluster) + .Table(versionedTable) .Range(range.ToRangeExpr(update, ctx)) - .Select(BuildColumnsList(tableDesc, update.Pos(), ctx, withSystemColumns)) + .Select(BuildColumnsList(tableDesc, update.Pos(), ctx, withSystemColumns)) .Settings().Build() .Done(); - const auto& filter = Build<TCoFilter>(ctx, update.Pos()) + const auto& filter = Build<TCoFilter>(ctx, update.Pos()) .Input(selectRange) .Lambda(update.Filter()) .Done(); - const auto& itemArg = Build<TCoArgument>(ctx, update.Pos()) + const auto& itemArg = Build<TCoArgument>(ctx, update.Pos()) .Name("item") .Done(); TVector<TCoNameValueTuple> valueTuples; - const auto& updateResultType = update.Update().Ref().GetTypeAnn()->Cast<TStructExprType>(); - THashSet<TStringBuf> updatedColumns; - - for (const auto& item : updateResultType->GetItems()) { + const auto& updateResultType = update.Update().Ref().GetTypeAnn()->Cast<TStructExprType>(); + THashSet<TStringBuf> updatedColumns; + + for (const auto& item : updateResultType->GetItems()) { const auto& name = item->GetName(); - updatedColumns.insert(name); + updatedColumns.insert(name); - const auto& tuple = Build<TCoNameValueTuple>(ctx, update.Pos()) + const auto& tuple = Build<TCoNameValueTuple>(ctx, update.Pos()) .Name().Build(name) .Value<TCoMember>() .Struct<TExprApplier>() @@ -859,125 +859,125 @@ TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, valueTuples.push_back(tuple); } - const auto& indexes = BuildSecondaryIndexVector(tableDesc, update.Pos(), ctx, &updatedColumns); - - TVector<TExprBase> updates; - updates.emplace_back(Build<TKiUpdateRow>(ctx, update.Pos()) - .Cluster(cluster) - .Table(versionedTable) + const auto& indexes = BuildSecondaryIndexVector(tableDesc, update.Pos(), ctx, &updatedColumns); + + TVector<TExprBase> updates; + updates.emplace_back(Build<TKiUpdateRow>(ctx, update.Pos()) + .Cluster(cluster) + .Table(versionedTable) .Key(ExtractNamedKeyTuples(itemArg, tableDesc, ctx)) .Update<TCoNameValueTupleList>() .Add(valueTuples) .Build() - .Done() - ); - - if (indexes) { - for (const auto& pair : indexes) { - TVector<TString> indexTablePk; - indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); - for (const auto& col : pair.second->KeyColumns) { - indexTablePk.push_back(col); - } - indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); - - TVector<TExprBase> keyToAdd; - TVector<TExprBase> keyToErase; - TVector<TCoNameValueTuple> dataColumnsUpdates; - - keyToAdd.reserve(indexTablePk.size()); - keyToErase.reserve(indexTablePk.size()); - dataColumnsUpdates.reserve(pair.second->DataColumns.size()); - - for (const TString& name : pair.second->DataColumns) { - if (updatedColumns.contains(name)) { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct<TExprApplier>() - .Apply(update.Update()) - .With(0, itemArg) - .Build() - .Name().Build(name) - .Build() - .Done(); - dataColumnsUpdates.push_back(tuple); - } else { - const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(name) - .Build() - .Done(); - dataColumnsUpdates.push_back(oldTuple); - } - } - - THashSet<TString> uniqColumns; - for (const TString& name : indexTablePk) { - if (!uniqColumns.insert(name).second) { - continue; - } - const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(name) - .Build() - .Done(); - - keyToErase.push_back(oldTuple); - - if (updatedColumns.contains(name)) { - const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(name) - .Value<TCoMember>() - .Struct<TExprApplier>() - .Apply(update.Update()) - .With(0, itemArg) - .Build() - .Name().Build(name) - .Build() - .Done(); - keyToAdd.push_back(tuple); - } else { - keyToAdd.push_back(oldTuple); - } - } - - const auto& erase = Build<TKiEraseRow>(ctx, update.Pos()) - .Cluster(cluster) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToErase) - .Build() - .Done(); - updates.push_back(erase); - - const auto& updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, update.Pos()) - .Cluster(cluster) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToAdd) - .Build() - .Update<TCoNameValueTupleList>() - .Add(dataColumnsUpdates) - .Build() - .Done(); - - updates.push_back(updateRowSecondaryIndex); - } - } - - effect = Build<TCoFlatMap>(ctx, update.Pos()) + .Done() + ); + + if (indexes) { + for (const auto& pair : indexes) { + TVector<TString> indexTablePk; + indexTablePk.reserve(pair.second->KeyColumns.size() + pk.size()); + for (const auto& col : pair.second->KeyColumns) { + indexTablePk.push_back(col); + } + indexTablePk.insert(indexTablePk.end(), pk.begin(), pk.end()); + + TVector<TExprBase> keyToAdd; + TVector<TExprBase> keyToErase; + TVector<TCoNameValueTuple> dataColumnsUpdates; + + keyToAdd.reserve(indexTablePk.size()); + keyToErase.reserve(indexTablePk.size()); + dataColumnsUpdates.reserve(pair.second->DataColumns.size()); + + for (const TString& name : pair.second->DataColumns) { + if (updatedColumns.contains(name)) { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct<TExprApplier>() + .Apply(update.Update()) + .With(0, itemArg) + .Build() + .Name().Build(name) + .Build() + .Done(); + dataColumnsUpdates.push_back(tuple); + } else { + const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(name) + .Build() + .Done(); + dataColumnsUpdates.push_back(oldTuple); + } + } + + THashSet<TString> uniqColumns; + for (const TString& name : indexTablePk) { + if (!uniqColumns.insert(name).second) { + continue; + } + const auto& oldTuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(name) + .Build() + .Done(); + + keyToErase.push_back(oldTuple); + + if (updatedColumns.contains(name)) { + const auto& tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(name) + .Value<TCoMember>() + .Struct<TExprApplier>() + .Apply(update.Update()) + .With(0, itemArg) + .Build() + .Name().Build(name) + .Build() + .Done(); + keyToAdd.push_back(tuple); + } else { + keyToAdd.push_back(oldTuple); + } + } + + const auto& erase = Build<TKiEraseRow>(ctx, update.Pos()) + .Cluster(cluster) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToErase) + .Build() + .Done(); + updates.push_back(erase); + + const auto& updateRowSecondaryIndex = Build<TKiUpdateRow>(ctx, update.Pos()) + .Cluster(cluster) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToAdd) + .Build() + .Update<TCoNameValueTupleList>() + .Add(dataColumnsUpdates) + .Build() + .Done(); + + updates.push_back(updateRowSecondaryIndex); + } + } + + effect = Build<TCoFlatMap>(ctx, update.Pos()) .Input(filter) - .Lambda<TCoLambda>() - .Args({itemArg}) - .Body<TCoAsList>() - .Add(updates) - .Build() - .Build() + .Lambda<TCoLambda>() + .Args({itemArg}) + .Body<TCoAsList>() + .Add(updates) + .Build() + .Build() .Done() .Ptr(); @@ -989,103 +989,103 @@ TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, TExprNode::TPtr KiDeleteTableToKql(TKiDeleteTable del, TExprContext& ctx, const TKikimrTablesData& tablesData, TExprNode::TPtr& effect, bool withSystemColumns) { - const auto& cluster = del.DataSink().Cluster(); - const auto& table = del.Table(); - const auto& tableDesc = tablesData.ExistingTable(TString(cluster.Value()), TString(table.Value())); + const auto& cluster = del.DataSink().Cluster(); + const auto& table = del.Table(); + const auto& tableDesc = tablesData.ExistingTable(TString(cluster.Value()), TString(table.Value())); + + const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, del.Pos(), ctx); + YQL_ENSURE(versionedTable.Path() == table); - const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, del.Pos(), ctx); - YQL_ENSURE(versionedTable.Path() == table); - TKikimrKeyRange range(ctx, tableDesc); - const auto& selectRange = Build<TKiSelectRange>(ctx, del.Pos()) - .Cluster(cluster) - .Table(versionedTable) + const auto& selectRange = Build<TKiSelectRange>(ctx, del.Pos()) + .Cluster(cluster) + .Table(versionedTable) .Range(range.ToRangeExpr(del, ctx)) - .Select(BuildColumnsList(tableDesc, del.Pos(), ctx, withSystemColumns)) + .Select(BuildColumnsList(tableDesc, del.Pos(), ctx, withSystemColumns)) .Settings().Build() .Done(); - const auto& filter = Build<TCoFilter>(ctx, del.Pos()) + const auto& filter = Build<TCoFilter>(ctx, del.Pos()) .Input(selectRange) .Lambda(del.Filter()) .Done(); - const auto& itemArg = Build<TCoArgument>(ctx, del.Pos()) + const auto& itemArg = Build<TCoArgument>(ctx, del.Pos()) .Name("item") .Done(); - const auto& indexes = BuildSecondaryIndexVector(tableDesc, del.Pos(), ctx); - - TVector<TExprBase> updates; - updates.reserve(indexes.size() + 1); - - const auto& tablePk = ExtractNamedKeyTuples(itemArg, tableDesc, ctx); - updates.emplace_back(Build<TKiEraseRow>(ctx, del.Pos()) - .Cluster(cluster) - .Table(versionedTable) - .Key(tablePk) - .Done() - ); - - const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; - if (indexes) { - TVector<TExprBase> keyToErase; - for (const auto& pair : indexes) { - keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); - - THashSet<TString> uniqColumns; - for (const auto& col : pair.second->KeyColumns) { - if (!uniqColumns.insert(col).second) { - continue; - } - const auto& member = Build<TCoNameValueTuple>(ctx, del.Pos()) - .Name().Build(col) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(col) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - - for (const auto& k : pk) { - if (!uniqColumns.insert(k).second) { - continue; - } - const auto& member = Build<TCoNameValueTuple>(ctx, del.Pos()) - .Name().Build(k) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(k) - .Build() - .Done(); - keyToErase.emplace_back(TExprBase(member)); - } - - const auto& erase = Build<TKiEraseRow>(ctx, del.Pos()) - .Cluster(cluster) - .Table(pair.first) - .Key<TCoNameValueTupleList>() - .Add(keyToErase) - .Build() - .Done(); - - keyToErase.clear(); - updates.push_back(erase); - } - } - - effect = Build<TCoFlatMap>(ctx, del.Pos()) - .Input(filter) - .template Lambda<TCoLambda>() - .Args({itemArg}) - .template Body<TCoAsList>() - .Add(updates) - .Build() - .Build() - .Done() - .Ptr(); - + const auto& indexes = BuildSecondaryIndexVector(tableDesc, del.Pos(), ctx); + + TVector<TExprBase> updates; + updates.reserve(indexes.size() + 1); + + const auto& tablePk = ExtractNamedKeyTuples(itemArg, tableDesc, ctx); + updates.emplace_back(Build<TKiEraseRow>(ctx, del.Pos()) + .Cluster(cluster) + .Table(versionedTable) + .Key(tablePk) + .Done() + ); + + const TVector<TString>& pk = tableDesc.Metadata->KeyColumnNames; + if (indexes) { + TVector<TExprBase> keyToErase; + for (const auto& pair : indexes) { + keyToErase.reserve(pair.second->KeyColumns.size() + pk.size()); + + THashSet<TString> uniqColumns; + for (const auto& col : pair.second->KeyColumns) { + if (!uniqColumns.insert(col).second) { + continue; + } + const auto& member = Build<TCoNameValueTuple>(ctx, del.Pos()) + .Name().Build(col) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(col) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + + for (const auto& k : pk) { + if (!uniqColumns.insert(k).second) { + continue; + } + const auto& member = Build<TCoNameValueTuple>(ctx, del.Pos()) + .Name().Build(k) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(k) + .Build() + .Done(); + keyToErase.emplace_back(TExprBase(member)); + } + + const auto& erase = Build<TKiEraseRow>(ctx, del.Pos()) + .Cluster(cluster) + .Table(pair.first) + .Key<TCoNameValueTupleList>() + .Add(keyToErase) + .Build() + .Done(); + + keyToErase.clear(); + updates.push_back(erase); + } + } + + effect = Build<TCoFlatMap>(ctx, del.Pos()) + .Input(filter) + .template Lambda<TCoLambda>() + .Args({itemArg}) + .template Body<TCoAsList>() + .Add(updates) + .Build() + .Build() + .Done() + .Ptr(); + return Build<TCoWorld>(ctx, del.Pos()) .Done() .Ptr(); @@ -1204,45 +1204,45 @@ TExprBase UnwrapKiReadTableValues(TExprBase input, const TKikimrTableDescription .Done(); } -bool IsKeySelectorPkPrefix(NNodes::TCoLambda keySelector, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { - auto checkKey = [keySelector, &tableDesc, columns] (const TExprBase& key, ui32 index) { - if (!key.Maybe<TCoMember>()) { - return false; - } - - auto member = key.Cast<TCoMember>(); - if (member.Struct().Raw() != keySelector.Args().Arg(0).Raw()) { - return false; - } - - auto column = member.Name().StringValue(); - auto columnIndex = tableDesc.GetKeyColumnIndex(column); - if (!columnIndex || *columnIndex != index) { - return false; - } - - if (columns) { - columns->emplace_back(std::move(column)); - } - - return true; - }; - - auto lambdaBody = keySelector.Body(); - if (auto maybeTuple = lambdaBody.Maybe<TExprList>()) { - auto tuple = maybeTuple.Cast(); - for (size_t i = 0; i < tuple.Size(); ++i) { - if (!checkKey(tuple.Item(i), i)) { - return false; - } - } - } else { - if (!checkKey(lambdaBody, 0)) { - return false; - } - } - - return true; -} - +bool IsKeySelectorPkPrefix(NNodes::TCoLambda keySelector, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { + auto checkKey = [keySelector, &tableDesc, columns] (const TExprBase& key, ui32 index) { + if (!key.Maybe<TCoMember>()) { + return false; + } + + auto member = key.Cast<TCoMember>(); + if (member.Struct().Raw() != keySelector.Args().Arg(0).Raw()) { + return false; + } + + auto column = member.Name().StringValue(); + auto columnIndex = tableDesc.GetKeyColumnIndex(column); + if (!columnIndex || *columnIndex != index) { + return false; + } + + if (columns) { + columns->emplace_back(std::move(column)); + } + + return true; + }; + + auto lambdaBody = keySelector.Body(); + if (auto maybeTuple = lambdaBody.Maybe<TExprList>()) { + auto tuple = maybeTuple.Cast(); + for (size_t i = 0; i < tuple.Size(); ++i) { + if (!checkKey(tuple.Item(i), i)) { + return false; + } + } + } else { + if (!checkKey(lambdaBody, 0)) { + return false; + } + } + + return true; +} + } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp index 7fb42bd72b..4cd98892e4 100644 --- a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp @@ -5,26 +5,26 @@ #include <ydb/library/yql/core/yql_expr_type_annotation.h> #include <ydb/library/yql/core/yql_join.h> - + namespace NYql { namespace { using namespace NNodes; -TExprNode::TPtr BuildMkqlVersionedTable(const TCoAtom& tableName, const TCoAtom& schemaVersion, - const TCoAtom& pathId, TExprContext& ctx, TPositionHandle pos) -{ - return Build<TMkqlVersionedTable>(ctx, pos) - .Table(tableName) - .SchemaVersion(schemaVersion) - .PathId(pathId) - .Done() - .Ptr(); -} - +TExprNode::TPtr BuildMkqlVersionedTable(const TCoAtom& tableName, const TCoAtom& schemaVersion, + const TCoAtom& pathId, TExprContext& ctx, TPositionHandle pos) +{ + return Build<TMkqlVersionedTable>(ctx, pos) + .Table(tableName) + .SchemaVersion(schemaVersion) + .PathId(pathId) + .Done() + .Ptr(); +} + TExprNode::TPtr MkqlRewriteCallables(TCallable callable, TExprContext& ctx, const TMaybe<TString>& rtParamName) { TMaybeNode<TCoParameter> readTarget; - + if (rtParamName) { readTarget = Build<TCoParameter>(ctx, callable.Pos()) .Name().Build(*rtParamName) @@ -34,28 +34,28 @@ TExprNode::TPtr MkqlRewriteCallables(TCallable callable, TExprContext& ctx, cons .Done(); } - if (auto maybeSelectRow = callable.Maybe<TKiSelectRow>()) { - auto selectRow = maybeSelectRow.Cast(); - auto vt = selectRow.Table(); + if (auto maybeSelectRow = callable.Maybe<TKiSelectRow>()) { + auto selectRow = maybeSelectRow.Cast(); + auto vt = selectRow.Table(); TExprNode::TListType children = { - BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, selectRow.Pos()), - selectRow.Key().Ptr(), - selectRow.Select().Ptr() + BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, selectRow.Pos()), + selectRow.Key().Ptr(), + selectRow.Select().Ptr() }; if (readTarget) { children.push_back(readTarget.Cast().Ptr()); } - return ctx.NewCallable(selectRow.Pos(), "SelectRow", std::move(children)); + return ctx.NewCallable(selectRow.Pos(), "SelectRow", std::move(children)); } - if (auto maybeSelectRange = callable.Maybe<TKiSelectRangeBase>()) { - auto selectRange = maybeSelectRange.Cast(); - auto vt = selectRange.Table(); + if (auto maybeSelectRange = callable.Maybe<TKiSelectRangeBase>()) { + auto selectRange = maybeSelectRange.Cast(); + auto vt = selectRange.Table(); TExprNode::TListType children = { - BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, selectRange.Pos()), + BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, selectRange.Pos()), selectRange.Range().Ptr(), selectRange.Select().Ptr(), selectRange.Settings().Ptr() @@ -76,30 +76,30 @@ TExprNode::TPtr MkqlRewriteCallables(TCallable callable, TExprContext& ctx, cons .Build(); } else { ctx.AddError(TIssue(ctx.GetPosition(selectRange.Pos()), TStringBuilder() << "Got unsupported callable: " - << selectRange.CallableName())); - return nullptr; + << selectRange.CallableName())); + return nullptr; } } - if (auto maybeUpdateRow = callable.Maybe<TKiUpdateRow>()) { - auto updateRow = maybeUpdateRow.Cast(); - auto vt = updateRow.Table(); - return Build<TMkqlUpdateRow>(ctx, updateRow.Pos()) - .Table(BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, updateRow.Pos())) - .Key(updateRow.Key()) - .Update(updateRow.Update()) - .Done() - .Ptr(); + if (auto maybeUpdateRow = callable.Maybe<TKiUpdateRow>()) { + auto updateRow = maybeUpdateRow.Cast(); + auto vt = updateRow.Table(); + return Build<TMkqlUpdateRow>(ctx, updateRow.Pos()) + .Table(BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, updateRow.Pos())) + .Key(updateRow.Key()) + .Update(updateRow.Update()) + .Done() + .Ptr(); } - if (auto maybeEraseRow = callable.Maybe<TKiEraseRow>()) { - auto eraseRow = maybeEraseRow.Cast(); - auto vt = eraseRow.Table(); - return Build<TMkqlEraseRow>(ctx, eraseRow.Pos()) - .Table(BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, eraseRow.Pos())) - .Key(eraseRow.Key()) - .Done() - .Ptr(); + if (auto maybeEraseRow = callable.Maybe<TKiEraseRow>()) { + auto eraseRow = maybeEraseRow.Cast(); + auto vt = eraseRow.Table(); + return Build<TMkqlEraseRow>(ctx, eraseRow.Pos()) + .Table(BuildMkqlVersionedTable(vt.Path(), vt.SchemaVersion(), vt.PathId(), ctx, eraseRow.Pos())) + .Key(eraseRow.Key()) + .Done() + .Ptr(); } if (auto setResult = callable.Maybe<TKiSetResult>()) { @@ -120,11 +120,11 @@ TExprNode::TPtr MkqlRewriteCallables(TCallable callable, TExprContext& ctx, cons } if (auto map = callable.Maybe<TKiMapParameter>()) { - return Build<TMkqlMapParameter>(ctx, map.Cast().Pos()) - .Input(map.Cast().Input()) - .Lambda(map.Cast().Lambda()) - .Done() - .Ptr(); + return Build<TMkqlMapParameter>(ctx, map.Cast().Pos()) + .Input(map.Cast().Input()) + .Lambda(map.Cast().Lambda()) + .Done() + .Ptr(); } if (auto map = callable.Maybe<TKiFlatMapParameter>()) { diff --git a/ydb/core/kqp/provider/yql_kikimr_opt.cpp b/ydb/core/kqp/provider/yql_kikimr_opt.cpp index 90f1fb19be..cdbb6e600d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt.cpp @@ -11,10 +11,10 @@ namespace { using namespace NNodes; using namespace NCommon; -bool CanPushPartialSort(const TKiPartialSort& node, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { - return IsKeySelectorPkPrefix(node.KeySelectorLambda(), tableDesc, columns); -} - +bool CanPushPartialSort(const TKiPartialSort& node, const TKikimrTableDescription& tableDesc, TVector<TString>* columns) { + return IsKeySelectorPkPrefix(node.KeySelectorLambda(), tableDesc, columns); +} + TExprNode::TPtr KiTrimReadTableWorld(TExprBase node) { if (auto maybeRead = node.Maybe<TCoLeft>().Input().Maybe<TKiReadTable>()) { YQL_CLOG(INFO, ProviderKikimr) << "KiTrimReadTableWorld"; @@ -48,7 +48,7 @@ TExprNode::TPtr KiEraseOverSelectRow(TExprBase node, TExprContext& ctx) { } auto map = node.Cast<TCoFlatMap>(); - + if (auto maybeErase = map.Lambda().Body().Maybe<TCoJust>().Input().Maybe<TKiEraseRow>()) { auto selectRow = map.Input().Cast<TKiSelectRow>(); auto eraseRow = maybeErase.Cast(); @@ -69,34 +69,34 @@ TExprNode::TPtr KiEraseOverSelectRow(TExprBase node, TExprContext& ctx) { } } - if (auto maybeAsList = map.Lambda().Body().Maybe<TCoAsList>()) { - auto asList = maybeAsList.Cast(); - if (asList.ArgCount() != 1) { - return node.Ptr(); - } - - if (auto maybeErase = asList.Arg(0).Maybe<TKiEraseRow>()) { - auto selectRow = map.Input().Cast<TKiSelectRow>(); - auto eraseRow = maybeErase.Cast(); - - YQL_ENSURE(selectRow.Cluster().Raw() == eraseRow.Cluster().Raw()); - - if (selectRow.Table().Raw() == eraseRow.Table().Raw()) { - - auto ret = Build<TCoAsList>(ctx, node.Pos()) - .Add<TKiEraseRow>() - .Cluster(selectRow.Cluster()) - .Table(selectRow.Table()) - .Key(selectRow.Key()) - .Build() - .Done(); - - YQL_CLOG(INFO, ProviderKikimr) << "KiEraseOverSelectRow"; - return ret.Ptr(); - } - } - } - + if (auto maybeAsList = map.Lambda().Body().Maybe<TCoAsList>()) { + auto asList = maybeAsList.Cast(); + if (asList.ArgCount() != 1) { + return node.Ptr(); + } + + if (auto maybeErase = asList.Arg(0).Maybe<TKiEraseRow>()) { + auto selectRow = map.Input().Cast<TKiSelectRow>(); + auto eraseRow = maybeErase.Cast(); + + YQL_ENSURE(selectRow.Cluster().Raw() == eraseRow.Cluster().Raw()); + + if (selectRow.Table().Raw() == eraseRow.Table().Raw()) { + + auto ret = Build<TCoAsList>(ctx, node.Pos()) + .Add<TKiEraseRow>() + .Cluster(selectRow.Cluster()) + .Table(selectRow.Table()) + .Key(selectRow.Key()) + .Build() + .Done(); + + YQL_CLOG(INFO, ProviderKikimr) << "KiEraseOverSelectRow"; + return ret.Ptr(); + } + } + } + return node.Ptr(); } @@ -149,11 +149,11 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx, read = flatmap.Input(); } - if (!read.Maybe<TKiSelectRange>()) { + if (!read.Maybe<TKiSelectRange>()) { return node.Ptr(); } - auto selectRange = read.Cast<TKiSelectRange>(); + auto selectRange = read.Cast<TKiSelectRange>(); if (HasSetting(selectRange.Settings().Ref(), "Reverse")) { // N.B. when SelectRange has a Reverse option we cannot optimize @@ -196,8 +196,8 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx, } } - auto& tableData = tablesData.ExistingTable(selectRange.Cluster().StringValue(), selectRange.Table().Path().StringValue()); - + auto& tableData = tablesData.ExistingTable(selectRange.Cluster().StringValue(), selectRange.Table().Path().StringValue()); + auto checkKey = [keySelector, &tableData, &passthroughFields] (TExprBase key, ui32 index) { if (!key.Maybe<TCoMember>()) { return false; @@ -252,14 +252,14 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx, .Build() .Done(); - TExprNode::TPtr newSelect = Build<TKiSelectRange>(ctx, selectRange.Pos()) - .Cluster(selectRange.Cluster()) - .Table(selectRange.Table()) - .Range(selectRange.Range()) - .Select(selectRange.Select()) - .Settings(newSettings) - .Done() - .Ptr(); + TExprNode::TPtr newSelect = Build<TKiSelectRange>(ctx, selectRange.Pos()) + .Cluster(selectRange.Cluster()) + .Table(selectRange.Table()) + .Range(selectRange.Range()) + .Select(selectRange.Select()) + .Settings(newSettings) + .Done() + .Ptr(); YQL_CLOG(INFO, ProviderKikimr) << "KiRedundantSortByPkReverse"; @@ -271,7 +271,7 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx, .Lambda(flatmap.Lambda()) .Done().Ptr(); } else { - return newSelect; + return newSelect; } } @@ -359,7 +359,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont read = flatmap.Input(); } - if (!read.Maybe<TKiSelectRangeBase>()) { + if (!read.Maybe<TKiSelectRangeBase>()) { return node.Ptr(); } @@ -487,172 +487,172 @@ TExprNode::TPtr KiSimplifyRowKey(TExprBase node, TExprContext& ctx) { return node.Ptr(); } -TExprNode::TPtr DoRewriteSelectIndexRange(const TKiSelectIndexRange& selectIndexRange, - const TKikimrTablesData& tablesData, TExprContext& ctx, - const TVector<TString>& extraColumns, const std::function<TExprBase(const TExprBase&)>& middleFilter = {}) -{ - const auto pos = selectIndexRange.Pos(); - - const auto& cluster = selectIndexRange.Cluster().Value(); - const auto& versionedTable = selectIndexRange.Table(); - const auto& indexTableName = selectIndexRange.IndexName().Value(); - - const auto& tableDesc = tablesData.ExistingTable(TString(cluster), TString(versionedTable.Path())); - const auto& indexTableDesc = tablesData.ExistingTable(TString(cluster), TString(indexTableName)); - - const auto& fetchItemArg = Build<TCoArgument>(ctx, pos) - .Name("fetchItem") - .Done(); - - bool needDataRead = false; - for (const auto& col : selectIndexRange.Select()) { - if (!indexTableDesc.Metadata->Columns.contains(TString(col.Value()))) { - needDataRead = true; - break; - } - } - - const bool indexFullScan = TKikimrKeyRange::IsFull(selectIndexRange.Range()); - // Fullscan from index table but without reading data from main is OK - if (indexFullScan && needDataRead) { - auto issue = TIssue(ctx.GetPosition(pos), "Given predicate is not suitable for used index"); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_KIKIMR_WRONG_INDEX_USAGE, issue); - if (!ctx.AddWarning(issue)) { - return nullptr; - } - } - - auto keyColumnsList = needDataRead ? BuildKeyColumnsList(tableDesc, pos, ctx) : selectIndexRange.Select(); - auto columns = MergeColumns(keyColumnsList, extraColumns, ctx); - - TExprBase selectKeyRange = Build<TKiSelectRange>(ctx, pos) - .Cluster(selectIndexRange.Cluster()) - .Table(BuildVersionedTable(*indexTableDesc.Metadata, pos, ctx)) - .Range(selectIndexRange.Range()) - .Select(columns) - .Settings(selectIndexRange.Settings()) - .Done(); - - if (middleFilter) { - selectKeyRange = middleFilter(selectKeyRange); - } - - if (!needDataRead) { - return TExprBase(selectKeyRange).Ptr(); - } - - const auto& fetchLambda = Build<TCoLambda>(ctx, pos) - .Args(fetchItemArg) - .Body<TKiSelectRow>() - .Cluster() - .Value(cluster) - .Build() - .Table(versionedTable) - .Key(ExtractNamedKeyTuples(fetchItemArg, tableDesc, ctx)) - .Select(selectIndexRange.Select()) - .Build() - .Done(); - - const auto& flatMap = Build<TCoFlatMap>(ctx, pos) - .Input(selectKeyRange) - .Lambda(fetchLambda) - .Done(); - - YQL_CLOG(INFO, ProviderKikimr) << "KiRewriteSelectIndexRange"; - return TExprBase(flatMap).Ptr(); -} - -TExprNode::TPtr KiRewritePartialTakeSortOverSelectIndexRange(TExprBase node, const TKikimrTablesData& tablesData, TExprContext& ctx) { - auto maybePartialTake = node.Maybe<TKiPartialTake>(); - if (!maybePartialTake) { - return node.Ptr(); - } - - auto partialTake = maybePartialTake.Cast(); - - auto maybePartialSort = partialTake.Input().Maybe<TKiPartialSort>(); - if (!maybePartialSort) { - return node.Ptr(); - } - - auto partialSort = maybePartialSort.Cast(); - - auto maybeSelectIndexRange = partialSort.Input().Maybe<TKiSelectIndexRange>(); - if (!maybeSelectIndexRange) { - return node.Ptr(); - } - - auto selectIndexRange = maybeSelectIndexRange.Cast(); - - const auto cluster = selectIndexRange.Cluster().StringValue(); - const auto indexTableName = selectIndexRange.IndexName().StringValue(); - - const auto& indexDesc = tablesData.ExistingTable(cluster, indexTableName); - - TVector<TString> sortByColumns; - if (!CanPushPartialSort(maybePartialSort.Cast(), indexDesc, &sortByColumns)) { - return node.Ptr(); - } - - auto filter = [&ctx, &node, &partialSort, &partialTake](const TExprBase& in) mutable { - auto out = Build<TKiPartialTake>(ctx, node.Pos()) - .Input<TKiPartialSort>() - .Input(in) - .SortDirections(partialSort.SortDirections()) - .KeySelectorLambda(ctx.DeepCopyLambda(partialSort.KeySelectorLambda().Ref())) - .Build() - .Count(partialTake.Count()) - .Done(); - return TExprBase(out); - }; - - return DoRewriteSelectIndexRange(selectIndexRange, tablesData, ctx, sortByColumns, filter); -} - -TExprNode::TPtr KiRewriteSelectIndexRange(TExprBase node, const TKikimrTablesData& tablesData, TExprContext& ctx) { - if (auto maybeSelectIndexRange = node.Maybe<TKiSelectIndexRange>()) { - return DoRewriteSelectIndexRange(maybeSelectIndexRange.Cast(), tablesData, ctx, {}); - } - - return node.Ptr(); -} - -TExprNode::TPtr KiApplyExtractMembersToSelectRange(TExprBase node, TExprContext& ctx) { - if (!node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectRangeBase>()) { - return node.Ptr(); - } - - auto extract = node.Cast<TCoExtractMembers>(); - - if (node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectRange>()) { - auto range = extract.Input().Cast<TKiSelectRange>(); - - YQL_CLOG(INFO, ProviderKikimr) << "KiApplyExtractMembersToSelectRange"; - return Build<TKiSelectRange>(ctx, node.Pos()) - .Cluster(range.Cluster()) - .Table(range.Table()) - .Range(range.Range()) - .Select(extract.Members()) - .Settings(range.Settings()) - .Done().Ptr(); - } else if (node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectIndexRange>()) { - auto range = extract.Input().Cast<TKiSelectIndexRange>(); - - YQL_CLOG(INFO, ProviderKikimr) << "KiApplyExtractMembersToSelectRange"; - return Build<TKiSelectIndexRange>(ctx, node.Pos()) - .Cluster(range.Cluster()) - .Table(range.Table()) - .Range(range.Range()) - .Select(extract.Members()) - .Settings(range.Settings()) - .IndexName(range.IndexName()) - .Done().Ptr(); - } else { +TExprNode::TPtr DoRewriteSelectIndexRange(const TKiSelectIndexRange& selectIndexRange, + const TKikimrTablesData& tablesData, TExprContext& ctx, + const TVector<TString>& extraColumns, const std::function<TExprBase(const TExprBase&)>& middleFilter = {}) +{ + const auto pos = selectIndexRange.Pos(); + + const auto& cluster = selectIndexRange.Cluster().Value(); + const auto& versionedTable = selectIndexRange.Table(); + const auto& indexTableName = selectIndexRange.IndexName().Value(); + + const auto& tableDesc = tablesData.ExistingTable(TString(cluster), TString(versionedTable.Path())); + const auto& indexTableDesc = tablesData.ExistingTable(TString(cluster), TString(indexTableName)); + + const auto& fetchItemArg = Build<TCoArgument>(ctx, pos) + .Name("fetchItem") + .Done(); + + bool needDataRead = false; + for (const auto& col : selectIndexRange.Select()) { + if (!indexTableDesc.Metadata->Columns.contains(TString(col.Value()))) { + needDataRead = true; + break; + } + } + + const bool indexFullScan = TKikimrKeyRange::IsFull(selectIndexRange.Range()); + // Fullscan from index table but without reading data from main is OK + if (indexFullScan && needDataRead) { + auto issue = TIssue(ctx.GetPosition(pos), "Given predicate is not suitable for used index"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_KIKIMR_WRONG_INDEX_USAGE, issue); + if (!ctx.AddWarning(issue)) { + return nullptr; + } + } + + auto keyColumnsList = needDataRead ? BuildKeyColumnsList(tableDesc, pos, ctx) : selectIndexRange.Select(); + auto columns = MergeColumns(keyColumnsList, extraColumns, ctx); + + TExprBase selectKeyRange = Build<TKiSelectRange>(ctx, pos) + .Cluster(selectIndexRange.Cluster()) + .Table(BuildVersionedTable(*indexTableDesc.Metadata, pos, ctx)) + .Range(selectIndexRange.Range()) + .Select(columns) + .Settings(selectIndexRange.Settings()) + .Done(); + + if (middleFilter) { + selectKeyRange = middleFilter(selectKeyRange); + } + + if (!needDataRead) { + return TExprBase(selectKeyRange).Ptr(); + } + + const auto& fetchLambda = Build<TCoLambda>(ctx, pos) + .Args(fetchItemArg) + .Body<TKiSelectRow>() + .Cluster() + .Value(cluster) + .Build() + .Table(versionedTable) + .Key(ExtractNamedKeyTuples(fetchItemArg, tableDesc, ctx)) + .Select(selectIndexRange.Select()) + .Build() + .Done(); + + const auto& flatMap = Build<TCoFlatMap>(ctx, pos) + .Input(selectKeyRange) + .Lambda(fetchLambda) + .Done(); + + YQL_CLOG(INFO, ProviderKikimr) << "KiRewriteSelectIndexRange"; + return TExprBase(flatMap).Ptr(); +} + +TExprNode::TPtr KiRewritePartialTakeSortOverSelectIndexRange(TExprBase node, const TKikimrTablesData& tablesData, TExprContext& ctx) { + auto maybePartialTake = node.Maybe<TKiPartialTake>(); + if (!maybePartialTake) { + return node.Ptr(); + } + + auto partialTake = maybePartialTake.Cast(); + + auto maybePartialSort = partialTake.Input().Maybe<TKiPartialSort>(); + if (!maybePartialSort) { + return node.Ptr(); + } + + auto partialSort = maybePartialSort.Cast(); + + auto maybeSelectIndexRange = partialSort.Input().Maybe<TKiSelectIndexRange>(); + if (!maybeSelectIndexRange) { + return node.Ptr(); + } + + auto selectIndexRange = maybeSelectIndexRange.Cast(); + + const auto cluster = selectIndexRange.Cluster().StringValue(); + const auto indexTableName = selectIndexRange.IndexName().StringValue(); + + const auto& indexDesc = tablesData.ExistingTable(cluster, indexTableName); + + TVector<TString> sortByColumns; + if (!CanPushPartialSort(maybePartialSort.Cast(), indexDesc, &sortByColumns)) { + return node.Ptr(); + } + + auto filter = [&ctx, &node, &partialSort, &partialTake](const TExprBase& in) mutable { + auto out = Build<TKiPartialTake>(ctx, node.Pos()) + .Input<TKiPartialSort>() + .Input(in) + .SortDirections(partialSort.SortDirections()) + .KeySelectorLambda(ctx.DeepCopyLambda(partialSort.KeySelectorLambda().Ref())) + .Build() + .Count(partialTake.Count()) + .Done(); + return TExprBase(out); + }; + + return DoRewriteSelectIndexRange(selectIndexRange, tablesData, ctx, sortByColumns, filter); +} + +TExprNode::TPtr KiRewriteSelectIndexRange(TExprBase node, const TKikimrTablesData& tablesData, TExprContext& ctx) { + if (auto maybeSelectIndexRange = node.Maybe<TKiSelectIndexRange>()) { + return DoRewriteSelectIndexRange(maybeSelectIndexRange.Cast(), tablesData, ctx, {}); + } + + return node.Ptr(); +} + +TExprNode::TPtr KiApplyExtractMembersToSelectRange(TExprBase node, TExprContext& ctx) { + if (!node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectRangeBase>()) { + return node.Ptr(); + } + + auto extract = node.Cast<TCoExtractMembers>(); + + if (node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectRange>()) { + auto range = extract.Input().Cast<TKiSelectRange>(); + + YQL_CLOG(INFO, ProviderKikimr) << "KiApplyExtractMembersToSelectRange"; + return Build<TKiSelectRange>(ctx, node.Pos()) + .Cluster(range.Cluster()) + .Table(range.Table()) + .Range(range.Range()) + .Select(extract.Members()) + .Settings(range.Settings()) + .Done().Ptr(); + } else if (node.Maybe<TCoExtractMembers>().Input().Maybe<TKiSelectIndexRange>()) { + auto range = extract.Input().Cast<TKiSelectIndexRange>(); + + YQL_CLOG(INFO, ProviderKikimr) << "KiApplyExtractMembersToSelectRange"; + return Build<TKiSelectIndexRange>(ctx, node.Pos()) + .Cluster(range.Cluster()) + .Table(range.Table()) + .Range(range.Range()) + .Select(extract.Members()) + .Settings(range.Settings()) + .IndexName(range.IndexName()) + .Done().Ptr(); + } else { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unexpected callable")); - return nullptr; - } -} - + return nullptr; + } +} + } // namespace TAutoPtr<IGraphTransformer> CreateKiLogicalOptProposalTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx) { @@ -725,11 +725,11 @@ TAutoPtr<IGraphTransformer> CreateKiLogicalOptProposalTransformer(TIntrusivePtr< return ret; } - ret = KiRewritePartialTakeSortOverSelectIndexRange(node, sessionCtx->Tables(), ctx); - if (ret != inputNode) { - return ret; - } - + ret = KiRewritePartialTakeSortOverSelectIndexRange(node, sessionCtx->Tables(), ctx); + if (ret != inputNode) { + return ret; + } + return ret; }, ctx, TOptimizeExprSettings(nullptr)); @@ -781,7 +781,7 @@ TAutoPtr<IGraphTransformer> CreateKiPhysicalOptProposalTransformer(TIntrusivePtr return status; } - status = OptimizeExpr(input, output, [sessionCtx](const TExprNode::TPtr& inputNode, TExprContext& ctx) { + status = OptimizeExpr(input, output, [sessionCtx](const TExprNode::TPtr& inputNode, TExprContext& ctx) { Y_UNUSED(ctx); auto ret = inputNode; @@ -792,11 +792,11 @@ TAutoPtr<IGraphTransformer> CreateKiPhysicalOptProposalTransformer(TIntrusivePtr return ret; } - ret = KiRewriteSelectIndexRange(node, sessionCtx->Tables(), ctx); - if (ret != inputNode) { - return ret; - } - + ret = KiRewriteSelectIndexRange(node, sessionCtx->Tables(), ctx); + if (ret != inputNode) { + return ret; + } + return ret; }, ctx, TOptimizeExprSettings(nullptr)); diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index b9c58591fd..0ba0b34952 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -13,10 +13,10 @@ using namespace NCommon; TKiOperation BuildTableOpNode(const TCoAtom& cluster, const TStringBuf& table, TYdbOperation op, TPositionHandle pos, TExprContext& ctx) { - return Build<TKiOperation>(ctx, pos) - .Cluster().Build(cluster) - .Table().Build(table) - .Operation<TCoAtom>().Build(ToString(op)) + return Build<TKiOperation>(ctx, pos) + .Cluster().Build(cluster) + .Table().Build(table) + .Operation<TCoAtom>().Build(ToString(op)) .Done(); } @@ -67,7 +67,7 @@ struct TKiExploreTxResults { TVector<TExprBase> Sync; TVector<TExprBase> Results; TVector<TExprBase> Effects; - TVector<TKiOperation> TableOperations; + TVector<TKiOperation> TableOperations; bool HasExecute; THashSet<const TExprNode*> GetSyncSet() const { @@ -116,8 +116,8 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T return dataSink.Raw() == ds.Raw(); }; - auto cluster = dataSink.Cluster(); - + auto cluster = dataSink.Cluster(); + if (auto maybeRead = node.Maybe<TKiReadTable>()) { auto read = maybeRead.Cast(); if (!checkDataSource(read.DataSource())) { @@ -144,7 +144,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T txRes.Ops.insert(node.Raw()); auto result = ExploreTx(write.World(), ctx, dataSink, txRes); auto tableOp = GetTableOp(write); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, tableOp, write.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, tableOp, write.Pos(), ctx)); txRes.Effects.push_back(node); return result; } diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp index dac264a4b2..662082c543 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp @@ -113,22 +113,22 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c return false; } - static const struct { - const TStringBuf Left = "Left"; - const TStringBuf Inner = "Inner"; - const TStringBuf LeftSemi = "LeftSemi"; - const TStringBuf LeftOnly = "LeftOnly"; - const TStringBuf RightSemi = "RightSemi"; - } joinNames; - - static const TVector<TStringBuf> allowedJoins { - joinNames.Left, - joinNames.Inner, - joinNames.LeftSemi, - joinNames.LeftOnly, - joinNames.RightSemi - }; - + static const struct { + const TStringBuf Left = "Left"; + const TStringBuf Inner = "Inner"; + const TStringBuf LeftSemi = "LeftSemi"; + const TStringBuf LeftOnly = "LeftOnly"; + const TStringBuf RightSemi = "RightSemi"; + } joinNames; + + static const TVector<TStringBuf> allowedJoins { + joinNames.Left, + joinNames.Inner, + joinNames.LeftSemi, + joinNames.LeftOnly, + joinNames.RightSemi + }; + const TStringBuf joinType = joinTuple.Type().Value(); if (Find(allowedJoins, joinType) == allowedJoins.cend()) { return false; @@ -141,23 +141,23 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c auto rightExpr = GetEquiJoinInputList(joinTuple.RightScope().Cast<TCoAtom>(), joinLabels, joinInputs); - TMaybeNode<TKiSelectRangeBase> rightSelect; + TMaybeNode<TKiSelectRangeBase> rightSelect; TMaybeNode<TCoFlatMap> rightFlatmap; TMaybeNode<TCoFilterNullMembers> rightFilterNull; TMaybeNode<TCoSkipNullMembers> rightSkipNull; - TMaybeNode<TCoAtom> indexTable; + TMaybeNode<TCoAtom> indexTable; - if (auto select = rightExpr.Maybe<TKiSelectRangeBase>()) { + if (auto select = rightExpr.Maybe<TKiSelectRangeBase>()) { rightSelect = select; } - if (auto select = rightExpr.Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRangeBase>()) { + if (auto select = rightExpr.Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRangeBase>()) { rightSelect = select; rightFlatmap = rightExpr.Cast<TCoFlatMap>(); } if (auto select = rightExpr.Maybe<TCoFlatMap>().Input().Maybe<TCoFilterNullMembers>() - .Input().Maybe<TKiSelectRangeBase>()) + .Input().Maybe<TKiSelectRangeBase>()) { rightSelect = select; rightFlatmap = rightExpr.Cast<TCoFlatMap>(); @@ -165,17 +165,17 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c } if (auto select = rightExpr.Maybe<TCoFlatMap>().Input().Maybe<TCoSkipNullMembers>() - .Input().Maybe<TKiSelectRangeBase>()) + .Input().Maybe<TKiSelectRangeBase>()) { rightSelect = select; rightFlatmap = rightExpr.Cast<TCoFlatMap>(); rightSkipNull = rightFlatmap.Input().Cast<TCoSkipNullMembers>(); } - if (auto indexSelect = rightSelect.Maybe<TKiSelectIndexRange>()) { - indexTable = indexSelect.Cast().IndexName(); - } - + if (auto indexSelect = rightSelect.Maybe<TKiSelectIndexRange>()) { + indexTable = indexSelect.Cast().IndexName(); + } + if (!rightSelect) { return false; } @@ -187,7 +187,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c const auto selectRange = rightSelect.Cast(); const TStringBuf cluster = selectRange.Cluster().Value(); - const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value(); + const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value(); const TKikimrTableDescription& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTable); auto rightKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, lookupTableDesc, selectRange.Range()); @@ -227,7 +227,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c TSet<TString> leftKeyColumnsSet; TSet<TString> rightKeyColumnsSet; - TVector<TColumnRange> keyColumnRanges(lookupTableDesc.Metadata->KeyColumnNames.size()); + TVector<TColumnRange> keyColumnRanges(lookupTableDesc.Metadata->KeyColumnNames.size()); for (size_t i = 0; i < joinKeyCount; ++i) { auto joinLeftLabel = joinTuple.LeftKeys().Item(i * 2); @@ -247,7 +247,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c TString rightColumnName = ToString(joinRightColumn.Value()); - auto keyColumnIdx = lookupTableDesc.GetKeyColumnIndex(rightColumnName); + auto keyColumnIdx = lookupTableDesc.GetKeyColumnIndex(rightColumnName); if (!keyColumnIdx) { // Non-key columns in join key currently not supported return false; @@ -300,24 +300,24 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c TCoAtomList lookupColumns = selectRange.Select(); bool requireIndexValues = false; // 'true' means that some requested columns are not presented in the index-table, // so read from data-table is required - if (indexTable) { + if (indexTable) { // In this case lookupTableDesc == indexTable, // so check whether index-table contains all lookup-columns for (const auto& lookupColumn : lookupColumns) { - if (!lookupTableDesc.Metadata->Columns.contains(TString(lookupColumn.Value()))) { - requireIndexValues = true; - break; - } - } - } + if (!lookupTableDesc.Metadata->Columns.contains(TString(lookupColumn.Value()))) { + requireIndexValues = true; + break; + } + } + } auto selectedColumns = (indexTable && requireIndexValues) - ? BuildKeyColumnsList(lookupTableDesc, selectRange.Pos(), ctx) + ? BuildKeyColumnsList(lookupTableDesc, selectRange.Pos(), ctx) : lookupColumns; - + auto lookup = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, TKeyRange(ctx, keyColumnRanges, {}), selectedColumns, false /* allowNulls */, ctx); - + // Skip null keys in lookup part as for equijoin semantics null != null, // so we can't have nulls in lookup part lookup = Build<TCoSkipNullMembers>(ctx, joinTuple.Pos()) @@ -341,9 +341,9 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .Done(); } - // If we have index table AND need data from main we cand add this flat map here - // because lookup is index table and does not have all columns - if (rightFlatmap && !requireIndexValues) { + // If we have index table AND need data from main we cand add this flat map here + // because lookup is index table and does not have all columns + if (rightFlatmap && !requireIndexValues) { lookup = Build<TCoFlatMap>(ctx, joinTuple.Pos()) .Input(lookup) .Lambda(rightFlatmap.Cast().Lambda()) @@ -360,7 +360,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c YQL_ENSURE(maybeInput); auto input = *maybeInput; - + for (auto item: input->InputType->GetItems()) { joinColumns.emplace_back(item->GetName(), input->FullName(item->GetName())); } @@ -384,7 +384,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c if (resultColumns) { resultColumns->push_back(column); - } + } auto tuple = Build<TCoNameValueTuple>(ctx, joinTuple.Pos()) .Name().Build(column) @@ -397,12 +397,12 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c resultTuples.emplace_back(std::move(tuple)); } }; - + auto injectRightDataKey = [&tablesData, &selectRange, &ctx] (TMaybeNode<TExprBase>& rightRow, TVector<TExprBase>& joinResultTuples) { const TStringBuf cluster = selectRange.Cluster(); - const TStringBuf table = selectRange.Table().Path(); + const TStringBuf table = selectRange.Table().Path(); const auto& desc = tablesData.ExistingTable(cluster, table); for (const auto& col : desc.Metadata->KeyColumnNames) { auto tuple = Build<TCoNameValueTuple>(ctx, selectRange.Pos()) @@ -415,7 +415,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c joinResultTuples.push_back(tuple); } }; - + auto getJoinResultExpr = [requireIndexValues, &indexTable, &addJoinResults, &injectRightDataKey, &ctx, &joinTuple] (TMaybeNode<TExprBase> leftRowArg, TMaybeNode<TExprBase> rightRowArg) @@ -447,89 +447,89 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c return std::make_pair(expr, resultColumns); }; - + auto finalizeJoinResultExpr = [&joinTuple, &tablesData, &ctx, - &selectRange, &addJoinResults, &rightFlatmap, &joinType] - (const TExprBase& input, const TVector<TString>& finishedColumns, bool needExtraRead) - -> NNodes::TExprBase + &selectRange, &addJoinResults, &rightFlatmap, &joinType] + (const TExprBase& input, const TVector<TString>& finishedColumns, bool needExtraRead) + -> NNodes::TExprBase { if (!needExtraRead) { return input; } TCoArgument joinResult = Build<TCoArgument>(ctx, joinTuple.Pos()) - .Name("joinResult") - .Done(); - - TVector<TExprBase> joinResultTuples; - for (const auto& col : finishedColumns) { - auto tuple = Build<TCoNameValueTuple>(ctx, joinTuple.Pos()) - .Name().Build(col) - .Value<TCoMember>() + .Name("joinResult") + .Done(); + + TVector<TExprBase> joinResultTuples; + for (const auto& col : finishedColumns) { + auto tuple = Build<TCoNameValueTuple>(ctx, joinTuple.Pos()) + .Name().Build(col) + .Value<TCoMember>() .Struct(joinResult) - .Name().Build(col) - .Build() - .Done(); - joinResultTuples.push_back(tuple); - } - + .Name().Build(col) + .Build() + .Done(); + joinResultTuples.push_back(tuple); + } + const TStringBuf cluster = selectRange.Cluster(); - const TStringBuf table = selectRange.Table().Path(); + const TStringBuf table = selectRange.Table().Path(); const auto& tableDesc = tablesData.ExistingTable(cluster, table); - TExprBase select = Build<TKiSelectRow>(ctx, joinTuple.Pos()) + TExprBase select = Build<TKiSelectRow>(ctx, joinTuple.Pos()) .Cluster(selectRange.Cluster()) .Table(selectRange.Table()) .Key(ExtractNamedKeyTuples(joinResult, tableDesc, ctx, tableDesc.Metadata->Name)) .Select(selectRange.Select()) .Done(); - - if (rightFlatmap) { - select = Build<TCoFlatMap>(ctx, joinTuple.Pos()) - .Input(select) - .Lambda(rightFlatmap.Cast().Lambda()) - .Done(); - } - + + if (rightFlatmap) { + select = Build<TCoFlatMap>(ctx, joinTuple.Pos()) + .Input(select) + .Lambda(rightFlatmap.Cast().Lambda()) + .Done(); + } + addJoinResults(joinTuple.RightScope(), select, joinResultTuples, nullptr); - - if (joinType == joinNames.Inner || joinType == joinNames.RightSemi) { - return Build<TCoFlatMap>(ctx, joinTuple.Pos()) - .Input(input) - .Lambda() - .Args({joinResult}) - .Body<TCoOptionalIf>() - .Predicate<TCoHasItems>() - .List<TCoToList>() - .Optional(select) - .Build() - .Build() - .Value<TCoAsStruct>() - .Add(joinResultTuples) - .Build() - .Build() - .Build() - .Done(); - } else if (joinType == joinNames.Left){ - return Build<TCoMap>(ctx, joinTuple.Pos()) - .Input(input) - .Lambda() - .Args({joinResult}) - .Body<TCoAsStruct>() - .Add(joinResultTuples) - .Build() - .Build() - .Done(); - } else { - YQL_ENSURE(false, "unknown join type to call finalizeJoinResultExpr " << joinType); - } - }; - + + if (joinType == joinNames.Inner || joinType == joinNames.RightSemi) { + return Build<TCoFlatMap>(ctx, joinTuple.Pos()) + .Input(input) + .Lambda() + .Args({joinResult}) + .Body<TCoOptionalIf>() + .Predicate<TCoHasItems>() + .List<TCoToList>() + .Optional(select) + .Build() + .Build() + .Value<TCoAsStruct>() + .Add(joinResultTuples) + .Build() + .Build() + .Build() + .Done(); + } else if (joinType == joinNames.Left){ + return Build<TCoMap>(ctx, joinTuple.Pos()) + .Input(input) + .Lambda() + .Args({joinResult}) + .Body<TCoAsStruct>() + .Add(joinResultTuples) + .Build() + .Build() + .Done(); + } else { + YQL_ENSURE(false, "unknown join type to call finalizeJoinResultExpr " << joinType); + } + }; + auto leftExpr = getLeftExpr(); TCoArgument rightRowArg = Build<TCoArgument>(ctx, joinTuple.Pos()) .Name("rightRow") .Done(); - if (joinType == joinNames.Left) { + if (joinType == joinNames.Left) { TExprBase rightNothing = Build<TCoNothing>(ctx, joinTuple.Pos()) .OptionalType<TCoTypeOf>() .Value<TCoToOptional>() @@ -539,8 +539,8 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .Done(); auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg); - - auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos()) + + auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos()) .Input(leftExpr) .Lambda() .Args({leftRowArg}) @@ -552,23 +552,23 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .Input(lookup) .Lambda() .Args({rightRowArg}) - .Body(joinResultExpr.first) + .Body(joinResultExpr.first) .Build() .Build() .ElseValue<TCoAsList>() - .Add(getJoinResultExpr(leftRowArg, rightNothing).first) + .Add(getJoinResultExpr(leftRowArg, rightNothing).first) .Build() .Build() .Build() .Done(); - idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues); + idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues); return true; } - - if (joinType == joinNames.Inner) { + + if (joinType == joinNames.Inner) { const auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg); - auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos()) + auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos()) .Input<TCoSkipNullMembers>() .Input(leftExpr) .Members() @@ -581,16 +581,16 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .Input(lookup) .Lambda() .Args({rightRowArg}) - .Body(joinResultExpr.first) + .Body(joinResultExpr.first) .Build() .Build() .Build() .Done(); - idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues); + idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues); return true; } - if (joinType == joinNames.LeftSemi) { + if (joinType == joinNames.LeftSemi) { idxLookupExpr = Build<TCoFlatMap>(ctx, joinTuple.Pos()) .Input<TCoSkipNullMembers>() .Input(leftExpr) @@ -604,14 +604,14 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .Predicate<TCoHasItems>() .List(lookup) .Build() - .Value(getJoinResultExpr(leftRowArg, {}).first) + .Value(getJoinResultExpr(leftRowArg, {}).first) .Build() .Build() .Done(); return true; } - if (joinType == joinNames.LeftOnly) { + if (joinType == joinNames.LeftOnly) { idxLookupExpr = Build<TCoFlatMap>(ctx, joinTuple.Pos()) .Input(leftExpr) .Lambda() @@ -622,14 +622,14 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c .List(lookup) .Build() .Build() - .Value(getJoinResultExpr(leftRowArg, {}).first) + .Value(getJoinResultExpr(leftRowArg, {}).first) .Build() .Build() .Done(); return true; } - if (joinType == joinNames.RightSemi) { + if (joinType == joinNames.RightSemi) { // In this case we iterate over left table (with deduplication) // and do index-lookup in the right one. diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp index a2e64d64b3..cce81efc42 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp @@ -12,21 +12,21 @@ namespace { using namespace NNodes; using namespace NCommon; -struct TTakeNode { - const TExprBase Input; - const TExprBase Count; - const bool IsPartial; -}; - -TTakeNode GetTakeChildren(TExprBase node) { - if (auto maybeCoTake = node.Maybe<TCoTake>()) { - auto coTake = maybeCoTake.Cast(); - return TTakeNode {coTake.Input(), coTake.Count(), false}; - } else { - auto kiPartialTake = node.Maybe<TKiPartialTake>().Cast(); - return TTakeNode {kiPartialTake.Input(), kiPartialTake.Count(), true}; - } -} +struct TTakeNode { + const TExprBase Input; + const TExprBase Count; + const bool IsPartial; +}; + +TTakeNode GetTakeChildren(TExprBase node) { + if (auto maybeCoTake = node.Maybe<TCoTake>()) { + auto coTake = maybeCoTake.Cast(); + return TTakeNode {coTake.Input(), coTake.Count(), false}; + } else { + auto kiPartialTake = node.Maybe<TKiPartialTake>().Cast(); + return TTakeNode {kiPartialTake.Input(), kiPartialTake.Count(), true}; + } +} template<typename T> TTableLookup::TCompareResult::TResult CompareValues(const T& left, const T& right) { @@ -288,24 +288,24 @@ TTableLookup::TCompareResult KiTableLookupCompare(TExprBase left, TExprBase righ << ", " << right.Ref().Content() << ")"); } - + TExprNode::TPtr KiApplyLimitToSelectRange(TExprBase node, TExprContext& ctx) { - if (!node.Maybe<TCoTake>() && !node.Maybe<TKiPartialTake>()) { - return node.Ptr(); - } - - auto takeNode = GetTakeChildren(node); - - if (!takeNode.Input.Maybe<TCoSkip>().Input().Maybe<TKiSelectRangeBase>() && - !takeNode.Input.Maybe<TKiSelectRangeBase>()) + if (!node.Maybe<TCoTake>() && !node.Maybe<TKiPartialTake>()) { + return node.Ptr(); + } + + auto takeNode = GetTakeChildren(node); + + if (!takeNode.Input.Maybe<TCoSkip>().Input().Maybe<TKiSelectRangeBase>() && + !takeNode.Input.Maybe<TKiSelectRangeBase>()) { return node.Ptr(); } - auto maybeSkip = takeNode.Input.Maybe<TCoSkip>(); + auto maybeSkip = takeNode.Input.Maybe<TCoSkip>(); TMaybeNode<TExprBase> limitValue; - if (auto maybeTakeCount = takeNode.Count.Maybe<TCoUint64>()) { + if (auto maybeTakeCount = takeNode.Count.Maybe<TCoUint64>()) { ui64 totalLimit; auto takeValue = FromString<ui64>(maybeTakeCount.Cast().Literal().Value()); @@ -326,7 +326,7 @@ TExprNode::TPtr KiApplyLimitToSelectRange(TExprBase node, TExprContext& ctx) { .Build() .Done(); } else { - limitValue = takeNode.Count; + limitValue = takeNode.Count; if (maybeSkip) { limitValue = Build<TCoPlus>(ctx, node.Pos()) .Left(limitValue.Cast()) @@ -337,14 +337,14 @@ TExprNode::TPtr KiApplyLimitToSelectRange(TExprBase node, TExprContext& ctx) { YQL_ENSURE(limitValue); - auto input = maybeSkip - ? takeNode.Input.Cast<TCoSkip>().Input() - : takeNode.Input; + auto input = maybeSkip + ? takeNode.Input.Cast<TCoSkip>().Input() + : takeNode.Input; + + bool isIndexRange = input.Maybe<TKiSelectIndexRange>().IsValid(); + + auto select = input.Cast<TKiSelectRangeBase>(); - bool isIndexRange = input.Maybe<TKiSelectIndexRange>().IsValid(); - - auto select = input.Cast<TKiSelectRangeBase>(); - if (HasSetting(select.Settings().Ref(), "ItemsLimit")) { return node.Ptr(); } @@ -357,104 +357,104 @@ TExprNode::TPtr KiApplyLimitToSelectRange(TExprBase node, TExprContext& ctx) { .Build() .Done(); - TExprNode::TPtr newSelect; - if (isIndexRange) { - newSelect = Build<TKiSelectIndexRange>(ctx, select.Pos()) - .Cluster(select.Cluster()) - .Table(select.Table()) - .Range(select.Range()) - .Select(select.Select()) - .IndexName(input.Cast<TKiSelectIndexRange>().IndexName()) - .Settings(newSettings) - .Done() - .Ptr(); - } else { - newSelect = Build<TKiSelectRange>(ctx, select.Pos()) - .Cluster(select.Cluster()) - .Table(select.Table()) - .Range(select.Range()) - .Select(select.Select()) - .Settings(newSettings) - .Done() - .Ptr(); - } + TExprNode::TPtr newSelect; + if (isIndexRange) { + newSelect = Build<TKiSelectIndexRange>(ctx, select.Pos()) + .Cluster(select.Cluster()) + .Table(select.Table()) + .Range(select.Range()) + .Select(select.Select()) + .IndexName(input.Cast<TKiSelectIndexRange>().IndexName()) + .Settings(newSettings) + .Done() + .Ptr(); + } else { + newSelect = Build<TKiSelectRange>(ctx, select.Pos()) + .Cluster(select.Cluster()) + .Table(select.Table()) + .Range(select.Range()) + .Select(select.Select()) + .Settings(newSettings) + .Done() + .Ptr(); + } YQL_CLOG(INFO, ProviderKikimr) << "KiApplyLimitToSelectRange"; if (maybeSkip) { - if (takeNode.IsPartial) { - return Build<TKiPartialTake>(ctx, node.Pos()) - .Input<TCoSkip>() - .Input(newSelect) - .Count(maybeSkip.Cast().Count()) - .Build() - .Count(takeNode.Count) - .Done().Ptr(); - } else { - return Build<TCoTake>(ctx, node.Pos()) - .Input<TCoSkip>() - .Input(newSelect) - .Count(maybeSkip.Cast().Count()) - .Build() - .Count(takeNode.Count) - .Done().Ptr(); - } - } else { - if (takeNode.IsPartial) { - return Build<TKiPartialTake>(ctx, node.Pos()) + if (takeNode.IsPartial) { + return Build<TKiPartialTake>(ctx, node.Pos()) + .Input<TCoSkip>() + .Input(newSelect) + .Count(maybeSkip.Cast().Count()) + .Build() + .Count(takeNode.Count) + .Done().Ptr(); + } else { + return Build<TCoTake>(ctx, node.Pos()) + .Input<TCoSkip>() + .Input(newSelect) + .Count(maybeSkip.Cast().Count()) + .Build() + .Count(takeNode.Count) + .Done().Ptr(); + } + } else { + if (takeNode.IsPartial) { + return Build<TKiPartialTake>(ctx, node.Pos()) + .Input(newSelect) + .Count(takeNode.Count) + .Done().Ptr(); + + } else { + return Build<TCoTake>(ctx, node.Pos()) .Input(newSelect) - .Count(takeNode.Count) - .Done().Ptr(); - - } else { - return Build<TCoTake>(ctx, node.Pos()) - .Input(newSelect) - .Count(takeNode.Count) - .Done().Ptr(); - } + .Count(takeNode.Count) + .Done().Ptr(); + } } } -TExprNode::TPtr BuildFallbackSelectRange(TCoFlatMap oldFlatMap, - TKiSelectRangeBase select, TMaybeNode<TCoFilterNullMembers> filterNull, - TMaybeNode<TCoSkipNullMembers> skipNull, const TKikimrTableDescription& tableDesc, - TExprContext& ctx) -{ - TKikimrKeyRange range(ctx, tableDesc); - TExprNode::TPtr input; - - input = Build<TKiSelectRange>(ctx, select.Pos()) - .Cluster(select.Cluster()) - .Table(select.Table()) - .Range(range.ToRangeExpr(select, ctx)) - .Select(select.Select()) - .Settings(select.Settings()) - .Done() - .Ptr(); - - if (filterNull) { - input = Build<TCoFilterNullMembers>(ctx, oldFlatMap.Pos()) - .Input(input) - .Members(filterNull.Cast().Members()) - .Done() - .Ptr(); - } - - if (skipNull) { - input = Build<TCoSkipNullMembers>(ctx, oldFlatMap.Pos()) - .Input(input) - .Members(skipNull.Cast().Members()) - .Done() - .Ptr(); - } - - return Build<TCoFlatMap>(ctx, oldFlatMap.Pos()) - .Input(input) - .Lambda(oldFlatMap.Lambda()) - .Done() - .Ptr(); -} - +TExprNode::TPtr BuildFallbackSelectRange(TCoFlatMap oldFlatMap, + TKiSelectRangeBase select, TMaybeNode<TCoFilterNullMembers> filterNull, + TMaybeNode<TCoSkipNullMembers> skipNull, const TKikimrTableDescription& tableDesc, + TExprContext& ctx) +{ + TKikimrKeyRange range(ctx, tableDesc); + TExprNode::TPtr input; + + input = Build<TKiSelectRange>(ctx, select.Pos()) + .Cluster(select.Cluster()) + .Table(select.Table()) + .Range(range.ToRangeExpr(select, ctx)) + .Select(select.Select()) + .Settings(select.Settings()) + .Done() + .Ptr(); + + if (filterNull) { + input = Build<TCoFilterNullMembers>(ctx, oldFlatMap.Pos()) + .Input(input) + .Members(filterNull.Cast().Members()) + .Done() + .Ptr(); + } + + if (skipNull) { + input = Build<TCoSkipNullMembers>(ctx, oldFlatMap.Pos()) + .Input(input) + .Members(skipNull.Cast().Members()) + .Done() + .Ptr(); + } + + return Build<TCoFlatMap>(ctx, oldFlatMap.Pos()) + .Input(input) + .Lambda(oldFlatMap.Lambda()) + .Done() + .Ptr(); +} + TExprNode::TPtr KiPushPredicateToSelectRange(TExprBase node, TExprContext& ctx, const TKikimrTablesData& tablesData, const TKikimrConfiguration& config) { @@ -467,21 +467,21 @@ TExprNode::TPtr KiPushPredicateToSelectRange(TExprBase node, TExprContext& ctx, return node.Ptr(); } - TMaybeNode<TKiSelectRangeBase> select; + TMaybeNode<TKiSelectRangeBase> select; TMaybeNode<TCoFilterNullMembers> filterNull; TMaybeNode<TCoSkipNullMembers> skipNull; - TMaybeNode<TCoAtom> indexTable; + TMaybeNode<TCoAtom> indexTable; - if (auto maybeRange = flatmap.Input().Maybe<TKiSelectRangeBase>()) { + if (auto maybeRange = flatmap.Input().Maybe<TKiSelectRangeBase>()) { select = maybeRange.Cast(); } - if (auto maybeRange = flatmap.Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKiSelectRangeBase>()) { + if (auto maybeRange = flatmap.Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKiSelectRangeBase>()) { select = maybeRange.Cast(); filterNull = flatmap.Input().Cast<TCoFilterNullMembers>(); } - if (auto maybeRange = flatmap.Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKiSelectRangeBase>()) { + if (auto maybeRange = flatmap.Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKiSelectRangeBase>()) { select = maybeRange.Cast(); skipNull = flatmap.Input().Cast<TCoSkipNullMembers>(); } @@ -490,60 +490,60 @@ TExprNode::TPtr KiPushPredicateToSelectRange(TExprBase node, TExprContext& ctx, return node.Ptr(); } - if (auto indexSelect = select.Maybe<TKiSelectIndexRange>()) { - indexTable = indexSelect.Cast().IndexName(); - } - + if (auto indexSelect = select.Maybe<TKiSelectIndexRange>()) { + indexTable = indexSelect.Cast().IndexName(); + } + auto selectRange = select.Cast(); - const auto& cluster = TString(selectRange.Cluster()); - - const auto& lookupTableName = indexTable ? TString(indexTable.Cast()) : TString(selectRange.Table().Path()); - auto& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTableName); + const auto& cluster = TString(selectRange.Cluster()); + + const auto& lookupTableName = indexTable ? TString(indexTable.Cast()) : TString(selectRange.Table().Path()); + auto& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTableName); if (!TKikimrKeyRange::IsFull(selectRange.Range())) { return node.Ptr(); } - if (indexTable) { - bool needDataRead = false; - for (const auto& col : selectRange.Select()) { - if (!lookupTableDesc.Metadata->Columns.contains(TString(col.Value()))) { - needDataRead = true; - break; - } - } - - if (!needDataRead) { - // All selected data present in index table - // apply this optimizer again after TKiSelectIndexRange -> TKiSelectRange rewriting - return node.Ptr(); - } - } - + if (indexTable) { + bool needDataRead = false; + for (const auto& col : selectRange.Select()) { + if (!lookupTableDesc.Metadata->Columns.contains(TString(col.Value()))) { + needDataRead = true; + break; + } + } + + if (!needDataRead) { + // All selected data present in index table + // apply this optimizer again after TKiSelectIndexRange -> TKiSelectRange rewriting + return node.Ptr(); + } + } + auto row = flatmap.Lambda().Args().Arg(0); auto predicate = TExprBase(flatmap.Lambda().Body().Ref().ChildPtr(0)); - TTableLookup lookup = ExtractTableLookup(row, predicate, lookupTableDesc.Metadata->KeyColumnNames, + TTableLookup lookup = ExtractTableLookup(row, predicate, lookupTableDesc.Metadata->KeyColumnNames, &KiTableLookupGetValue, &KiTableLookupCanCompare, &KiTableLookupCompare, ctx, config.HasAllowNullCompareInIndex()); - auto& dataTableDesc = tablesData.ExistingTable(cluster, TString(selectRange.Table().Path())); - + auto& dataTableDesc = tablesData.ExistingTable(cluster, TString(selectRange.Table().Path())); + if (lookup.IsFullScan()) { - // WARNING: Sometimes we want to check index table usage here. - // Ok, but we must be aware about other optimizations. - // Example SqlIn using secondary index rewriten to EquiJoin on - // KiSelectIndexRange. The problem is if predicate contains also - // "AND non_index_fiels" clause - // (WHERE secondary_key IN (...) AND non_index == ) - // check for lookup.IsFullScan() returns true here because - // EquiJoin optimization has bot been made yet. - // See KIKIMR-11041 for more examples and consider KiRewriteSelectIndexRange method - + // WARNING: Sometimes we want to check index table usage here. + // Ok, but we must be aware about other optimizations. + // Example SqlIn using secondary index rewriten to EquiJoin on + // KiSelectIndexRange. The problem is if predicate contains also + // "AND non_index_fiels" clause + // (WHERE secondary_key IN (...) AND non_index == ) + // check for lookup.IsFullScan() returns true here because + // EquiJoin optimization has bot been made yet. + // See KIKIMR-11041 for more examples and consider KiRewriteSelectIndexRange method + return node.Ptr(); } TVector<TExprBase> fetches; - + for (auto& keyRange : lookup.GetKeyRanges()) { TExprBase predicate = keyRange.GetResidualPredicate() ? keyRange.GetResidualPredicate().Cast() @@ -553,56 +553,56 @@ TExprNode::TPtr KiPushPredicateToSelectRange(TExprBase node, TExprContext& ctx, auto newBody = ctx.ChangeChild(flatmap.Lambda().Body().Ref(), 0, predicate.Ptr()); - TExprNode::TPtr input; - - if (indexTable) { - if (!keyRange.IsEquiRange()) { - input = TKikimrKeyRange::BuildIndexReadRangeExpr(lookupTableDesc, keyRange, selectRange.Select(), - config.HasAllowNullCompareInIndex(), dataTableDesc, ctx).Ptr(); - } else { - TCoAtomList columnsToSelect = BuildKeyColumnsList(dataTableDesc, node.Pos(), ctx); - - input = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, keyRange, columnsToSelect, - config.HasAllowNullCompareInIndex(), ctx).Ptr(); - const auto& fetchItemArg = Build<TCoArgument>(ctx, node.Pos()) - .Name("fetchItem") - .Done(); - - input = Build<TCoFlatMap>(ctx, node.Pos()) - .Input(input) - .Lambda() - .Args(fetchItemArg) - .Body<TKiSelectRow>() - .Cluster(selectRange.Cluster()) - .Table(selectRange.Table()) - .Key(ExtractNamedKeyTuples(fetchItemArg, dataTableDesc, ctx)) - .Select(selectRange.Select()) - .Build() - .Build() - .Done() - .Ptr(); - } - } else { - input = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, keyRange, selectRange.Select(), - config.HasAllowNullCompareInIndex(), ctx).Ptr(); - } - - if (filterNull) { - input = Build<TCoFilterNullMembers>(ctx, node.Pos()) - .Input(input) - .Members(filterNull.Cast().Members()) - .Done() - .Ptr(); - } - - if (skipNull) { - input = Build<TCoSkipNullMembers>(ctx, node.Pos()) - .Input(input) - .Members(skipNull.Cast().Members()) - .Done() - .Ptr(); - } - + TExprNode::TPtr input; + + if (indexTable) { + if (!keyRange.IsEquiRange()) { + input = TKikimrKeyRange::BuildIndexReadRangeExpr(lookupTableDesc, keyRange, selectRange.Select(), + config.HasAllowNullCompareInIndex(), dataTableDesc, ctx).Ptr(); + } else { + TCoAtomList columnsToSelect = BuildKeyColumnsList(dataTableDesc, node.Pos(), ctx); + + input = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, keyRange, columnsToSelect, + config.HasAllowNullCompareInIndex(), ctx).Ptr(); + const auto& fetchItemArg = Build<TCoArgument>(ctx, node.Pos()) + .Name("fetchItem") + .Done(); + + input = Build<TCoFlatMap>(ctx, node.Pos()) + .Input(input) + .Lambda() + .Args(fetchItemArg) + .Body<TKiSelectRow>() + .Cluster(selectRange.Cluster()) + .Table(selectRange.Table()) + .Key(ExtractNamedKeyTuples(fetchItemArg, dataTableDesc, ctx)) + .Select(selectRange.Select()) + .Build() + .Build() + .Done() + .Ptr(); + } + } else { + input = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, keyRange, selectRange.Select(), + config.HasAllowNullCompareInIndex(), ctx).Ptr(); + } + + if (filterNull) { + input = Build<TCoFilterNullMembers>(ctx, node.Pos()) + .Input(input) + .Members(filterNull.Cast().Members()) + .Done() + .Ptr(); + } + + if (skipNull) { + input = Build<TCoSkipNullMembers>(ctx, node.Pos()) + .Input(input) + .Members(skipNull.Cast().Members()) + .Done() + .Ptr(); + } + auto fetch = Build<TCoFlatMap>(ctx, node.Pos()) .Input(input) .Lambda() @@ -644,8 +644,8 @@ TExprNode::TPtr KiApplyExtractMembersToSelectRow(TExprBase node, TExprContext& c TKikimrKeyRange::TKikimrKeyRange(TExprContext& ctx, const TKikimrTableDescription& table) : Table(table) - , KeyRange(ctx, Table.Metadata->KeyColumnNames.size(), TMaybeNode<TExprBase>()) -{} + , KeyRange(ctx, Table.Metadata->KeyColumnNames.size(), TMaybeNode<TExprBase>()) +{} TKikimrKeyRange::TKikimrKeyRange(const TKikimrTableDescription& table, const TKeyRange& keyRange) : Table(table) @@ -757,8 +757,8 @@ TExprList TKikimrKeyRange::ToRangeExpr(TExprBase owner, TExprContext& ctx) { TVector<TExprBase> columnRanges; for (size_t i = 0; i < KeyRange.GetColumnRangesCount(); ++i) { - const auto& column = Table.Metadata->KeyColumnNames[i]; - const auto& range = KeyRange.GetColumnRange(i); + const auto& column = Table.Metadata->KeyColumnNames[i]; + const auto& range = KeyRange.GetColumnRange(i); auto type = Table.GetColumnType(column); YQL_ENSURE(type); @@ -800,33 +800,33 @@ TExprList TKikimrKeyRange::ToRangeExpr(TExprBase owner, TExprContext& ctx) { .Done(); } -static TVector<TExprNodePtr> GetSkipNullKeys(const NCommon::TKeyRange& keyRange, - const TKikimrTableDescription& tableDesc, const NYql::TPositionHandle& pos, TExprContext& ctx) -{ - TVector<TExprNodePtr> skipNullKeys; - for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { - const auto& column = tableDesc.Metadata->KeyColumnNames[i]; - auto& range = keyRange.GetColumnRange(i); - if (range.IsDefined() && !range.IsNull()) { - skipNullKeys.push_back(ctx.NewAtom(pos, column)); - } - } - return skipNullKeys; -} - +static TVector<TExprNodePtr> GetSkipNullKeys(const NCommon::TKeyRange& keyRange, + const TKikimrTableDescription& tableDesc, const NYql::TPositionHandle& pos, TExprContext& ctx) +{ + TVector<TExprNodePtr> skipNullKeys; + for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { + const auto& column = tableDesc.Metadata->KeyColumnNames[i]; + auto& range = keyRange.GetColumnRange(i); + if (range.IsDefined() && !range.IsNull()) { + skipNullKeys.push_back(ctx.NewAtom(pos, column)); + } + } + return skipNullKeys; +} + NNodes::TExprBase TKikimrKeyRange::BuildReadRangeExpr(const TKikimrTableDescription& tableDesc, - const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, - TExprContext& ctx) + const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, + TExprContext& ctx) { YQL_ENSURE(tableDesc.Metadata); TString cluster = tableDesc.Metadata->Cluster; - const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, select.Pos(), ctx); - + const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, select.Pos(), ctx); + if (keyRange.IsEquiRange()) { TVector<TExprBase> columnTuples; for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { - const auto& column = tableDesc.Metadata->KeyColumnNames[i]; + const auto& column = tableDesc.Metadata->KeyColumnNames[i]; auto& range = keyRange.GetColumnRange(i); auto tuple = Build<TCoNameValueTuple>(ctx, select.Pos()) @@ -840,7 +840,7 @@ NNodes::TExprBase TKikimrKeyRange::BuildReadRangeExpr(const TKikimrTableDescript return Build<TCoToList>(ctx, select.Pos()) .Optional<TKiSelectRow>() .Cluster().Build(cluster) - .Table(versionedTable) + .Table(versionedTable) .Key<TCoNameValueTupleList>() .Add(columnTuples) .Build() @@ -850,7 +850,7 @@ NNodes::TExprBase TKikimrKeyRange::BuildReadRangeExpr(const TKikimrTableDescript } else { TVector<TExprNodePtr> skipNullKeys; if (!allowNulls) { - skipNullKeys = GetSkipNullKeys(keyRange, tableDesc, select.Pos(), ctx); + skipNullKeys = GetSkipNullKeys(keyRange, tableDesc, select.Pos(), ctx); } TVector<TCoNameValueTuple> settings; @@ -869,7 +869,7 @@ NNodes::TExprBase TKikimrKeyRange::BuildReadRangeExpr(const TKikimrTableDescript return Build<TKiSelectRange>(ctx, select.Pos()) .Cluster().Build(cluster) - .Table(versionedTable) + .Table(versionedTable) .Range(tableKeyRange.ToRangeExpr(select, ctx)) .Select(select) .Settings() @@ -879,51 +879,51 @@ NNodes::TExprBase TKikimrKeyRange::BuildReadRangeExpr(const TKikimrTableDescript } } -NNodes::TExprBase TKikimrKeyRange::BuildIndexReadRangeExpr(const TKikimrTableDescription& lookupTableDesc, - const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, - const TKikimrTableDescription& dataTableDesc, TExprContext& ctx) -{ - YQL_ENSURE(lookupTableDesc.Metadata); - YQL_ENSURE(dataTableDesc.Metadata); - TString cluster = lookupTableDesc.Metadata->Cluster; - - const auto versionedTable = BuildVersionedTable(*dataTableDesc.Metadata, select.Pos(), ctx); - - YQL_ENSURE(!keyRange.IsEquiRange()); - - TVector<TExprNodePtr> skipNullKeys; - if (!allowNulls) { - skipNullKeys = GetSkipNullKeys(keyRange, lookupTableDesc, select.Pos(), ctx); - } - - TVector<TCoNameValueTuple> settings; - if (!skipNullKeys.empty()) { - auto setting = Build<TCoNameValueTuple>(ctx, select.Pos()) - .Name().Build("SkipNullKeys") - .Value<TCoAtomList>() - .Add(skipNullKeys) - .Build() - .Done(); - - settings.push_back(setting); - } - - TKikimrKeyRange tableKeyRange(lookupTableDesc, keyRange); - - return Build<TKiSelectIndexRange>(ctx, select.Pos()) - .Cluster().Build(cluster) - .Table(versionedTable) - .Range(tableKeyRange.ToRangeExpr(select, ctx)) - .Select(select) - .Settings() - .Add(settings) - .Build() - .IndexName() - .Value(lookupTableDesc.Metadata->Name) - .Build() - .Done(); -} - +NNodes::TExprBase TKikimrKeyRange::BuildIndexReadRangeExpr(const TKikimrTableDescription& lookupTableDesc, + const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, + const TKikimrTableDescription& dataTableDesc, TExprContext& ctx) +{ + YQL_ENSURE(lookupTableDesc.Metadata); + YQL_ENSURE(dataTableDesc.Metadata); + TString cluster = lookupTableDesc.Metadata->Cluster; + + const auto versionedTable = BuildVersionedTable(*dataTableDesc.Metadata, select.Pos(), ctx); + + YQL_ENSURE(!keyRange.IsEquiRange()); + + TVector<TExprNodePtr> skipNullKeys; + if (!allowNulls) { + skipNullKeys = GetSkipNullKeys(keyRange, lookupTableDesc, select.Pos(), ctx); + } + + TVector<TCoNameValueTuple> settings; + if (!skipNullKeys.empty()) { + auto setting = Build<TCoNameValueTuple>(ctx, select.Pos()) + .Name().Build("SkipNullKeys") + .Value<TCoAtomList>() + .Add(skipNullKeys) + .Build() + .Done(); + + settings.push_back(setting); + } + + TKikimrKeyRange tableKeyRange(lookupTableDesc, keyRange); + + return Build<TKiSelectIndexRange>(ctx, select.Pos()) + .Cluster().Build(cluster) + .Table(versionedTable) + .Range(tableKeyRange.ToRangeExpr(select, ctx)) + .Select(select) + .Settings() + .Add(settings) + .Build() + .IndexName() + .Value(lookupTableDesc.Metadata->Name) + .Build() + .Done(); +} + TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData, const TKikimrConfiguration& config, TExprContext& ctx) { @@ -937,15 +937,15 @@ TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesDat auto flatMap = node.Cast<TCoFlatMap>(); // SqlIn expected to be rewritten to (FlatMap <collection> (OptionalIf ...)) - // or (FlatMap <collection> (FlatListIf ...)) - if (!flatMap.Lambda().Body().Maybe<TCoOptionalIf>() && !flatMap.Lambda().Body().Maybe<TCoFlatListIf>()) { + // or (FlatMap <collection> (FlatListIf ...)) + if (!flatMap.Lambda().Body().Maybe<TCoOptionalIf>() && !flatMap.Lambda().Body().Maybe<TCoFlatListIf>()) { return node.Ptr(); } if (!flatMap.Input().Maybe<TKiSelectRangeBase>()) { return node.Ptr(); } - + auto selectRange = flatMap.Input().Cast<TKiSelectRangeBase>(); TMaybeNode<TCoAtom> indexTable; @@ -954,7 +954,7 @@ TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesDat } // retrieve selected ranges - const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value(); + const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value(); const TKikimrTableDescription& tableDesc = tablesData.ExistingTable(selectRange.Cluster().Value(), lookupTable); auto selectKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, tableDesc, selectRange.Range()); if (!selectKeyRange) { diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index fe1fedc382..635d164827 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -43,7 +43,7 @@ struct TKikimrData { DataSinkNames.insert(TKiUpdateTable::CallableName()); DataSinkNames.insert(TKiDeleteTable::CallableName()); DataSinkNames.insert(TKiCreateTable::CallableName()); - DataSinkNames.insert(TKiAlterTable::CallableName()); + DataSinkNames.insert(TKiAlterTable::CallableName()); DataSinkNames.insert(TKiDropTable::CallableName()); DataSinkNames.insert(TKiCreateUser::CallableName()); DataSinkNames.insert(TKiAlterUser::CallableName()); @@ -57,7 +57,7 @@ struct TKikimrData { KqlNames.insert(TKiSelectRow::CallableName()); KqlNames.insert(TKiSelectRange::CallableName()); - KqlNames.insert(TKiSelectIndexRange::CallableName()); + KqlNames.insert(TKiSelectIndexRange::CallableName()); KqlNames.insert(TKiUpdateRow::CallableName()); KqlNames.insert(TKiEraseRow::CallableName()); KqlNames.insert(TKiSetResult::CallableName()); @@ -150,7 +150,7 @@ TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster TKikimrTableDescription& TKikimrTablesData::GetTable(const TString& cluster, const TString& table) { auto desc = Tables.FindPtr(std::make_pair(cluster, table)); - YQL_ENSURE(desc, "Unexpected empty metadata, cluster '" << cluster << "', table '" << table << "'"); + YQL_ENSURE(desc, "Unexpected empty metadata, cluster '" << cluster << "', table '" << table << "'"); return *desc; } @@ -355,44 +355,44 @@ bool TKikimrKey::Extract(const TExprNode& key) { return false; } - if (key.ChildrenSize() > 1) { - for (ui32 i = 1; i < key.ChildrenSize(); ++i) { - auto tag = key.Child(i)->Child(0); + if (key.ChildrenSize() > 1) { + for (ui32 i = 1; i < key.ChildrenSize(); ++i) { + auto tag = key.Child(i)->Child(0); if (tag->Content() == TStringBuf("view")) { - const TExprNode* viewNode = key.Child(i)->Child(1); - if (!viewNode->IsCallable("String")) { + const TExprNode* viewNode = key.Child(i)->Child(1); + if (!viewNode->IsCallable("String")) { Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Pos()), "Expected String")); - return false; - } - - if (viewNode->ChildrenSize() != 1 || !EnsureAtom(*viewNode->Child(0), Ctx)) { + return false; + } + + if (viewNode->ChildrenSize() != 1 || !EnsureAtom(*viewNode->Child(0), Ctx)) { Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Dynamic views names are not supported")); - return false; - } + return false; + } if (viewNode->Child(0)->Content().empty()) { Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Secondary index name must not be empty")); - return false; - } - View = viewNode->Child(0)->Content(); - - } else { + return false; + } + View = viewNode->Child(0)->Content(); + + } else { Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), TStringBuilder() << "Unexpected tag for kikimr key child: " << tag->Content())); - return false; - } - } - } + return false; + } + } + } return true; } -NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metadata, TPositionHandle pos, TExprContext& ctx) { - return Build<TKiVersionedTable>(ctx, pos) +NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metadata, TPositionHandle pos, TExprContext& ctx) { + return Build<TKiVersionedTable>(ctx, pos) .Path().Build(metadata.Name) .SchemaVersion().Build(ToString(metadata.SchemaVersion)) - .PathId().Build(metadata.PathId.ToString()) - .Done(); -} - -NNodes::TCoAtomList BuildColumnsList( + .PathId().Build(metadata.PathId.ToString()) + .Done(); +} + +NNodes::TCoAtomList BuildColumnsList( const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx, @@ -417,73 +417,73 @@ NNodes::TCoAtomList BuildColumnsList( } } - return Build<TCoAtomList>(ctx, pos) + return Build<TCoAtomList>(ctx, pos) .Add(columnsToSelect) .Done(); -} +} NNodes::TCoAtomList BuildKeyColumnsList(const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx) { - TVector<TExprBase> columnsToSelect; - columnsToSelect.reserve(table.Metadata->KeyColumnNames.size()); - for (auto key : table.Metadata->KeyColumnNames) { - auto value = table.Metadata->Columns.at(key); - auto atom = Build<TCoAtom>(ctx, pos) - .Value(value.Name) - .Done(); - - columnsToSelect.push_back(atom); - } - - return Build<TCoAtomList>(ctx, pos) - .Add(columnsToSelect) - .Done(); -} - -NNodes::TCoAtomList MergeColumns(const NNodes::TCoAtomList& col1, const TVector<TString>& col2, TExprContext& ctx) { - TVector<TCoAtom> columns; - THashSet<TString> uniqColumns; - columns.reserve(col1.Size() + col2.size()); - - for (const auto& c : col1) { - YQL_ENSURE(uniqColumns.emplace(c.StringValue()).second); - columns.push_back(c); - } - - for (const auto& c : col2) { - if (uniqColumns.emplace(c).second) { - auto atom = Build<TCoAtom>(ctx, col1.Pos()) - .Value(c) - .Done(); - columns.push_back(atom); - } - } - - return Build<TCoAtomList>(ctx, col1.Pos()) - .Add(columns) - .Done(); -} - -TCoNameValueTupleList ExtractNamedKeyTuples(TCoArgument itemArg, const TKikimrTableDescription& tableDesc, - TExprContext& ctx, const TString& tablePrefix) -{ - TVector<TExprBase> keyTuples; - for (TString& keyColumnName : tableDesc.Metadata->KeyColumnNames) { - auto tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) - .Name().Build(keyColumnName) - .Value<TCoMember>() - .Struct(itemArg) - .Name().Build(tablePrefix.empty() ? keyColumnName : TString::Join(tablePrefix, ".", keyColumnName)) - .Build() - .Done(); - - keyTuples.push_back(tuple); - } - - return Build<TCoNameValueTupleList>(ctx, itemArg.Pos()) - .Add(keyTuples) - .Done(); -} - + TVector<TExprBase> columnsToSelect; + columnsToSelect.reserve(table.Metadata->KeyColumnNames.size()); + for (auto key : table.Metadata->KeyColumnNames) { + auto value = table.Metadata->Columns.at(key); + auto atom = Build<TCoAtom>(ctx, pos) + .Value(value.Name) + .Done(); + + columnsToSelect.push_back(atom); + } + + return Build<TCoAtomList>(ctx, pos) + .Add(columnsToSelect) + .Done(); +} + +NNodes::TCoAtomList MergeColumns(const NNodes::TCoAtomList& col1, const TVector<TString>& col2, TExprContext& ctx) { + TVector<TCoAtom> columns; + THashSet<TString> uniqColumns; + columns.reserve(col1.Size() + col2.size()); + + for (const auto& c : col1) { + YQL_ENSURE(uniqColumns.emplace(c.StringValue()).second); + columns.push_back(c); + } + + for (const auto& c : col2) { + if (uniqColumns.emplace(c).second) { + auto atom = Build<TCoAtom>(ctx, col1.Pos()) + .Value(c) + .Done(); + columns.push_back(atom); + } + } + + return Build<TCoAtomList>(ctx, col1.Pos()) + .Add(columns) + .Done(); +} + +TCoNameValueTupleList ExtractNamedKeyTuples(TCoArgument itemArg, const TKikimrTableDescription& tableDesc, + TExprContext& ctx, const TString& tablePrefix) +{ + TVector<TExprBase> keyTuples; + for (TString& keyColumnName : tableDesc.Metadata->KeyColumnNames) { + auto tuple = Build<TCoNameValueTuple>(ctx, itemArg.Pos()) + .Name().Build(keyColumnName) + .Value<TCoMember>() + .Struct(itemArg) + .Name().Build(tablePrefix.empty() ? keyColumnName : TString::Join(tablePrefix, ".", keyColumnName)) + .Build() + .Done(); + + keyTuples.push_back(tuple); + } + + return Build<TCoNameValueTupleList>(ctx, itemArg.Pos()) + .Add(keyTuples) + .Done(); +} + TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TCoNameValueTupleList& operations, TExprContext& ctx) { TVector<NKqpProto::TKqpTableOp> protoOps; for (const auto& op : operations) { @@ -505,37 +505,37 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TCoNameValueTupleLi TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TKiOperationList& operations, TExprContext& ctx) { TVector<NKqpProto::TKqpTableOp> protoOps; - for (const auto& op : operations) { - auto table = TString(op.Table()); + for (const auto& op : operations) { + auto table = TString(op.Table()); auto tableOp = FromString<TYdbOperation>(TString(op.Operation())); auto pos = ctx.GetPosition(op.Pos()); - + NKqpProto::TKqpTableOp protoOp; protoOp.MutablePosition()->SetRow(pos.Row); protoOp.MutablePosition()->SetColumn(pos.Column); - protoOp.SetTable(table); - protoOp.SetOperation((ui32)tableOp); - - protoOps.push_back(protoOp); - } - - return protoOps; -} - + protoOp.SetTable(table); + protoOp.SetOperation((ui32)tableOp); + + protoOps.push_back(protoOp); + } + + return protoOps; +} + void TableDescriptionToTableInfo(const TKikimrTableDescription& desc, NKqpProto::TKqpTableInfo* info) { - YQL_ENSURE(desc.Metadata); - info->SetTableName(desc.Metadata->Name); + YQL_ENSURE(desc.Metadata); + info->SetTableName(desc.Metadata->Name); info->MutableTableId()->SetOwnerId(desc.Metadata->PathId.OwnerId()); info->MutableTableId()->SetTableId(desc.Metadata->PathId.TableId()); - info->SetSchemaVersion(desc.Metadata->SchemaVersion); - if (desc.Metadata->Indexes) { - info->SetHasIndexTables(true); - } -} - + info->SetSchemaVersion(desc.Metadata->SchemaVersion); + if (desc.Metadata->Indexes) { + info->SetHasIndexTables(true); + } +} + bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto::TKqpTableOp>& operations, const TVector<NKqpProto::TKqpTableInfo>& tableInfos, NKikimrKqp::EIsolationLevel isolationLevel, bool strictDml, - EKikimrQueryType queryType, TExprContext& ctx) + EKikimrQueryType queryType, TExprContext& ctx) { if (IsClosed()) { TString message = TStringBuilder() << "Cannot perform operations on closed transaction."; @@ -555,27 +555,27 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto } THashMap<TString, NKqpProto::TKqpTableInfo> tableInfoMap; - for (const auto& info : tableInfos) { - tableInfoMap.insert(std::make_pair(info.GetTableName(), info)); + for (const auto& info : tableInfos) { + tableInfoMap.insert(std::make_pair(info.GetTableName(), info)); TKikimrPathId pathId(info.GetTableId().GetOwnerId(), info.GetTableId().GetTableId()); TableByIdMap.insert(std::make_pair(pathId, info.GetTableName())); - } - + } + for (const auto& op : operations) { - const auto& table = op.GetTable(); - + const auto& table = op.GetTable(); + auto newOp = TYdbOperation(op.GetOperation()); TPosition pos(op.GetPosition().GetColumn(), op.GetPosition().GetRow()); - const auto info = tableInfoMap.FindPtr(table); - if (!info) { - TString message = TStringBuilder() - << "Unable to find table info for table '" << table << "'"; - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_SCHEME_ERROR, message)); - return false; - } - + const auto info = tableInfoMap.FindPtr(table); + if (!info) { + TString message = TStringBuilder() + << "Unable to find table info for table '" << table << "'"; + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_SCHEME_ERROR, message)); + return false; + } + if (queryType == EKikimrQueryType::Dml && (newOp & KikimrSchemeOps())) { TString message = TStringBuilder() << "Operation '" << newOp << "' can't be performed in data query"; @@ -612,7 +612,7 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto } auto& currentOps = TableOperations[table]; - + if (currentOps & KikimrModifyOps()) { if (KikimrRequireUnmodifiedOps() & newOp) { TString message = TStringBuilder() << "Operation '" << newOp @@ -628,12 +628,12 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto return false; } } - - if (info->GetHasIndexTables()) { - TString message = TStringBuilder() << "Multiple modification of table with secondary indexes is not supported yet"; - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_OPERATION, message)); - return false; - } + + if (info->GetHasIndexTables()) { + TString message = TStringBuilder() << "Multiple modification of table with secondary indexes is not supported yet"; + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_OPERATION, message)); + return false; + } } if ((KikimrRequireUnmodifiedOps() & newOp) && isolationLevel != NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE) { @@ -720,15 +720,15 @@ bool IsKikimrSystemColumn(const TStringBuf columnName) { return KikimrSystemColumns().FindPtr(columnName); } -bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos) { - if (metadata->Indexes.empty()) { - ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() - << "No global indexes for table " << metadata->Name)); - return false; - } - return true; -} - +bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos) { + if (metadata->Indexes.empty()) { + ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() + << "No global indexes for table " << metadata->Name)); + return false; + } + return true; +} + bool AddDmlIssue(const TIssue& issue, bool strictDml, TExprContext& ctx) { if (strictDml) { TIssue newIssue; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index d9955495aa..6903e6f193 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -48,7 +48,7 @@ public: TMaybe<bool> UseScanQuery; EKikimrStatsMode StatsMode = EKikimrStatsMode::None; TMaybe<bool> DocumentApiRestricted; - TMaybe<NKikimrKqp::TRlPath> RlPath; + TMaybe<NKikimrKqp::TRlPath> RlPath; bool ForceNewEngine() const { return UseNewEngine && *UseNewEngine; } }; @@ -69,7 +69,7 @@ enum class EKikimrQueryType { Unspecified = 0, Dml, Ddl, - YqlScript, + YqlScript, YqlInternal, Scan, YqlScriptStreaming, @@ -109,7 +109,7 @@ struct TKikimrQueryContext : TThrRefBase { TVector<ui64> ExecutionOrder; NActors::TActorId ReplyTarget; - TMaybe<NKikimrKqp::TRlPath> RlPath; + TMaybe<NKikimrKqp::TRlPath> RlPath; TIntrusivePtr<ITimeProvider> TimeProvider; TIntrusivePtr<IRandomProvider> RandomProvider; @@ -167,7 +167,7 @@ struct TKikimrQueryContext : TThrRefBase { InProgress.clear(); ExecutionOrder.clear(); - RlPath.Clear(); + RlPath.Clear(); CachedNow.reset(); std::get<0>(CachedRandom).reset(); @@ -266,7 +266,7 @@ public: virtual bool ApplyTableOperations(const TVector<NKqpProto::TKqpTableOp>& operations, const TVector<NKqpProto::TKqpTableInfo>& tableInfo, NKikimrKqp::EIsolationLevel isolationLevel, - bool strictDml, EKikimrQueryType queryType, TExprContext& ctx) = 0; + bool strictDml, EKikimrQueryType queryType, TExprContext& ctx) = 0; virtual ~IKikimrTransactionContext() = default; }; @@ -312,7 +312,7 @@ public: } bool ApplyTableOperations(const TVector<NKqpProto::TKqpTableOp>& operations, const TVector<NKqpProto::TKqpTableInfo>& tableInfo, - NKikimrKqp::EIsolationLevel isolationLevel, bool strictDml, EKikimrQueryType queryType, TExprContext& ctx) override; + NKikimrKqp::EIsolationLevel isolationLevel, bool strictDml, EKikimrQueryType queryType, TExprContext& ctx) override; }; class TKikimrSessionContext : public TThrRefBase { diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index 2bddfdf045..e1aa3cda2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -32,7 +32,7 @@ private: virtual TStatus HandleUpdateTable(NNodes::TKiUpdateTable node, TExprContext& ctx) = 0; virtual TStatus HandleDeleteTable(NNodes::TKiDeleteTable node, TExprContext& ctx) = 0; virtual TStatus HandleCreateTable(NNodes::TKiCreateTable node, TExprContext& ctx) = 0; - virtual TStatus HandleAlterTable(NNodes::TKiAlterTable node, TExprContext& ctx) = 0; + virtual TStatus HandleAlterTable(NNodes::TKiAlterTable node, TExprContext& ctx) = 0; virtual TStatus HandleDropTable(NNodes::TKiDropTable node, TExprContext& ctx) = 0; virtual TStatus HandleCreateUser(NNodes::TKiCreateUser node, TExprContext& ctx) = 0; virtual TStatus HandleAlterUser(NNodes::TKiAlterUser node, TExprContext& ctx) = 0; @@ -84,17 +84,17 @@ public: return Target; } - const TMaybe<TString>& GetView() const { - return View; - } - + const TMaybe<TString>& GetView() const { + return View; + } + bool Extract(const TExprNode& key); private: TExprContext& Ctx; TMaybe<Type> KeyType; TString Target; - TMaybe<TString> View; + TMaybe<TString> View; }; struct TKiExecDataQuerySettings { @@ -115,11 +115,11 @@ public: static bool IsFull(NNodes::TExprList list); static TMaybe<NCommon::TKeyRange> GetPointKeyRange(TExprContext& ctx, const TKikimrTableDescription& table, NNodes::TExprList range); static NNodes::TExprBase BuildReadRangeExpr(const TKikimrTableDescription& tableDesc, - const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, - TExprContext& ctx); - static NNodes::TExprBase BuildIndexReadRangeExpr(const TKikimrTableDescription& lookupTableDesc, - const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, - const TKikimrTableDescription& dataTableDesc, TExprContext& ctx); + const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, + TExprContext& ctx); + static NNodes::TExprBase BuildIndexReadRangeExpr(const TKikimrTableDescription& lookupTableDesc, + const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, + const TKikimrTableDescription& dataTableDesc, TExprContext& ctx); NNodes::TExprList ToRangeExpr(NNodes::TExprBase owner, TExprContext& ctx); @@ -204,28 +204,28 @@ NNodes::TMaybeNode<NNodes::TExprBase> TranslateToMkql(NNodes::TExprBase node, TE NNodes::TExprBase UnwrapKiReadTableValues(NNodes::TExprBase input, const TKikimrTableDescription& tableDesc, NNodes::TCoAtomList columns, TExprContext& ctx); -NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metadata, TPositionHandle pos, TExprContext& ctx); -NNodes::TCoAtomList BuildColumnsList( +NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metadata, TPositionHandle pos, TExprContext& ctx); +NNodes::TCoAtomList BuildColumnsList( const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx, bool withSystemColumns = false ); -NNodes::TCoAtomList BuildKeyColumnsList( - const TKikimrTableDescription& table, - TPositionHandle pos, - TExprContext& ctx -); - -NNodes::TCoAtomList MergeColumns(const NNodes::TCoAtomList& col1, const TVector<TString>& col2, TExprContext& ctx); - +NNodes::TCoAtomList BuildKeyColumnsList( + const TKikimrTableDescription& table, + TPositionHandle pos, + TExprContext& ctx +); + +NNodes::TCoAtomList MergeColumns(const NNodes::TCoAtomList& col1, const TVector<TString>& col2, TExprContext& ctx); + bool IsKqlPureExpr(NNodes::TExprBase expr); bool IsKqlPureLambda(NNodes::TCoLambda lambda); -bool IsKeySelectorPkPrefix(NNodes::TCoLambda lambda, const TKikimrTableDescription& desc, TVector<TString>* columns); +bool IsKeySelectorPkPrefix(NNodes::TCoLambda lambda, const TKikimrTableDescription& desc, TVector<TString>* columns); + +NNodes::TCoNameValueTupleList ExtractNamedKeyTuples(NNodes::TCoArgument arg, + const TKikimrTableDescription& desc, TExprContext& ctx, const TString& tablePrefix = TString()); -NNodes::TCoNameValueTupleList ExtractNamedKeyTuples(NNodes::TCoArgument arg, - const TKikimrTableDescription& desc, TExprContext& ctx, const TString& tablePrefix = TString()); - const TTypeAnnotationNode* GetReadTableRowType(TExprContext& ctx, const TKikimrTablesData& tablesData, const TString& cluster, const TString& table, NNodes::TCoAtomList select, bool withSystemColumns = false); @@ -279,6 +279,6 @@ const TYdbOperations& KikimrRequireUnmodifiedOps(); const TMap<TString, NKikimr::NUdf::EDataSlot>& KikimrSystemColumns(); bool IsKikimrSystemColumn(const TStringBuf columnName); -bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos); - +bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos); + } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_results.cpp b/ydb/core/kqp/provider/yql_kikimr_results.cpp index 7f3ce488e4..8ec93f416d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_results.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_results.cpp @@ -330,18 +330,18 @@ NKikimrMiniKQL::TResult GetKikimrUnpackedRunResult(const NKikimrMiniKQL::TResult return result; } -TVector<NKikimrMiniKQL::TResult*> UnpackKikimrRunResult(const NKikimrMiniKQL::TResult& runResult, google::protobuf::Arena* arena) { +TVector<NKikimrMiniKQL::TResult*> UnpackKikimrRunResult(const NKikimrMiniKQL::TResult& runResult, google::protobuf::Arena* arena) { const NKikimrMiniKQL::TTupleType* tupleType; const NKikimrMiniKQL::TValue* tupleValue; UnpackKikimrRunResult(runResult, tupleType, tupleValue); - TVector<NKikimrMiniKQL::TResult*> results; + TVector<NKikimrMiniKQL::TResult*> results; ui32 resultsSize = tupleType->ElementSize(); for (ui32 i = 0; i < resultsSize; ++i) { - results.push_back(google::protobuf::Arena::CreateMessage<NKikimrMiniKQL::TResult>(arena)); - NKikimrMiniKQL::TResult* result = results.back(); - *result->MutableType() = tupleType->GetElement(i); - *result->MutableValue() = tupleValue->GetTuple(i); + results.push_back(google::protobuf::Arena::CreateMessage<NKikimrMiniKQL::TResult>(arena)); + NKikimrMiniKQL::TResult* result = results.back(); + *result->MutableType() = tupleType->GetElement(i); + *result->MutableValue() = tupleValue->GetTuple(i); } return results; @@ -370,11 +370,11 @@ bool IsRawKikimrResult(const NKikimrMiniKQL::TResult& result) { return structType.GetMember(0).GetName() != "Data" || structType.GetMember(1).GetName() != "Truncated"; } -NKikimrMiniKQL::TResult* KikimrResultToProto(const NKikimrMiniKQL::TResult& result, const TVector<TString>& columnHints, - const IDataProvider::TFillSettings& fillSettings, google::protobuf::Arena* arena) +NKikimrMiniKQL::TResult* KikimrResultToProto(const NKikimrMiniKQL::TResult& result, const TVector<TString>& columnHints, + const IDataProvider::TFillSettings& fillSettings, google::protobuf::Arena* arena) { - NKikimrMiniKQL::TResult* packedResult = google::protobuf::Arena::CreateMessage<NKikimrMiniKQL::TResult>(arena); - auto* packedType = packedResult->MutableType(); + NKikimrMiniKQL::TResult* packedResult = google::protobuf::Arena::CreateMessage<NKikimrMiniKQL::TResult>(arena); + auto* packedType = packedResult->MutableType(); packedType->SetKind(NKikimrMiniKQL::ETypeKind::Struct); auto* dataMember = packedType->MutableStruct()->AddMember(); dataMember->SetName("Data"); @@ -383,7 +383,7 @@ NKikimrMiniKQL::TResult* KikimrResultToProto(const NKikimrMiniKQL::TResult& resu truncatedMember->MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); truncatedMember->MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<bool>::Id); - auto* packedValue = packedResult->MutableValue(); + auto* packedValue = packedResult->MutableValue(); auto* dataValue = packedValue->AddStruct(); auto* dataType = dataMember->MutableType(); auto* truncatedValue = packedValue->AddStruct(); @@ -549,18 +549,18 @@ bool ExportTypeToKikimrProto(const TTypeAnnotationNode& type, NKikimrMiniKQL::TT case ETypeAnnotationKind::Data: { protoType.SetKind(NKikimrMiniKQL::ETypeKind::Data); auto slot = type.Cast<TDataExprType>()->GetSlot(); - auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId; - if (typeId == NYql::NProto::TypeIds::Decimal) { - auto dataProto = protoType.MutableData(); - dataProto->SetScheme(typeId); - const auto paramsDataType = type.Cast<TDataExprParamsType>(); - ui8 precision = ::FromString<ui8>(paramsDataType->GetParamOne()); - ui8 scale = ::FromString<ui8>(paramsDataType->GetParamTwo()); - dataProto->MutableDecimalParams()->SetPrecision(precision); - dataProto->MutableDecimalParams()->SetScale(scale); - } else { - protoType.MutableData()->SetScheme(typeId); - } + auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId; + if (typeId == NYql::NProto::TypeIds::Decimal) { + auto dataProto = protoType.MutableData(); + dataProto->SetScheme(typeId); + const auto paramsDataType = type.Cast<TDataExprParamsType>(); + ui8 precision = ::FromString<ui8>(paramsDataType->GetParamOne()); + ui8 scale = ::FromString<ui8>(paramsDataType->GetParamTwo()); + dataProto->MutableDecimalParams()->SetPrecision(precision); + dataProto->MutableDecimalParams()->SetScale(scale); + } else { + protoType.MutableData()->SetScheme(typeId); + } return true; } @@ -887,11 +887,11 @@ TMaybe<TString> GetTableListResult(const IKikimrGateway::TListPathResult& res, } } - std::unique_ptr<NKikimrMiniKQL::TResult> packedResult(KikimrResultToProto(result, {}, fillSettings, nullptr)); + std::unique_ptr<NKikimrMiniKQL::TResult> packedResult(KikimrResultToProto(result, {}, fillSettings, nullptr)); if (fillSettings.Format == IDataProvider::EResultFormat::Yson) { NYson::EYsonFormat ysonFormat = NCommon::GetYsonFormat(fillSettings); - auto yson = KqpResultToYson(*packedResult, ysonFormat, ctx); + auto yson = KqpResultToYson(*packedResult, ysonFormat, ctx); if (!yson) { return Nothing(); } @@ -901,7 +901,7 @@ TMaybe<TString> GetTableListResult(const IKikimrGateway::TListPathResult& res, YQL_ENSURE(fillSettings.Format == IDataProvider::EResultFormat::Custom); YQL_ENSURE(fillSettings.FormatDetails == KikimrMkqlProtoFormat); - TVector<char> buffer(packedResult->ByteSize()); + TVector<char> buffer(packedResult->ByteSize()); Y_PROTOBUF_SUPPRESS_NODISCARD packedResult->SerializeToArray(buffer.data(), buffer.size()); return TString(buffer.data(), buffer.size()); } @@ -924,11 +924,11 @@ TMaybe<TString> GetTableMetadataResult(const TKikimrTableDescription& table, result.MutableValue()->SetBytes(metaYson); - std::unique_ptr<NKikimrMiniKQL::TResult> packedResult(KikimrResultToProto(result, {}, fillSettings, nullptr)); + std::unique_ptr<NKikimrMiniKQL::TResult> packedResult(KikimrResultToProto(result, {}, fillSettings, nullptr)); if (fillSettings.Format == IDataProvider::EResultFormat::Yson) { NYson::EYsonFormat ysonFormat = NCommon::GetYsonFormat(fillSettings); - auto yson = KqpResultToYson(*packedResult, ysonFormat, ctx); + auto yson = KqpResultToYson(*packedResult, ysonFormat, ctx); if (!yson) { return Nothing(); } @@ -938,7 +938,7 @@ TMaybe<TString> GetTableMetadataResult(const TKikimrTableDescription& table, YQL_ENSURE(fillSettings.Format == IDataProvider::EResultFormat::Custom); YQL_ENSURE(fillSettings.FormatDetails == KikimrMkqlProtoFormat); - TVector<char> buffer(packedResult->ByteSize()); + TVector<char> buffer(packedResult->ByteSize()); Y_PROTOBUF_SUPPRESS_NODISCARD packedResult->SerializeToArray(buffer.data(), buffer.size()); return TString(buffer.data(), buffer.size()); } diff --git a/ydb/core/kqp/provider/yql_kikimr_results.h b/ydb/core/kqp/provider/yql_kikimr_results.h index c11fc652e8..c3460f8a07 100644 --- a/ydb/core/kqp/provider/yql_kikimr_results.h +++ b/ydb/core/kqp/provider/yql_kikimr_results.h @@ -10,13 +10,13 @@ NKikimrMiniKQL::TResult GetKikimrUnpackedRunResult(const NKikimrMiniKQL::TResult void GetKikimrUnpackedRunResult(const NKikimrMiniKQL::TResult& runResult, ui32 index, const NKikimrMiniKQL::TType*& type, const NKikimrMiniKQL::TValue*& value); -TVector<NKikimrMiniKQL::TResult*> UnpackKikimrRunResult(const NKikimrMiniKQL::TResult& runResult, google::protobuf::Arena* arena); +TVector<NKikimrMiniKQL::TResult*> UnpackKikimrRunResult(const NKikimrMiniKQL::TResult& runResult, google::protobuf::Arena* arena); void KikimrResultToYson(const TStringStream& stream, NYson::TYsonWriter& writer, const NKikimrMiniKQL::TResult& result, const TVector<TString>& columnHints, const IDataProvider::TFillSettings& fillSettings, bool& truncated); -NKikimrMiniKQL::TResult* KikimrResultToProto(const NKikimrMiniKQL::TResult& result, const TVector<TString>& columnHints, - const IDataProvider::TFillSettings& fillSettings, google::protobuf::Arena* arena); +NKikimrMiniKQL::TResult* KikimrResultToProto(const NKikimrMiniKQL::TResult& result, const TVector<TString>& columnHints, + const IDataProvider::TFillSettings& fillSettings, google::protobuf::Arena* arena); bool IsRawKikimrResult(const NKikimrMiniKQL::TResult& result); diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 65d98b7745..692165204a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -23,7 +23,7 @@ const TTypeAnnotationNode* GetExpectedRowType(const TKikimrTableDescription& tab auto columnType = tableDesc.GetColumnType(column); if (!columnType) { - ctx.AddError(TIssue(pos, TStringBuilder() + ctx.AddError(TIssue(pos, TStringBuilder() << "No such column: " << column << ", table: " << FullTableName(tableDesc.Metadata->Cluster, tableDesc.Metadata->Name))); return nullptr; @@ -114,30 +114,30 @@ private: switch (key.GetKeyType()) { case TKikimrKey::Type::Table: - { + { auto readTable = node.Cast<TKiReadTable>(); - - const TKikimrTableDescription* tableDesc; - if ((tableDesc = SessionCtx->Tables().EnsureTableExists(cluster, key.GetTablePath(), node.Pos(), ctx)) == nullptr) { + + const TKikimrTableDescription* tableDesc; + if ((tableDesc = SessionCtx->Tables().EnsureTableExists(cluster, key.GetTablePath(), node.Pos(), ctx)) == nullptr) { return TStatus::Error; } - if (const auto& view = key.GetView()) { - if (!ValidateTableHasIndex(tableDesc->Metadata, ctx, node.Pos())) { - return TStatus::Error; - } - if (tableDesc->Metadata->GetIndexMetadata(view.GetRef()).first == nullptr) { + if (const auto& view = key.GetView()) { + if (!ValidateTableHasIndex(tableDesc->Metadata, ctx, node.Pos())) { + return TStatus::Error; + } + if (tableDesc->Metadata->GetIndexMetadata(view.GetRef()).first == nullptr) { ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() - << "Required global index not found, index name: " << view.GetRef())); - return TStatus::Error; - } - } + << "Required global index not found, index name: " << view.GetRef())); + return TStatus::Error; + } + } bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); auto selectType = GetReadTableRowType( ctx, SessionCtx->Tables(), TString(readTable.DataSource().Cluster()), key.GetTablePath(), readTable.GetSelectColumns(ctx, SessionCtx->Tables(), sysColumnsEnabled), sysColumnsEnabled ); - + if (!selectType) { return TStatus::Error; } @@ -258,8 +258,8 @@ namespace { } return true; } -} - +} + class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer { public: @@ -364,8 +364,8 @@ private: for (auto& keyColumnName : table->Metadata->KeyColumnNames) { if (!rowType->FindItem(keyColumnName)) { ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() - << "Missing key column in input: " << keyColumnName - << " for table: " << table->Metadata->Name)); + << "Missing key column in input: " << keyColumnName + << " for table: " << table->Metadata->Name)); return TStatus::Error; } } @@ -554,7 +554,7 @@ private: TString cluster = TString(create.DataSink().Cluster()); TString table = TString(create.Table()); - auto columnTypeError = GetColumnTypeErrorFn(ctx); + auto columnTypeError = GetColumnTypeErrorFn(ctx); TKikimrTableMetadataPtr meta = new TKikimrTableMetadata(cluster, table); meta->DoesExist = true; @@ -612,7 +612,7 @@ private: } } - for (const auto& index : create.Indexes()) { + for (const auto& index : create.Indexes()) { const auto type = index.Type().Value(); TIndexDescription::EType indexType; @@ -624,28 +624,28 @@ private: YQL_ENSURE(false, "Unknown index type: " << type); } - TVector<TString> indexColums; - TVector<TString> dataColums; - - for (const auto& indexCol : index.Columns()) { - if (!meta->Columns.contains(TString(indexCol.Value()))) { + TVector<TString> indexColums; + TVector<TString> dataColums; + + for (const auto& indexCol : index.Columns()) { + if (!meta->Columns.contains(TString(indexCol.Value()))) { ctx.AddError(TIssue(ctx.GetPosition(indexCol.Pos()), TStringBuilder() - << "Index column: " << indexCol.Value() << " was not found in the index table")); - return IGraphTransformer::TStatus::Error; - } - indexColums.emplace_back(TString(indexCol.Value())); - } - - for (const auto& dataCol : index.DataColumns()) { - if (!meta->Columns.contains(TString(dataCol.Value()))) { - ctx.AddError(TIssue(ctx.GetPosition(dataCol.Pos()), TStringBuilder() - << "Data column: " << dataCol.Value() << " was not found in the index table")); - return IGraphTransformer::TStatus::Error; - } - dataColums.emplace_back(TString(dataCol.Value())); - } - - // IndexState and version, pathId are ignored for create table with index request + << "Index column: " << indexCol.Value() << " was not found in the index table")); + return IGraphTransformer::TStatus::Error; + } + indexColums.emplace_back(TString(indexCol.Value())); + } + + for (const auto& dataCol : index.DataColumns()) { + if (!meta->Columns.contains(TString(dataCol.Value()))) { + ctx.AddError(TIssue(ctx.GetPosition(dataCol.Pos()), TStringBuilder() + << "Data column: " << dataCol.Value() << " was not found in the index table")); + return IGraphTransformer::TStatus::Error; + } + dataColums.emplace_back(TString(dataCol.Value())); + } + + // IndexState and version, pathId are ignored for create table with index request TIndexDescription indexDesc( TString(index.Name().Value()), indexColums, @@ -656,10 +656,10 @@ private: 0, 0 ); - - meta->Indexes.push_back(indexDesc); - } - + + meta->Indexes.push_back(indexDesc); + } + for (auto columnFamily : create.ColumnFamilies()) { if (auto maybeTupleList = columnFamily.Maybe<TCoNameValueTupleList>()) { TColumnFamily family; @@ -854,26 +854,26 @@ private: return TStatus::Ok; } - virtual TStatus HandleAlterTable(TKiAlterTable node, TExprContext& ctx) override { + virtual TStatus HandleAlterTable(TKiAlterTable node, TExprContext& ctx) override { auto table = SessionCtx->Tables().EnsureTableExists(TString(node.DataSink().Cluster()), TString(node.Table().Value()), node.Pos(), ctx); - if (!table) { - return TStatus::Error; - } - - if (!table->Metadata) { - return TStatus::Error; - } - - if (!EnsureModifyPermissions(table->Metadata->Cluster, table->Metadata->Name, node.Pos(), ctx)) { - return TStatus::Error; - } - + if (!table) { + return TStatus::Error; + } + + if (!table->Metadata) { + return TStatus::Error; + } + + if (!EnsureModifyPermissions(table->Metadata->Cluster, table->Metadata->Name, node.Pos(), ctx)) { + return TStatus::Error; + } + if (!CheckDocApiModifiation(*table->Metadata, node.Pos(), ctx)) { return TStatus::Error; } YQL_ENSURE(!node.Actions().Empty()); - + for (const auto& action : node.Actions()) { auto name = action.Name().Value(); if (name == "renameTo") { @@ -910,7 +910,7 @@ private: columnTypeError(typeNode.Pos(), name, "Only core YQL data types are currently supported"); return TStatus::Error; } - + auto dataType = actualType->Cast<TDataExprType>(); if (!ValidateColumnDataType(dataType, typeNode, name, ctx)) { @@ -927,7 +927,7 @@ private: return TStatus::Error; } } - } + } } else if (name == "dropColumns") { auto listNode = action.Value().Cast<TCoAtomList>(); THashSet<TString> keyColumns; @@ -936,21 +936,21 @@ private: } for (auto dropColumn : listNode) { TString name(dropColumn.Value()); - + if (!table->Metadata->Columns.FindPtr(name)) { ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) << " Column \"" << name << "\" does not exist")); return TStatus::Error; } - + if (keyColumns.find(name) != keyColumns.end()) { ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) << " Column: \"" << name << "\" is a key column. Key column drop is not supported")); return TStatus::Error; } - } + } } else if (name == "alterColumns") { auto listNode = action.Value().Cast<TExprList>(); for (size_t i = 0; i < listNode.Size(); ++i) { @@ -972,56 +972,56 @@ private: << "\". Several column families for a single column are not yet supported")); return TStatus::Error; } - } - } else if (name == "addIndex") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); - auto name = TString(nameNode.Value()); - if (name == "indexColumns" || name == "dataColumns") { - auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); - for (auto column : columnList) { - TString columnName(column.Value()); - if (!table->Metadata->Columns.FindPtr(columnName)) { - ctx.AddError(TIssue(ctx.GetPosition(column.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << columnName << "\" does not exist")); - return TStatus::Error; - } - } - } - } - } else if (name == "dropIndex") { - auto nameNode = action.Value().Cast<TCoAtom>(); - auto name = TString(nameNode.Value()); - - const auto& indexes = table->Metadata->Indexes; - - auto cmp = [name](const TIndexDescription& desc) { - return name == desc.Name; - }; - - if (std::find_if(indexes.begin(), indexes.end(), cmp) == indexes.end()) { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Index: \"" << name << "\" does not exist")); - return TStatus::Error; - } + } + } else if (name == "addIndex") { + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto name = TString(nameNode.Value()); + if (name == "indexColumns" || name == "dataColumns") { + auto columnList = columnTuple.Item(1).Cast<TCoAtomList>(); + for (auto column : columnList) { + TString columnName(column.Value()); + if (!table->Metadata->Columns.FindPtr(columnName)) { + ctx.AddError(TIssue(ctx.GetPosition(column.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << columnName << "\" does not exist")); + return TStatus::Error; + } + } + } + } + } else if (name == "dropIndex") { + auto nameNode = action.Value().Cast<TCoAtom>(); + auto name = TString(nameNode.Value()); + + const auto& indexes = table->Metadata->Indexes; + + auto cmp = [name](const TIndexDescription& desc) { + return name == desc.Name; + }; + + if (std::find_if(indexes.begin(), indexes.end(), cmp) == indexes.end()) { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Index: \"" << name << "\" does not exist")); + return TStatus::Error; + } } else if (name != "addColumnFamilies" && name != "alterColumnFamilies" && name != "setTableSettings") { ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), TStringBuilder() << "Unknown alter table action: " << name)); return TStatus::Error; - } - } - + } + } + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - + return TStatus::Ok; + } + virtual TStatus HandleCreateUser(TKiCreateUser node, TExprContext& ctx) override { for (const auto& setting : node.Settings()) { auto name = setting.Name().Value(); @@ -1159,7 +1159,7 @@ private: switch (SessionCtx->Query().Type) { case EKikimrQueryType::YqlScript: case EKikimrQueryType::YqlScriptStreaming: - case EKikimrQueryType::YqlInternal: + case EKikimrQueryType::YqlInternal: break; default: @@ -1249,7 +1249,7 @@ private: auto selectRow = call.Cast(); auto selectType = GetReadTableRowType(ctx, SessionCtx->Tables(), TString(selectRow.Cluster()), - TString(selectRow.Table().Path()), selectRow.Select(), sysColumnsEnabled); + TString(selectRow.Table().Path()), selectRow.Select(), sysColumnsEnabled); if (!selectType) { return TStatus::Error; } @@ -1265,14 +1265,14 @@ private: auto selectRange = call.Cast(); auto selectType = GetReadTableRowType(ctx, SessionCtx->Tables(), TString(selectRange.Cluster()), - TString(selectRange.Table().Path()), selectRange.Select(), sysColumnsEnabled); + TString(selectRange.Table().Path()), selectRange.Select(), sysColumnsEnabled); if (!selectType) { return TStatus::Error; } auto listSelectType = ctx.MakeType<TListExprType>(selectType); - node.Ptr()->SetTypeAnn(listSelectType); + node.Ptr()->SetTypeAnn(listSelectType); return TStatus::Ok; } diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp index ea5654652e..decd2cb301 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp @@ -262,7 +262,7 @@ public: if (!GetYqlDefaultModuleResolver(ModuleResolverState->ExprCtx, ModuleResolverState->ModuleResolver)) { TStringStream errorStream; - ModuleResolverState->ExprCtx.IssueManager.GetIssues().PrintTo(errorStream); + ModuleResolverState->ExprCtx.IssueManager.GetIssues().PrintTo(errorStream); KQP_PROXY_LOG_E("Failed to load default YQL libraries: " << errorStream.Str()); PassAway(); @@ -515,11 +515,11 @@ public: TProcessResult<TKqpSessionInfo*> result; TKqpDbCountersPtr dbCounters; - - const auto deadline = TInstant::MicroSeconds(event.GetDeadlineUs()); - - if (CheckRequestDeadline(requestInfo, deadline, result) && - CreateNewSessionWorker(requestInfo, TString(DefaultKikimrPublicClusterName), true, request.GetDatabase(), result)) + + const auto deadline = TInstant::MicroSeconds(event.GetDeadlineUs()); + + if (CheckRequestDeadline(requestInfo, deadline, result) && + CreateNewSessionWorker(requestInfo, TString(DefaultKikimrPublicClusterName), true, request.GetDatabase(), result)) { auto& response = *responseEv->Record.MutableResponse(); response.SetSessionId(result.Value->SessionId); @@ -1008,7 +1008,7 @@ private: TKqpDbCountersPtr dbCounters) { Y_UNUSED(requestInfo); - const auto& event = holder.GetRef(); + const auto& event = holder.GetRef(); Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetYdbStatus()); @@ -1080,26 +1080,26 @@ private: return Send(SelfId(), response.Release(), 0, requestId); } - bool CheckRequestDeadline(const TKqpRequestInfo& requestInfo, const TInstant deadline, TProcessResult<TKqpSessionInfo*>& result) - { - if (!deadline) { - return true; - } - auto now = TInstant::Now(); - if (now >= deadline) { - TString error = TStringBuilder() << "Request deadline has expired for " << now - deadline << " seconds"; - KQP_PROXY_LOG_E(requestInfo << error); - - // In theory client should not see this status due to internal grpc deadline accounting. - result.YdbStatus = Ydb::StatusIds::TIMEOUT; - result.Error = error; - return false; - } else { - KQP_PROXY_LOG_D(requestInfo << "Request has " << deadline - now << " seconds to be completed"); - return true; - } - } - + bool CheckRequestDeadline(const TKqpRequestInfo& requestInfo, const TInstant deadline, TProcessResult<TKqpSessionInfo*>& result) + { + if (!deadline) { + return true; + } + auto now = TInstant::Now(); + if (now >= deadline) { + TString error = TStringBuilder() << "Request deadline has expired for " << now - deadline << " seconds"; + KQP_PROXY_LOG_E(requestInfo << error); + + // In theory client should not see this status due to internal grpc deadline accounting. + result.YdbStatus = Ydb::StatusIds::TIMEOUT; + result.Error = error; + return false; + } else { + KQP_PROXY_LOG_D(requestInfo << "Request has " << deadline - now << " seconds to be completed"); + return true; + } + } + bool CreateNewSessionWorker(const TKqpRequestInfo& requestInfo, const TString& cluster, bool longSession, const TString& database, TProcessResult<TKqpSessionInfo*>& result) { diff --git a/ydb/core/kqp/runtime/kqp_channel_storage.cpp b/ydb/core/kqp/runtime/kqp_channel_storage.cpp index 5417923e6c..d39d009090 100644 --- a/ydb/core/kqp/runtime/kqp_channel_storage.cpp +++ b/ydb/core/kqp/runtime/kqp_channel_storage.cpp @@ -206,7 +206,7 @@ public: TKqpChannelStorage(ui64 txId, ui64 channelId, TWakeUpCallback&& wakeUp, const TActorContext& ctx) { SelfActor = new TKqpChannelStorageActor(txId, channelId, std::move(wakeUp)); - ctx.RegisterWithSameMailbox(SelfActor); + ctx.RegisterWithSameMailbox(SelfActor); } ~TKqpChannelStorage() { diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp index 25239c35d5..b5169c19d8 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.cpp +++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp @@ -207,7 +207,7 @@ TRuntimeNode TKqpProgramBuilder::KqpLookupTable(const TTableId& tableId, const T } TRuntimeNode TKqpProgramBuilder::KqpUpsertRows(const TTableId& tableId, const TRuntimeNode& rows, - const TArrayRef<TKqpTableColumn>& upsertColumns) + const TArrayRef<TKqpTableColumn>& upsertColumns) { auto streamType = AS_TYPE(TStreamType, rows.GetStaticType()); auto rowType = AS_TYPE(TStructType, streamType->GetItemType()); diff --git a/ydb/core/kqp/runtime/kqp_program_builder.h b/ydb/core/kqp/runtime/kqp_program_builder.h index b751f01169..348d25eeea 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.h +++ b/ydb/core/kqp/runtime/kqp_program_builder.h @@ -57,7 +57,7 @@ public: const TArrayRef<TKqpTableColumn>& keyColumns, const TArrayRef<TKqpTableColumn>& columns); TRuntimeNode KqpUpsertRows(const TTableId& tableId, const TRuntimeNode& rows, - const TArrayRef<TKqpTableColumn>& upsertColumns); + const TArrayRef<TKqpTableColumn>& upsertColumns); TRuntimeNode KqpDeleteRows(const TTableId& tableId, const TRuntimeNode& rows); diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index d718ad82e7..9bc7889f92 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -57,13 +57,13 @@ NMiniKQL::IFunctionRegistry* UdfFrFactory(const NScheme::TTypeRegistry& typeRegi return funcRegistry.Release(); } -TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings() { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_KqpYqlSyntaxVersion"); - setting.SetValue("1"); - return {setting}; -} - +TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings() { + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_KqpYqlSyntaxVersion"); + setting.SetValue("1"); + return {setting}; +} + TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { // EnableKikimrBacktraceFormat(); // Very slow, enable only when required locally @@ -104,13 +104,13 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { ServerSettings->SetFeatureFlags(settings.FeatureFlags); ServerSettings->SetNodeCount(settings.NodeCount); ServerSettings->SetEnableKqpSpilling(enableSpilling); - ServerSettings->SetEnableDataColumnForIndexTable(true); + ServerSettings->SetEnableDataColumnForIndexTable(true); ServerSettings->SetKeepSnapshotTimeout(settings.KeepSnapshotTimeout); ServerSettings->SetFrFactory(&UdfFrFactory); ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); ServerSettings->SetEnableNotNullColumns(true); - if (settings.LogStream) - ServerSettings->SetLogBackend(new TStreamLogBackend(settings.LogStream)); + if (settings.LogStream) + ServerSettings->SetLogBackend(new TStreamLogBackend(settings.LogStream)); Server.Reset(MakeHolder<Tests::TServer>(*ServerSettings)); Server->EnableGRpc(grpcPort); @@ -166,9 +166,9 @@ TKikimrRunner::TKikimrRunner(const NKikimrConfig::TFeatureFlags& featureFlags, c TKikimrRunner::TKikimrRunner(const TString& authToken, const TString& domainRoot, ui32 nodeCount) : TKikimrRunner(TKikimrSettings() - .SetAuthToken(authToken) - .SetDomainRoot(domainRoot) - .SetNodeCount(nodeCount)) {} + .SetAuthToken(authToken) + .SetDomainRoot(domainRoot) + .SetNodeCount(nodeCount)) {} void TKikimrRunner::CreateSampleTables() { Client->CreateTable("/Root", R"( @@ -251,12 +251,12 @@ void TKikimrRunner::CreateSampleTables() { ); CREATE TABLE `KeyValue2` ( - Key String, - Value String, - PRIMARY KEY (Key) - ); - - + Key String, + Value String, + PRIMARY KEY (Key) + ); + + CREATE TABLE `Test` ( Group Uint32, Name String, @@ -330,9 +330,9 @@ void TKikimrRunner::CreateSampleTables() { (2u, "Two"); REPLACE INTO `KeyValue2` (Key, Value) VALUES - ("1", "One"), - ("2", "Two"); - + ("1", "One"), + ("2", "Two"); + REPLACE INTO `Test` (Group, Name, Amount, Comment) VALUES (1u, "Anna", 3500ul, "None"), (1u, "Paul", 300ul, "None"), @@ -693,21 +693,21 @@ TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& return CollectStreamResultImpl(it); } -TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table) { - auto it = session.ReadTable(table).GetValueSync(); - UNIT_ASSERT(it.IsSuccess()); - - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - streamPart.GetIssues().PrintTo(Cerr); - if (streamPart.EOS()) { - return {}; - } - UNIT_ASSERT_C(false, "Status: " << streamPart.GetStatus()); - } - return NYdb::FormatResultSetYson(streamPart.ExtractPart()); -} - +TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table) { + auto it = session.ReadTable(table).GetValueSync(); + UNIT_ASSERT(it.IsSuccess()); + + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + streamPart.GetIssues().PrintTo(Cerr); + if (streamPart.EOS()) { + return {}; + } + UNIT_ASSERT_C(false, "Status: " << streamPart.GetStatus()); + } + return NYdb::FormatResultSetYson(streamPart.ExtractPart()); +} + ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) { ui32 result = 0; @@ -786,77 +786,77 @@ NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& return NJson::TJsonValue(); } -void CreateSampleTablesWithIndex(TSession& session) { - auto res = session.ExecuteSchemeQuery(R"( - --!syntax_v1 - CREATE TABLE `/Root/SecondaryKeys` ( - Key Int32, - Fk Int32, - Value String, - PRIMARY KEY (Key), - INDEX Index GLOBAL ON (Fk) - ); - CREATE TABLE `/Root/SecondaryComplexKeys` ( - Key Int32, - Fk1 Int32, - Fk2 String, - Value String, - PRIMARY KEY (Key), - INDEX Index GLOBAL ON (Fk1, Fk2) - ); - CREATE TABLE `/Root/SecondaryWithDataColumns` ( - Key String, - Index2 String, - Value String, - ExtPayload String, - PRIMARY KEY (Key), - INDEX Index GLOBAL ON (Index2) - COVER (Value) - ) - - )").GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - - auto result = session.ExecuteDataQuery(R"( +void CreateSampleTablesWithIndex(TSession& session) { + auto res = session.ExecuteSchemeQuery(R"( + --!syntax_v1 + CREATE TABLE `/Root/SecondaryKeys` ( + Key Int32, + Fk Int32, + Value String, + PRIMARY KEY (Key), + INDEX Index GLOBAL ON (Fk) + ); + CREATE TABLE `/Root/SecondaryComplexKeys` ( + Key Int32, + Fk1 Int32, + Fk2 String, + Value String, + PRIMARY KEY (Key), + INDEX Index GLOBAL ON (Fk1, Fk2) + ); + CREATE TABLE `/Root/SecondaryWithDataColumns` ( + Key String, + Index2 String, + Value String, + ExtPayload String, + PRIMARY KEY (Key), + INDEX Index GLOBAL ON (Index2) + COVER (Value) + ) + + )").GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + + auto result = session.ExecuteDataQuery(R"( PRAGMA kikimr.UseNewEngine = "true"; REPLACE INTO `KeyValue` (Key, Value) VALUES - (3u, "Three"), - (4u, "Four"), - (10u, "Ten"), - (NULL, "Null Value"); - + (3u, "Three"), + (4u, "Four"), + (10u, "Ten"), + (NULL, "Null Value"); + REPLACE INTO `Test` (Group, Name, Amount, Comment) VALUES - (1u, "Jack", 100500ul, "Just Jack"), - (3u, "Harry", 5600ul, "Not Potter"), - (3u, "Joshua", 8202ul, "Very popular name in GB"), - (3u, "Muhammad", 887773ul, "Also very popular name in GB"), - (4u, "Hugo", 77, "Boss"); - - REPLACE INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES - (1, 1, "Payload1"), - (2, 2, "Payload2"), - (5, 5, "Payload5"), - (NULL, 6, "Payload6"), - (7, NULL, "Payload7"), - (NULL, NULL, "Payload8"); - - REPLACE INTO `/Root/SecondaryComplexKeys` (Key, Fk1, Fk2, Value) VALUES - (1, 1, "Fk1", "Payload1"), - (2, 2, "Fk2", "Payload2"), - (5, 5, "Fk5", "Payload5"), - (NULL, 6, "Fk6", "Payload6"), - (7, NULL, "Fk7", "Payload7"), - (NULL, NULL, NULL, "Payload8"); - - REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"); - - )", TTxControl::BeginTx().CommitTx()).GetValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); -} - + (1u, "Jack", 100500ul, "Just Jack"), + (3u, "Harry", 5600ul, "Not Potter"), + (3u, "Joshua", 8202ul, "Very popular name in GB"), + (3u, "Muhammad", 887773ul, "Also very popular name in GB"), + (4u, "Hugo", 77, "Boss"); + + REPLACE INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES + (1, 1, "Payload1"), + (2, 2, "Payload2"), + (5, 5, "Payload5"), + (NULL, 6, "Payload6"), + (7, NULL, "Payload7"), + (NULL, NULL, "Payload8"); + + REPLACE INTO `/Root/SecondaryComplexKeys` (Key, Fk1, Fk2, Value) VALUES + (1, 1, "Fk1", "Payload1"), + (2, 2, "Fk2", "Payload2"), + (5, 5, "Fk5", "Payload5"), + (NULL, 6, "Fk6", "Payload6"), + (7, NULL, "Fk7", "Payload7"), + (NULL, NULL, NULL, "Payload8"); + + REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES + ("Primary1", "Secondary1", "Value1"); + + )", TTxControl::BeginTx().CommitTx()).GetValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + void WaitForKqpProxyInit(const NYdb::TDriver& driver) { NYdb::NTable::TTableClient client(driver); diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index d828b798a3..c1d64b65c3 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -39,20 +39,20 @@ #define Y_UNIT_TEST_NEW_ENGINE(N) Y_UNIT_TEST_TWIN(N, UseNewEngine) -#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \ - template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \ - struct TTestRegistration##N { \ - TTestRegistration##N() { \ - TCurrentTest::AddTest(#N "-" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, false>), false); \ - TCurrentTest::AddTest(#N "+" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, false>), false); \ - TCurrentTest::AddTest(#N "-" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, true>), false); \ - TCurrentTest::AddTest(#N "+" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, true>), false); \ - } \ - }; \ - static TTestRegistration##N testRegistration##N; \ - template<bool OPT1, bool OPT2> \ - void N(NUnitTest::TTestContext&) - +#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \ + template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \ + struct TTestRegistration##N { \ + TTestRegistration##N() { \ + TCurrentTest::AddTest(#N "-" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, false>), false); \ + TCurrentTest::AddTest(#N "+" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, false>), false); \ + TCurrentTest::AddTest(#N "-" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, true>), false); \ + TCurrentTest::AddTest(#N "+" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, true>), false); \ + } \ + }; \ + static TTestRegistration##N testRegistration##N; \ + template<bool OPT1, bool OPT2> \ + void N(NUnitTest::TTestContext&) + template <bool UseNewEngine, bool ForceVersionV1 = false> TString Query(const TString& tmpl) { return TStringBuilder() @@ -70,8 +70,8 @@ namespace NKqp { const TString KikimrDefaultUtDomainRoot = "Root"; -TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings(); - +TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings(); + struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> { NKikimrConfig::TAppConfig AppConfig; TVector<NKikimrKqp::TKqpSetting> KqpSettings; @@ -80,7 +80,7 @@ struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> { ui32 NodeCount = 1; bool WithSampleTables = true; TDuration KeepSnapshotTimeout = TDuration::Zero(); - IOutputStream* LogStream = nullptr; + IOutputStream* LogStream = nullptr; TKikimrSettings& SetAppConfig(const NKikimrConfig::TAppConfig& value) { AppConfig = value; return *this; } TKikimrSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; } @@ -121,7 +121,7 @@ public: const TString& GetEndpoint() const { return Endpoint; } const NYdb::TDriver& GetDriver() const { return *Driver; } - NYdb::NScheme::TSchemeClient GetSchemeClient() const { return NYdb::NScheme::TSchemeClient(*Driver); } + NYdb::NScheme::TSchemeClient GetSchemeClient() const { return NYdb::NScheme::TSchemeClient(*Driver); } Tests::TClient& GetTestClient() const { return *Client; } Tests::TServer& GetTestServer() const { return *Server; } @@ -225,14 +225,14 @@ TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it); ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); -TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table); - +TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table); + inline void AssertSuccessResult(const NYdb::TStatus& result) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } -void CreateSampleTablesWithIndex(NYdb::NTable::TSession& session); - +void CreateSampleTablesWithIndex(NYdb::NTable::TSession& session); + // KQP proxy needs to asynchronously receive tenants info before it is able to serve requests that have // database name specified. Before that it returns errors. // This method retries a simple query until it succeeds. diff --git a/ydb/core/kqp/ut/kqp_document_api_ut.cpp b/ydb/core/kqp/ut/kqp_document_api_ut.cpp index 0a8bbd8508..d51bb01516 100644 --- a/ydb/core/kqp/ut/kqp_document_api_ut.cpp +++ b/ydb/core/kqp/ut/kqp_document_api_ut.cpp @@ -44,51 +44,51 @@ Y_UNIT_TEST_SUITE(KqpDocumentApi) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - Y_UNIT_TEST(RestrictWriteExplicitPrepare) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTables(session); - - auto text = R"( - DECLARE $rows AS 'List<Struct<Key1:String?, Key2:String?>>'; - UPSERT INTO [/Root/DocumentApiTest] (Key1, Key2) - SELECT Key1, Key2 FROM AS_TABLE($rows); - )"; - - { - auto prepareResult = session.PrepareDataQuery(text).ExtractValueSync(); - - prepareResult.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(!prepareResult.IsSuccess()); - UNIT_ASSERT(HasIssue(prepareResult.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_OPERATION)); - } - - auto prepareSettings = TPrepareDataQuerySettings().RequestType("_document_api_request"); - auto prepareResult = session.PrepareDataQuery(text, prepareSettings).ExtractValueSync(); - UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString()); - - auto params = prepareResult.GetQuery().GetParamsBuilder() - .AddParam("$rows") - .BeginList() - .AddListItem() - .BeginStruct() - .AddMember("Key1").OptionalString("k1") - .AddMember("Key2").OptionalString("k2") - .EndStruct() - .EndList() - .Build() - .Build(); - - auto execSettings = TExecDataQuerySettings().RequestType("_document_api_request"); - auto result = prepareResult.GetQuery().Execute( - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - std::move(params), execSettings).ExtractValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - + Y_UNIT_TEST(RestrictWriteExplicitPrepare) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTables(session); + + auto text = R"( + DECLARE $rows AS 'List<Struct<Key1:String?, Key2:String?>>'; + UPSERT INTO [/Root/DocumentApiTest] (Key1, Key2) + SELECT Key1, Key2 FROM AS_TABLE($rows); + )"; + + { + auto prepareResult = session.PrepareDataQuery(text).ExtractValueSync(); + + prepareResult.GetIssues().PrintTo(Cerr); + UNIT_ASSERT(!prepareResult.IsSuccess()); + UNIT_ASSERT(HasIssue(prepareResult.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_OPERATION)); + } + + auto prepareSettings = TPrepareDataQuerySettings().RequestType("_document_api_request"); + auto prepareResult = session.PrepareDataQuery(text, prepareSettings).ExtractValueSync(); + UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString()); + + auto params = prepareResult.GetQuery().GetParamsBuilder() + .AddParam("$rows") + .BeginList() + .AddListItem() + .BeginStruct() + .AddMember("Key1").OptionalString("k1") + .AddMember("Key2").OptionalString("k2") + .EndStruct() + .EndList() + .Build() + .Build(); + + auto execSettings = TExecDataQuerySettings().RequestType("_document_api_request"); + auto result = prepareResult.GetQuery().Execute( + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + std::move(params), execSettings).ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + Y_UNIT_TEST(AllowRead) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_indexes_multishard_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_multishard_ut.cpp index a98d26ed28..c80b2282be 100644 --- a/ydb/core/kqp/ut/kqp_indexes_multishard_ut.cpp +++ b/ydb/core/kqp/ut/kqp_indexes_multishard_ut.cpp @@ -1,74 +1,74 @@ -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> - -#include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <library/cpp/json/json_reader.h> - -#include <util/string/printf.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScripting; - -namespace { - -void CreateTableWithMultishardIndex(Tests::TClient& client) { - const TString scheme = R"(Name: "MultiShardIndexed" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "fk" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"])"; - - NKikimrSchemeOp::TTableDescription desc; - bool parseOk = ::google::protobuf::TextFormat::ParseFromString(scheme, &desc); - UNIT_ASSERT(parseOk); - - auto status = client.TClient::CreateTableWithUniformShardedIndex("/Root", desc, "index", {"fk"}); - UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::MSTATUS_OK); -} - -} - -Y_UNIT_TEST_SUITE(KqpMultishardIndex) { - Y_UNIT_TEST_NEW_ENGINE(SortedRangeReadDesc) { - TKikimrRunner kikimr(SyntaxV1Settings()); - CreateTableWithMultishardIndex(kikimr.GetTestClient()); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - const TString query(Q_(R"( - UPSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES - (1, 1000000000, "v1"), - (2, 2000000000, "v2"), - (3, 3000000000, "v3"), - (4, 4294967295, "v4"); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const TString query(Q_(R"( - SELECT * FROM `/Root/MultiShardIndexed` VIEW index ORDER BY fk DESC LIMIT 1; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[4294967295u];[4u];[\"v4\"]]]"); - } - } -} - -} -} +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> + +#include <ydb/public/sdk/cpp/client/ydb_table/table.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/string/printf.h> + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScripting; + +namespace { + +void CreateTableWithMultishardIndex(Tests::TClient& client) { + const TString scheme = R"(Name: "MultiShardIndexed" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "fk" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"])"; + + NKikimrSchemeOp::TTableDescription desc; + bool parseOk = ::google::protobuf::TextFormat::ParseFromString(scheme, &desc); + UNIT_ASSERT(parseOk); + + auto status = client.TClient::CreateTableWithUniformShardedIndex("/Root", desc, "index", {"fk"}); + UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::MSTATUS_OK); +} + +} + +Y_UNIT_TEST_SUITE(KqpMultishardIndex) { + Y_UNIT_TEST_NEW_ENGINE(SortedRangeReadDesc) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndex(kikimr.GetTestClient()); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + const TString query(Q_(R"( + UPSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1, 1000000000, "v1"), + (2, 2000000000, "v2"), + (3, 3000000000, "v3"), + (4, 4294967295, "v4"); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + SELECT * FROM `/Root/MultiShardIndexed` VIEW index ORDER BY fk DESC LIMIT 1; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[4294967295u];[4u];[\"v4\"]]]"); + } + } +} + +} +} diff --git a/ydb/core/kqp/ut/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_ut.cpp index dc924bcab8..505cf09e6b 100644 --- a/ydb/core/kqp/ut/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/kqp_indexes_ut.cpp @@ -4,10 +4,10 @@ #include <ydb/core/kqp/kqp_impl.h> #include <ydb/core/kqp/kqp_metadata_loader.h> #include <ydb/core/kqp/host/kqp_host.h> - + #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <ydb/library/yql/core/services/mounts/yql_mounts.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> @@ -15,130 +15,130 @@ #include <util/string/printf.h> -namespace NKikimr { -namespace NKqp { +namespace NKikimr { +namespace NKqp { -using namespace NYdb; -using namespace NYdb::NTable; +using namespace NYdb; +using namespace NYdb::NTable; using namespace NYdb::NScripting; - -using NYql::TExprContext; -using NYql::TExprNode; - -namespace { - -constexpr const char* TestCluster = "local_ut"; - -TIntrusivePtr<NKqp::IKqpGateway> GetIcGateway(Tests::TServer& server) { + +using NYql::TExprContext; +using NYql::TExprNode; + +namespace { + +constexpr const char* TestCluster = "local_ut"; + +TIntrusivePtr<NKqp::IKqpGateway> GetIcGateway(Tests::TServer& server) { auto counters = MakeIntrusive<TKqpRequestCounters>(); counters->Counters = new TKqpCounters(server.GetRuntime()->GetAppData(0).Counters); counters->TxProxyMon = new NTxProxy::TTxProxyMon(server.GetRuntime()->GetAppData(0).Counters); std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(server.GetRuntime()->GetAnyNodeActorSystem(), false); return NKqp::CreateKikimrIcGateway(TestCluster, "/Root", std::move(loader), server.GetRuntime()->GetAnyNodeActorSystem(), server.GetRuntime()->GetNodeId(0), counters, MakeMiniKQLCompileServiceID()); -} - -TIntrusivePtr<IKqpHost> CreateKikimrQueryProcessor(TIntrusivePtr<IKqpGateway> gateway, - const TString& cluster, NYql::IModuleResolver::TPtr moduleResolver, - const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, - const TVector<NKikimrKqp::TKqpSetting>& settings = {}, bool keepConfigChanges = false) -{ - - NYql::TKikimrConfiguration::TPtr kikimrConfig = MakeIntrusive<NYql::TKikimrConfiguration>(); - auto defaultSettingsData = NResource::Find("kqp_default_settings.txt"); - TStringInput defaultSettingsStream(defaultSettingsData); - NKikimrKqp::TKqpDefaultSettings defaultSettings; - UNIT_ASSERT(TryParseFromTextFormat(defaultSettingsStream, defaultSettings)); - kikimrConfig->Init(defaultSettings.GetDefaultSettings(), cluster, settings, true); - +} + +TIntrusivePtr<IKqpHost> CreateKikimrQueryProcessor(TIntrusivePtr<IKqpGateway> gateway, + const TString& cluster, NYql::IModuleResolver::TPtr moduleResolver, + const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, + const TVector<NKikimrKqp::TKqpSetting>& settings = {}, bool keepConfigChanges = false) +{ + + NYql::TKikimrConfiguration::TPtr kikimrConfig = MakeIntrusive<NYql::TKikimrConfiguration>(); + auto defaultSettingsData = NResource::Find("kqp_default_settings.txt"); + TStringInput defaultSettingsStream(defaultSettingsData); + NKikimrKqp::TKqpDefaultSettings defaultSettings; + UNIT_ASSERT(TryParseFromTextFormat(defaultSettingsStream, defaultSettings)); + kikimrConfig->Init(defaultSettings.GetDefaultSettings(), cluster, settings, true); + return NKqp::CreateKqpHost(gateway, cluster, "/Root", kikimrConfig, moduleResolver, funcRegistry, - keepConfigChanges); -} - -NYql::NNodes::TExprBase GetExpr(const TString& ast, NYql::TExprContext& ctx, NYql::IModuleResolver* moduleResolver) { - NYql::TAstParseResult astRes = NYql::ParseAst(ast); - YQL_ENSURE(astRes.IsOk()); - NYql::TExprNode::TPtr result; - YQL_ENSURE(CompileExpr(*astRes.Root, result, ctx, moduleResolver)); - return NYql::NNodes::TExprBase(result); -} - -void CreateTableWithIndexWithState( - Tests::TServer& server, - const TString& indexName, - TIntrusivePtr<NKqp::IKqpGateway> gateway, - NYql::TIndexDescription::EIndexState state) -{ - using NYql::TKikimrTableMetadataPtr; - using NYql::TKikimrTableMetadata; - using NYql::TKikimrColumnMetadata; - - TKikimrTableMetadataPtr metadata = new TKikimrTableMetadata(TestCluster, server.GetSettings().DomainName + "/IndexedTableWithState"); + keepConfigChanges); +} + +NYql::NNodes::TExprBase GetExpr(const TString& ast, NYql::TExprContext& ctx, NYql::IModuleResolver* moduleResolver) { + NYql::TAstParseResult astRes = NYql::ParseAst(ast); + YQL_ENSURE(astRes.IsOk()); + NYql::TExprNode::TPtr result; + YQL_ENSURE(CompileExpr(*astRes.Root, result, ctx, moduleResolver)); + return NYql::NNodes::TExprBase(result); +} + +void CreateTableWithIndexWithState( + Tests::TServer& server, + const TString& indexName, + TIntrusivePtr<NKqp::IKqpGateway> gateway, + NYql::TIndexDescription::EIndexState state) +{ + using NYql::TKikimrTableMetadataPtr; + using NYql::TKikimrTableMetadata; + using NYql::TKikimrColumnMetadata; + + TKikimrTableMetadataPtr metadata = new TKikimrTableMetadata(TestCluster, server.GetSettings().DomainName + "/IndexedTableWithState"); metadata->Columns["key"] = TKikimrColumnMetadata("key", 0, "Uint32", false); metadata->Columns["value"] = TKikimrColumnMetadata("value", 0, "String", false); metadata->Columns["value2"] = TKikimrColumnMetadata("value2", 0, "String", false); metadata->ColumnOrder = {"key", "value", "value2"}; - metadata->Indexes.push_back( + metadata->Indexes.push_back( NYql::TIndexDescription( - indexName, - {"value"}, - TVector<TString>(), + indexName, + {"value"}, + TVector<TString>(), NYql::TIndexDescription::EType::GlobalSync, - state, - 0, - 0, - 0 + state, + 0, + 0, + 0 ) - ); - metadata->KeyColumnNames.push_back("key"); - - { - auto result = gateway->CreateTable(metadata, true).ExtractValueSync(); - UNIT_ASSERT(result.Success()); - } -} - -} // namespace - - -Y_UNIT_TEST_SUITE(KqpIndexMetadata) { - Y_UNIT_TEST_QUAD(HandleNotReadyIndex, WithMvcc, UseNewEngine) { - using namespace NYql; - using namespace NYql::NNodes; - - auto setting = NKikimrKqp::TKqpSetting(); - + ); + metadata->KeyColumnNames.push_back("key"); + + { + auto result = gateway->CreateTable(metadata, true).ExtractValueSync(); + UNIT_ASSERT(result.Success()); + } +} + +} // namespace + + +Y_UNIT_TEST_SUITE(KqpIndexMetadata) { + Y_UNIT_TEST_QUAD(HandleNotReadyIndex, WithMvcc, UseNewEngine) { + using namespace NYql; + using namespace NYql::NNodes; + + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto& server = kikimr.GetTestServer(); - auto gateway = GetIcGateway(server); - const TString& indexName = "value_index"; - - CreateTableWithIndexWithState(server, indexName, gateway, NYql::TIndexDescription::EIndexState::NotReady); - - TExprContext moduleCtx; - NYql::IModuleResolver::TPtr moduleResolver; - YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); - auto qp = CreateKikimrQueryProcessor(gateway, TestCluster, moduleResolver, - server.GetFunctionRegistry()); - - { + auto& server = kikimr.GetTestServer(); + auto gateway = GetIcGateway(server); + const TString& indexName = "value_index"; + + CreateTableWithIndexWithState(server, indexName, gateway, NYql::TIndexDescription::EIndexState::NotReady); + + TExprContext moduleCtx; + NYql::IModuleResolver::TPtr moduleResolver; + YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); + auto qp = CreateKikimrQueryProcessor(gateway, TestCluster, moduleResolver, + server.GetFunctionRegistry()); + + { const TString query = Q_(R"( UPSERT INTO `/Root/IndexedTableWithState` (key, value, value2) VALUES (1, "qq", "ww") )"); - auto explainResult = qp->SyncExplainDataQuery(query, true); - - UNIT_ASSERT(explainResult.Success()); - - TExprContext exprCtx; - VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(), - [&indexName](const TExprNode::TPtr& exprNode) { + auto explainResult = qp->SyncExplainDataQuery(query, true); + + UNIT_ASSERT(explainResult.Success()); + + TExprContext exprCtx; + VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(), + [&indexName](const TExprNode::TPtr& exprNode) { if (UseNewEngine) { if (TMaybeNode<TKqpUpsertRows>(exprNode)) { UNIT_ASSERT(!TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName)); @@ -157,66 +157,66 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { TStringBuf toErase = erase.Table().Path().Value(); UNIT_ASSERT(!toErase.Contains(indexName)); } - } - return true; - }); - } - - { - const TString query = Q_("SELECT * FROM `/Root/IndexedTableWithState`:value_index WHERE value = \"q\";"); - auto result = qp->SyncExplainDataQuery(query, true); - UNIT_ASSERT(!result.Success()); - UNIT_ASSERT_C(result.Issues().ToString().Contains("No global indexes for table"), result.Issues().ToString()); - } - } - - Y_UNIT_TEST_QUAD(HandleWriteOnlyIndex, WithMvcc, UseNewEngine) { - using namespace NYql; - using namespace NYql::NNodes; - - auto setting = NKikimrKqp::TKqpSetting(); + } + return true; + }); + } + + { + const TString query = Q_("SELECT * FROM `/Root/IndexedTableWithState`:value_index WHERE value = \"q\";"); + auto result = qp->SyncExplainDataQuery(query, true); + UNIT_ASSERT(!result.Success()); + UNIT_ASSERT_C(result.Issues().ToString().Contains("No global indexes for table"), result.Issues().ToString()); + } + } + + Y_UNIT_TEST_QUAD(HandleWriteOnlyIndex, WithMvcc, UseNewEngine) { + using namespace NYql; + using namespace NYql::NNodes; + + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - - auto& server = kikimr.GetTestServer(); - auto gateway = GetIcGateway(server); - const TString& indexName = "value_index"; - - CreateTableWithIndexWithState(server, indexName, gateway, NYql::TIndexDescription::EIndexState::WriteOnly); - - TExprContext moduleCtx; - NYql::IModuleResolver::TPtr moduleResolver; - YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); - auto qp = CreateKikimrQueryProcessor(gateway, TestCluster, moduleResolver, - server.GetFunctionRegistry()); - - { + + auto& server = kikimr.GetTestServer(); + auto gateway = GetIcGateway(server); + const TString& indexName = "value_index"; + + CreateTableWithIndexWithState(server, indexName, gateway, NYql::TIndexDescription::EIndexState::WriteOnly); + + TExprContext moduleCtx; + NYql::IModuleResolver::TPtr moduleResolver; + YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); + auto qp = CreateKikimrQueryProcessor(gateway, TestCluster, moduleResolver, + server.GetFunctionRegistry()); + + { const TString query = Q_(R"( UPSERT INTO `/Root/IndexedTableWithState` ("key", "value", "value2") VALUES (1, "qq", "ww"); )"); - auto explainResult = qp->SyncExplainDataQuery(query, true); - UNIT_ASSERT(explainResult.Success()); - - TExprContext exprCtx; - bool indexUpdated = false; - bool indexCleaned = false; - VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(), - [&indexName, &indexUpdated, &indexCleaned](const TExprNode::TPtr& exprNode) mutable { + auto explainResult = qp->SyncExplainDataQuery(query, true); + UNIT_ASSERT(explainResult.Success()); + + TExprContext exprCtx; + bool indexUpdated = false; + bool indexCleaned = false; + VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(), + [&indexName, &indexUpdated, &indexCleaned](const TExprNode::TPtr& exprNode) mutable { if (UseNewEngine) { if (TMaybeNode<TKqpUpsertRows>(exprNode)) { if (TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName)) { indexUpdated = true; } - } + } if (TMaybeNode<TKqpDeleteRows>(exprNode)) { if (TKqpDeleteRows(exprNode).Table().Path().Value().Contains(indexName)) { indexCleaned = true; } - } + } } else { if (auto maybeupdate = TMaybeNode<TKiUpdateRow>(exprNode)) { auto update = maybeupdate.Cast(); @@ -232,770 +232,770 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { indexCleaned = true; } } - } - return true; - }); - UNIT_ASSERT(indexUpdated); - UNIT_ASSERT(indexCleaned); - } - - { - const TString query = Q_("SELECT * FROM `/Root/IndexedTableWithState`:value_index WHERE value = \"q\";"); - auto result = qp->SyncExplainDataQuery(query, true); - UNIT_ASSERT(!result.Success()); - UNIT_ASSERT(result.Issues().ToString().Contains("is not ready to use")); - } - } -} - -Y_UNIT_TEST_SUITE(KqpIndexes) { - Y_UNIT_TEST_NEW_ENGINE(NullInIndexTableNoDataRead) { - auto setting = NKikimrKqp::TKqpSetting(); - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - + } + return true; + }); + UNIT_ASSERT(indexUpdated); + UNIT_ASSERT(indexCleaned); + } + + { + const TString query = Q_("SELECT * FROM `/Root/IndexedTableWithState`:value_index WHERE value = \"q\";"); + auto result = qp->SyncExplainDataQuery(query, true); + UNIT_ASSERT(!result.Success()); + UNIT_ASSERT(result.Issues().ToString().Contains("is not ready to use")); + } + } +} + +Y_UNIT_TEST_SUITE(KqpIndexes) { + Y_UNIT_TEST_NEW_ENGINE(NullInIndexTableNoDataRead) { + auto setting = NKikimrKqp::TKqpSetting(); + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + const TString query(Q1_(R"( SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[#];[[7]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 2 : 1); - - int phase = UseNewEngine ? 1 : 0; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).name(), "/Root/SecondaryKeys/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).reads().rows(), 2); - - } - } - - Y_UNIT_TEST_NEW_ENGINE(NullInIndexTable) { - auto setting = NKikimrKqp::TKqpSetting(); - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[#];[[7]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 2 : 1); + + int phase = UseNewEngine ? 1 : 0; + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).name(), "/Root/SecondaryKeys/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).reads().rows(), 2); + + } + } + + Y_UNIT_TEST_NEW_ENGINE(NullInIndexTable) { + auto setting = NKikimrKqp::TKqpSetting(); + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + { const TString query(Q1_(R"( SELECT * FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[#;#;[\"Payload8\"]];[#;[7];[\"Payload7\"]]]"); - } -// We can't validate it now. Try to improve it after migrate to MVCC -#if 0 - if (UseNewEngine) { - NKikimrMiniKQL::TResult result; - auto& oldClient = kikimr.GetTestClient(); - bool success = oldClient.FlatQuery( - "(" - "(let row '('('Key (Int32 '7))))" - "(let pgmReturn (AsList" - " (EraseRow '/Root/SecondaryKeys row)" - "))" - "(return pgmReturn)" - ")", - result); - - UNIT_ASSERT(success); - - { - const TString query(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[#;#;[\"Payload8\"]];[#;[7];[\"Payload7\"]]]"); + } +// We can't validate it now. Try to improve it after migrate to MVCC +#if 0 + if (UseNewEngine) { + NKikimrMiniKQL::TResult result; + auto& oldClient = kikimr.GetTestClient(); + bool success = oldClient.FlatQuery( + "(" + "(let row '('('Key (Int32 '7))))" + "(let pgmReturn (AsList" + " (EraseRow '/Root/SecondaryKeys row)" + "))" + "(return pgmReturn)" + ")", + result); + + UNIT_ASSERT(success); + + { + const TString query(Q_(R"( SELECT * FROM `/Root/SecondaryKeys` : Index WHERE Fk IS NULL; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - } -#endif - } - - Y_UNIT_TEST_NEW_ENGINE(WriteWithParamsFieldOrder) { - auto setting = NKikimrKqp::TKqpSetting(); - auto serverSettings = TKikimrSettings() - .SetKqpSettings({setting}); - TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); + } + } +#endif + } + + Y_UNIT_TEST_NEW_ENGINE(WriteWithParamsFieldOrder) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { const TString keyColumnName = "key"; - auto builder = TTableBuilder() - .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); - - builder.AddNullableColumn("index_0", EPrimitiveType::Utf8); - builder.AddSecondaryIndex("index_0_name", TVector<TString>{"index_0", keyColumnName}); - builder.AddNullableColumn("value", EPrimitiveType::Uint32); - builder.SetPrimaryKeyColumns({keyColumnName}); - - auto result = session.CreateTable("/Root/Test1", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { + auto builder = TTableBuilder() + .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); + + builder.AddNullableColumn("index_0", EPrimitiveType::Utf8); + builder.AddSecondaryIndex("index_0_name", TVector<TString>{"index_0", keyColumnName}); + builder.AddNullableColumn("value", EPrimitiveType::Uint32); + builder.SetPrimaryKeyColumns({keyColumnName}); + + auto result = session.CreateTable("/Root/Test1", builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { const TString query1(Q1_(R"( DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; - UPSERT INTO `/Root/Test1` - SELECT * FROM AS_TABLE($items); - )")); - - TParamsBuilder builder; - builder.AddParam("$items").BeginList() - .AddListItem() - .BeginStruct() - .AddMember("key").Uint64(646464646464) - .AddMember("index_0").Utf8("SomeUtf8Data") - .AddMember("value").Uint32(323232) - .EndStruct() - .EndList() - .Build(); - - static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); - auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/Test1"); - const TString expected = R"([[[646464646464u];["SomeUtf8Data"];[323232u]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/Test1/index_0_name/indexImplTable"); - const TString expected = R"([[["SomeUtf8Data"];[646464646464u]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(DataColumnUpsertMixedSemantic, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); - auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) - .SetKqpSettings({setting}); - TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - // Just check table prepared - { + UPSERT INTO `/Root/Test1` + SELECT * FROM AS_TABLE($items); + )")); + + TParamsBuilder builder; + builder.AddParam("$items").BeginList() + .AddListItem() + .BeginStruct() + .AddMember("key").Uint64(646464646464) + .AddMember("index_0").Utf8("SomeUtf8Data") + .AddMember("value").Uint32(323232) + .EndStruct() + .EndList() + .Build(); + + static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); + auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/Test1"); + const TString expected = R"([[[646464646464u];["SomeUtf8Data"];[323232u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/Test1/index_0_name/indexImplTable"); + const TString expected = R"([[["SomeUtf8Data"];[646464646464u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST_QUAD(DataColumnUpsertMixedSemantic, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + // Just check table prepared + { const auto yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using pk and some other column. Index will be read from table value from user input. - // For the first row - insert semantic, the pk absent in the table - // for the second row - update semantic, the pk found - // This test checks the implementation has correct handle not exact semantic for table lookup - // (the lenth of lookup result is not equal to the lengh of the user input) - { + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using pk and some other column. Index will be read from table value from user input. + // For the first row - insert semantic, the pk absent in the table + // for the second row - update semantic, the pk found + // This test checks the implementation has correct handle not exact semantic for table lookup + // (the lenth of lookup result is not equal to the lengh of the user input) + { const TString query1(Q1_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES - ("Primary0", "Value0_", "Something"), - ("Primary1", "Value1_", "Something"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary0"];["Value0_"]];[["Secondary1"];["Primary1"];["Value1_"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(DataColumnWriteNull, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + ("Primary0", "Value0_", "Something"), + ("Primary1", "Value1_", "Something"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary0"];["Value0_"]];[["Secondary1"];["Primary1"];["Value1_"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST_QUAD(DataColumnWriteNull, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - // Just check table prepared - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using pk and some other column. Index will be read from table value from user input. - { - const TString query1(Q_(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + // Just check table prepared + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using pk and some other column. Index will be read from table value from user input. + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES - ("Primary1", NULL, "Something"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - // Upsert using pk and some other column. But pass null as input. - - { - - auto param = db.GetParamsBuilder() - .AddParam("$rows") - .EmptyList( - TTypeBuilder() - .BeginStruct() - .AddMember("Key").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() - .AddMember("Value").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() - .AddMember("ExtPayload").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() - .EndStruct() - .Build() - ) - .Build() - .Build(); - + ("Primary1", NULL, "Something"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + // Upsert using pk and some other column. But pass null as input. + + { + + auto param = db.GetParamsBuilder() + .AddParam("$rows") + .EmptyList( + TTypeBuilder() + .BeginStruct() + .AddMember("Key").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() + .AddMember("Value").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() + .AddMember("ExtPayload").BeginOptional().Primitive(EPrimitiveType::String).EndOptional() + .EndStruct() + .Build() + ) + .Build() + .Build(); + const TString query1(Q1_(R"( DECLARE $rows AS List<Struct< - Key : String?, - Value : String?, - ExtPayload : String? + Key : String?, + Value : String?, + ExtPayload : String? >>; UPSERT INTO `/Root/SecondaryWithDataColumns` - SELECT * FROM AS_TABLE($rows); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), param) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - - Y_UNIT_TEST_QUAD(DataColumnWrite, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); - auto serverSettings = TKikimrSettings() + SELECT * FROM AS_TABLE($rows); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), param) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + + Y_UNIT_TEST_QUAD(DataColumnWrite, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) - .SetKqpSettings({setting}); - TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using previous inserved pk and fk, check data column realy updated - { - const TString query1(Q_(R"( + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using previous inserved pk and fk, check data column realy updated + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1_1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1_1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using previous inserved pk but without fk, check data column still realy updated - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1", "Value1_1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1_1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using previous inserved pk but without fk, check data column still realy updated + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES - ("Primary1", "Value1_2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1_2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using new pk without fk - { - const TString query1(Q_(R"( + ("Primary1", "Value1_2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1_2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using new pk without fk + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES - ("Primary2", "Value2_1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Upsert using pk, fk, and some other column. Data column in index must have old value - { - const TString query1(Q_(R"( + ("Primary2", "Value2_1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Upsert using pk, fk, and some other column. Data column in index must have old value + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, ExtPayload) VALUES - ("Primary1", "Secondary1", "Something"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Replace row - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1", "Something"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Replace row + { + const TString query1(Q_(R"( REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1_3"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Replace row but no specify data column - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1", "Value1_3"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];["Value1_3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Replace row but no specify data column + { + const TString query1(Q_(R"( REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2) VALUES - ("Primary1", "Secondary1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Replace row specify data column but no index column - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2_1"]];[["Secondary1"];["Primary1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Replace row specify data column but no index column + { + const TString query1(Q_(R"( REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES - ("Primary2", "Value2_3"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2_3"]];[["Secondary1"];["Primary1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Replace just by pk - { - const TString query1(Q_(R"( + ("Primary2", "Value2_3"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2_3"]];[["Secondary1"];["Primary1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Replace just by pk + { + const TString query1(Q_(R"( REPLACE INTO `/Root/SecondaryWithDataColumns` (Key) VALUES - ("Primary2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];#];[["Secondary1"];["Primary1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Delete - { - const TString query1(Q_(R"( + ("Primary2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];#];[["Secondary1"];["Primary1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Delete + { + const TString query1(Q_(R"( DELETE FROM `/Root/SecondaryWithDataColumns` ON (Key) VALUES - ("Primary2"), ("Primary1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = ""; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Insert new row in empty table - { - const TString query1(Q_(R"( + ("Primary2"), ("Primary1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = ""; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Insert new row in empty table + { + const TString query1(Q_(R"( INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Insert row with same pk - { - const TString query1(R"( + ("Primary1", "Secondary1", "Value1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Insert row with same pk + { + const TString query1(R"( INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1_1"); - )"); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); - - } - - // Index table has not been changed - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Insert new row but no specify index column - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1", "Value1_1"); + )"); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); + + } + + // Index table has not been changed + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Insert new row but no specify index column + { + const TString query1(Q_(R"( INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES - ("Primary2", "Value2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1"];["Primary1"];["Value1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Update on - { - const TString query1(Q_(R"( + ("Primary2", "Value2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1"];["Primary1"];["Value1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Update on + { + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, Value) VALUES - ("NonExistPrimary", "Secondary1_1", "Value1_1"), - ("Primary1", "Secondary1_1", "Value1_1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - - const TString query1(Q_(R"( + ("NonExistPrimary", "Secondary1_1", "Value1_1"), + ("Primary1", "Secondary1_1", "Value1_1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Value) VALUES - ("Primary1", "Value1_2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - - const TString query1(Q_(R"( + ("Primary1", "Value1_2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, ExtPayload) VALUES - ("Primary1", "Secondary1_1", "Something"); - )")); - - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Update where - update data column - { - const TString query1(Q_(R"( + ("Primary1", "Secondary1_1", "Something"); + )")); + + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Update where - update data column + { + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` SET Value = "Value1_3" - WHERE Key = "Primary1"; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Update were - do not touch index - { - const TString query1(Q_(R"( + WHERE Key = "Primary1"; + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Update were - do not touch index + { + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` SET ExtPayload = "Something2" - WHERE Key = "Primary1"; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - // Update wete - update index column - { - const TString query1(Q_(R"( + WHERE Key = "Primary1"; + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_1"];["Primary1"];["Value1_3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + // Update wete - update index column + { + const TString query1(Q_(R"( UPDATE `/Root/SecondaryWithDataColumns` SET Index2 = "Secondary1_2" - WHERE Key = "Primary1"; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_2"];["Primary1"];["Value1_3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - } - - Y_UNIT_TEST_QUAD(DataColumnSelect, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + WHERE Key = "Primary1"; + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + const TString expected = R"([[#;["Primary2"];["Value2"]];[["Secondary1_2"];["Primary1"];["Value1_3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + } + + Y_UNIT_TEST_QUAD(DataColumnSelect, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - { - const TString query1(Q_(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES - ("p2", "s2", "2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const TString query1(Q_(R"( + ("p2", "s2", "2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES - (1, 111, "Secondary1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + (1, 111, "Secondary1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 = 'Secondary1'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value1\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - } else { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); - - } - } - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value1\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + } else { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); + + } + } + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); const TString query = Q1_(R"( SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 IN ('Secondary1'); - )"); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value1\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - + )"); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value1\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases().size(), UseNewEngine ? 2 : 1, stats.DebugString()); - + ui32 index = 0; if (UseNewEngine) { UNIT_ASSERT(stats.query_phases(index).table_access().empty()); @@ -1005,64 +1005,64 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 1); - } - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - + } + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + const TString query = Q1_(R"( SELECT t2.Value FROM `/Root/SecondaryKeys` as t1 INNER JOIN `/Root/SecondaryWithDataColumns` VIEW Index as t2 ON t2.Index2 = t1.Value; - )"); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Value1\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - + )"); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Value1\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup if (WithMvcc && !UseNewEngine) { phaseCount--; } UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), phaseCount); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/SecondaryKeys"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 5); - + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/SecondaryKeys"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 5); + int idx = UseNewEngine ? 2 : 1; UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases(idx).table_access().size(), 1, stats.DebugString()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1); - } - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - + } + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + const TString query = Q1_(R"( SELECT t2.Value FROM `/Root/SecondaryWithDataColumns` VIEW Index as t1 INNER JOIN `/Root/KeyValue2` as t2 ON t2.Key = t1.Value WHERE t1.Index2 = "s2"; - )"); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Two\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - + )"); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Two\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup if (WithMvcc && !UseNewEngine) { phaseCount--; @@ -1071,7 +1071,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { phaseCount++; } UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), phaseCount); - + int idx = 0; if (UseNewEngine) { idx = 1; @@ -1079,7 +1079,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1); - + idx++; if (UseNewEngine) { idx++; @@ -1087,92 +1087,92 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/KeyValue2"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1); - } - } - - - Y_UNIT_TEST_QUAD(SelectConcurentTX, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + } + } + + + Y_UNIT_TEST_QUAD(SelectConcurentTX, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"), - ("Primary2", "Secondary2", "Value2"), - ("Primary3", "Secondary3", "Value3"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - const TString query1(Q_(R"( + ("Primary1", "Secondary1", "Value1"), + ("Primary2", "Secondary2", "Value2"), + ("Primary3", "Secondary3", "Value3"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + const TString query1(Q_(R"( SELECT Index2 FROM `/Root/TestTable` WHERE Key = "Primary1"; - )")); - - // Start tx1, select from table by pk (without touching index table) and without commit - auto result1 = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW())) - .ExtractValueSync(); + )")); + + // Start tx1, select from table by pk (without touching index table) and without commit + auto result1 = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW())) + .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result1.GetStatus(), NYdb::EStatus::SUCCESS); - { - auto yson = NYdb::FormatResultSetYson(result1.GetResultSet(0)); + { + auto yson = NYdb::FormatResultSetYson(result1.GetResultSet(0)); UNIT_ASSERT_VALUES_EQUAL(yson, R"([[["Secondary1"]]])"); - } - - { - // In other tx, update string - const TString query1(Q_(R"( + } + + { + // In other tx, update string + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1New", "Value1New") - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - + ("Primary1", "Secondary1New", "Value1New") + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + const TString query2(Q1_(R"( SELECT Index2 FROM `/Root/TestTable` VIEW Index WHERE Index2 = "Secondary1"; - )")); - - UNIT_ASSERT(result1.GetTransaction()); - // Continue tx1, select from index table only - auto result2 = session.ExecuteDataQuery( - query2, - TTxControl::Tx(result1.GetTransaction().GetRef()).CommitTx()) - .ExtractValueSync(); + )")); + + UNIT_ASSERT(result1.GetTransaction()); + // Continue tx1, select from index table only + auto result2 = session.ExecuteDataQuery( + query2, + TTxControl::Tx(result1.GetTransaction().GetRef()).CommitTx()) + .ExtractValueSync(); // read only tx should succeed in MVCC case - UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), WithMvcc ? NYdb::EStatus::SUCCESS : NYdb::EStatus::ABORTED, result2.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), WithMvcc ? NYdb::EStatus::SUCCESS : NYdb::EStatus::ABORTED, result2.GetIssues().ToString()); } - Y_UNIT_TEST_QUAD(SelectConcurentTX2, WithMvcc, UseNewEngine) { + Y_UNIT_TEST_QUAD(SelectConcurentTX2, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) @@ -1196,12 +1196,12 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { } { - const TString query1(Q_(R"( + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES ("Primary1", "Secondary1", "Value1"), ("Primary2", "Secondary2", "Value2"), ("Primary3", "Secondary3", "Value3"); - )")); + )")); auto result = session.ExecuteDataQuery( query1, @@ -1227,10 +1227,10 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { { // In other tx, update string - const TString query(Q_(R"( + const TString query(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES ("Primary1", "Secondary1New", "Value1New") - )")); + )")); auto result = session.ExecuteDataQuery( query, @@ -1239,10 +1239,10 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT(result.IsSuccess()); } - const TString query2(Q_(R"( + const TString query2(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES ("Primary4", "Secondary4", "Value4") - )")); + )")); UNIT_ASSERT(result1.GetTransaction()); // Continue tx1, write to table should fail in both MVCC and non-MVCC scenarios @@ -1250,110 +1250,110 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { query2, TTxControl::Tx(result1.GetTransaction().GetRef()).CommitTx()) .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), NYdb::EStatus::ABORTED, result2.GetIssues().ToString().c_str()); - } - - Y_UNIT_TEST_QUAD(UpsertWithoutExtraNullDelete, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), NYdb::EStatus::ABORTED, result2.GetIssues().ToString().c_str()); + } + + Y_UNIT_TEST_QUAD(UpsertWithoutExtraNullDelete, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"); - )")); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - if (UseNewEngine) { + ("Primary1", "Secondary1", "Value1"); + )")); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + if (UseNewEngine) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access().size(), 2); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(0).updates().rows(), 1); - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).updates().rows(), 1); UNIT_ASSERT(!stats.query_phases(3).table_access(0).has_deletes()); - } else { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(!stats.query_phases(2).table_access(0).has_deletes()); - } - } - - { + } else { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT(!stats.query_phases(2).table_access(0).has_deletes()); + } + } + + { const TString query1 = Q1_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1_1", "Value1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + ("Primary1", "Secondary1_1", "Value1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + int idx = UseNewEngine ? 3 : 2; - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2); - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1); @@ -1361,1805 +1361,1805 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).updates().rows(), 1); UNIT_ASSERT(stats.query_phases(idx).table_access(1).has_deletes()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).deletes().rows(), 1); - } - - { - // Upsert without touching index + } + + { + // Upsert without touching index const TString query2 = Q1_(R"( UPSERT INTO `/Root/TestTable` (Key, Value) VALUES - ("Primary1", "Value1_1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query2, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + ("Primary1", "Value1_1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query2, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3); - - // One read from main table - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - + + // One read from main table + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + int idx = UseNewEngine ? 3 : 2; UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2); - - // One update of main table + + // One update of main table UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1); - - // No touching index + + // No touching index UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); if (!UseNewEngine) { // BUG UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_updates()); UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_deletes()); } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - // Update without touching index + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + // Update without touching index const TString query2 = Q1_(R"( UPDATE `/Root/TestTable` ON (Key, Value) VALUES - ("Primary1", "Value1_2"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query2, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + ("Primary1", "Value1_2"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query2, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 2); // BUG - + int idx = UseNewEngine ? 1 : 0; UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1); - // One read of main table + // One read of main table UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1); - - // One update of index table + + // One update of index table idx += (UseNewEngine ? 2 : 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1); - - // Thats it, no phase for index table - we remove it on compile time - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - // Update without touching index + + // Thats it, no phase for index table - we remove it on compile time + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + // Update without touching index const TString query2 = Q1_(R"( UPDATE `/Root/TestTable` SET Value = "Value1_3" - WHERE Key = "Primary1"; - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query2, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - - // One read of main table - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); - - // One update of main table - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - } - + WHERE Key = "Primary1"; + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query2, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); + + // One read of main table + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); + + // One update of main table + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + } + Y_UNIT_TEST_TWIN(KeyIndex, WithMvcc) { - auto setting = NKikimrKqp::TKqpSetting(); + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - // Table with complex index - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index", "Key"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + // Table with complex index + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index", "Key"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPSERT INTO `/Root/TestTable` (Key, Index) VALUES - ("Primary1", "Secondary1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - // Empty table no read in stats - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - // One update - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - // Index update without detele - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - const TString query1(R"( + ("Primary1", "Secondary1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + // Empty table no read in stats + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + // One update + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + // Index update without detele + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPSERT INTO `/Root/TestTable` (Key, Index) VALUES - ("Primary1", "Secondary1_1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - // One read to find previous index value - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - // Update main table - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - // Update index with deletion - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - const TString query1(R"( + ("Primary1", "Secondary1_1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + // One read to find previous index value + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + // Update main table + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + // Update index with deletion + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPDATE `/Root/TestTable` ON (Key, Index) VALUES - ("Primary1", "Secondary1_2"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - // Update main table - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - // Update index - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_2"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - const TString query1(R"( + ("Primary1", "Secondary1_2"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + // Update main table + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + // Update index + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_2"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPDATE `/Root/TestTable` SET Index = "Secondary1_3" - WHERE Key = "Primary1"; - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).updates().rows(), 1); - UNIT_ASSERT(stats.query_phases(1).table_access(1).has_deletes()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).deletes().rows(), 1); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_3"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - } - + WHERE Key = "Primary1"; + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).updates().rows(), 1); + UNIT_ASSERT(stats.query_phases(1).table_access(1).has_deletes()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(1).deletes().rows(), 1); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_3"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + } + Y_UNIT_TEST_TWIN(KeyIndex2, WithMvcc) { - auto setting = NKikimrKqp::TKqpSetting(); + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - // pk is <Key,Index> - // index is <Index,Key> - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Index"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index", "Key"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + // pk is <Key,Index> + // index is <Index,Key> + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Index"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index", "Key"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPSERT INTO `/Root/TestTable` (Key, Index) VALUES - ("Primary1", "Secondary1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - // Empty table without delete - UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - // Upsert on new pk - const TString query1(R"( + ("Primary1", "Secondary1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + // Empty table without delete + UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + // Upsert on new pk + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPSERT INTO `/Root/TestTable` (Key, Index) VALUES - ("Primary1", "Secondary1_1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - // read nothing - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - // no deletion - UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - { - // Update on non existing key - do nothing - const TString query1(R"( + ("Primary1", "Secondary1_1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + // read nothing + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + // no deletion + UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + { + // Update on non existing key - do nothing + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; UPDATE `/Root/TestTable` ON (Key, Index) VALUES - ("Primary1", "Secondary1_2"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 0); - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary1_1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - } - + ("Primary1", "Secondary1_2"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 0); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 0); + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary1_1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + } + Y_UNIT_TEST_TWIN(ReplaceWithoutExtraNullDelete, WithMvcc) { - auto setting = NKikimrKqp::TKqpSetting(); + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(R"( + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + ("Primary1", "Secondary1", "Value1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); - } - - { - const TString query1(R"( + UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); + } + + { + const TString query1(R"( PRAGMA Kikimr.UseNewEngine = 'false'; REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1_1", "Value1"); - )"); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); + ("Primary1", "Secondary1_1", "Value1"); + )"); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); - UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); - } - } - - - Y_UNIT_TEST_QUAD(SecondaryIndexUpsert1DeleteUpdate, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); + } + } + + + Y_UNIT_TEST_QUAD(SecondaryIndexUpsert1DeleteUpdate, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + const auto& config = kikimr.GetTestServer().GetSettings().AppConfig; auto& tableSettings = config.GetTableServiceConfig(); bool useSchemeCacheMeta = tableSettings.GetUseSchemeCacheMetadata(); - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "SomeOldIndex", "SomeOldValue"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - const TString query1(Q_(R"( + ("Primary1", "SomeOldIndex", "SomeOldValue"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"), - ("Primary2", "Secondary2", "Value2"), - ("Primary3", "Secondary3", "Value3"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary2"];["Primary2"]];[["Secondary3"];["Primary3"]]])"; + ("Primary1", "Secondary1", "Value1"), + ("Primary2", "Secondary2", "Value2"), + ("Primary3", "Secondary3", "Value3"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]];[["Secondary2"];["Primary2"]];[["Secondary3"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - auto result = session.DescribeTable("/Root/TestTable/Index/indexImplTable").ExtractValueSync(); + } + + { + auto result = session.DescribeTable("/Root/TestTable/Index/indexImplTable").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( SELECT * FROM `/Root/TestTable/Index/indexImplTable`; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); // KIKIMR-7997 UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), useSchemeCacheMeta ? NYdb::EStatus::SCHEME_ERROR : NYdb::EStatus::GENERIC_ERROR); - } - - { + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/TestTable` VIEW WrongView WHERE Index2 = 'Secondary2'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - - { + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary2'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value2\"]]]"); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary1'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary2"];["Primary2"]];[["Secondary3"];["Primary3"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary2"];["Primary2"]];[["Secondary3"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary3"];["Primary3"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary3"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary3', 'Secondary3_1'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary3"];["Secondary3_1"];["Value3"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary3"];["Secondary3_1"];["Value3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary3_1"];["Primary3"]]])"; + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary3_1"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( UPDATE `/Root/TestTable` SET Index2 = 'Secondary3_2' WHERE Key = 'Primary3'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary3"];["Secondary3_2"];["Value3"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary3"];["Secondary3_2"];["Value3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary3_2"];["Primary3"]]])"; + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary3_2"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexUpsert2Update, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexUpsert2Update, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Index2A", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2", "Index2A"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Index2A", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2", "Index2A"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const TString query1(Q_(R"( + } + + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Index2A) VALUES - ("Primary1", "Secondary1", "Secondary1A"), - ("Primary2", "Secondary2", "Secondary2A"), - ("Primary3", "Secondary3", "Secondary3A"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; + ("Primary1", "Secondary1", "Secondary1A"), + ("Primary2", "Secondary2", "Secondary2A"), + ("Primary3", "Secondary3", "Secondary3A"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary1', 'Secondary1_1'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Secondary1_1"];["Secondary1A"]];[["Primary2"];["Secondary2"];["Secondary2A"]];[["Primary3"];["Secondary3"];["Secondary3A"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Secondary1_1"];["Secondary1A"]];[["Primary2"];["Secondary2"];["Secondary2A"]];[["Primary3"];["Secondary3"];["Secondary3A"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_1"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_1"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( UPDATE `/Root/TestTable` SET Index2 = 'Secondary1_2' WHERE Key = 'Primary1'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Secondary1_2"];["Secondary1A"]];[["Primary2"];["Secondary2"];["Secondary2A"]];[["Primary3"];["Secondary3"];["Secondary3A"]]])"; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Secondary1_2"];["Secondary1A"]];[["Primary2"];["Secondary2"];["Secondary2A"]];[["Primary3"];["Secondary3"];["Secondary3A"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1_2"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1_2"];["Secondary1A"];["Primary1"]];[["Secondary2"];["Secondary2A"];["Primary2"]];[["Secondary3"];["Secondary3A"];["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexUpdateOnUsingIndex, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexUpdateOnUsingIndex, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", "Index2"); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", "Index2"); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const TString query1(Q_(R"( + } + + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Val1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]]])"; + ("Primary1", "Secondary1", "Val1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - + } + + { + const TString query(Q1_(R"( UPDATE `/Root/TestTable` ON (Key, Index2, Value) (SELECT Key, Index2, 'Val1_1' as Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Secondary1"];["Val1_1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[["Secondary1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexSelect, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Secondary1"];["Val1_1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[["Secondary1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexSelect, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - { + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + { const TString query(Q1_(R"( SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); + } + + { const TString query(Q1_(R"( SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2"; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2"; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); + } + + { const TString query(Q1_(R"( SELECT Key, Fk, Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2" ORDER BY Fk, Key; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2];[2];[\"Payload2\"]]]"); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2];[2];[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - - if (UseNewEngine) { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + + if (UseNewEngine) { UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, [](const NYql::TIssue& issue) { return issue.Message.Contains("Given predicate is not suitable for used index: Index"); }), result.GetIssues().ToString()); - } else { + } else { UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, [](const NYql::TIssue& issue) { return issue.Message.Contains("Given predicate is not suitable for used index"); }), result.GetIssues().ToString()); - } - - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); - } - - { + } + + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload2\"]]]"); + } + + { const TString query(Q1_(R"( SELECT Fk FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - if (UseNewEngine) { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + if (UseNewEngine) { UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE, [](const NYql::TIssue& issue) { return issue.Message.Contains("Given predicate is not suitable for used index: Index"); }), result.GetIssues().ToString()); - } else { - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - } - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); - } - - { + } else { + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + } + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[2]]]"); + } + + { const TString query(Q1_(R"( SELECT Value, Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Key > 2 ORDER BY Key DESC; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload7\"];[7]];[[\"Payload5\"];[5]]]"); - - auto result2 = session.ExplainDataQuery(query) - .ExtractValueSync(); - - - UNIT_ASSERT(result2.IsSuccess()); - - // TODO: Enable after KIKIMR-10368 - //UNIT_ASSERT_C(result2.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result2.GetAst()); - //UNIT_ASSERT_C(result2.GetAst().Contains("'\"ItemsLimit\""), result2.GetAst()); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Payload7\"];[7]];[[\"Payload5\"];[5]]]"); + + auto result2 = session.ExplainDataQuery(query) + .ExtractValueSync(); + + + UNIT_ASSERT(result2.IsSuccess()); + + // TODO: Enable after KIKIMR-10368 + //UNIT_ASSERT_C(result2.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result2.GetAst()); + //UNIT_ASSERT_C(result2.GetAst().Contains("'\"ItemsLimit\""), result2.GetAst()); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Int64) - .AddNullableColumn("Index2", EPrimitiveType::Int64) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Int64) + .AddNullableColumn("Index2", EPrimitiveType::Int64) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - (1, 1001, "Value1"), - (2, 1002, "Value2"), - (3, 1003, "Value3"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { + (1, 1001, "Value1"), + (2, 1002, "Value2"), + (3, 1003, "Value3"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3];[\"Value3\"]];[[1002];[2];[\"Value2\"]];[[1001];[1];[\"Value1\"]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3];[\"Value3\"]];[[1002];[2];[\"Value2\"]];[[1001];[1];[\"Value1\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3];[\"Value3\"]];[[1002];[2];[\"Value2\"]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3];[\"Value3\"]];[[1002];[2];[\"Value2\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3]];[[1002];[2]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1003];[3]];[[1002];[2]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t WHERE t.Index2 < 1003 ORDER BY t.Index2 DESC; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]];[[1001];[1];[\"Value1\"]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]];[[1001];[1];[\"Value1\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1001];[1];[\"Value1\"]];[[1002];[2];[\"Value2\"]];[[1003];[3];[\"Value3\"]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1001];[1];[\"Value1\"]];[[1002];[2];[\"Value2\"]];[[1003];[3];[\"Value3\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - } else { - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1001];[1];[\"Value1\"]];[[1002];[2];[\"Value2\"]]]"); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + } else { + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1001];[1];[\"Value1\"]];[[1002];[2];[\"Value2\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t - WHERE t.Index2 > 1001 - ORDER BY t.Index2 LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]];[[1003];[3];[\"Value3\"]]]"); - } - } - - { + WHERE t.Index2 > 1001 + ORDER BY t.Index2 LIMIT 2; + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]];[[1003];[3];[\"Value3\"]]]"); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW Index as t - WHERE t.Index2 = 1002 - ORDER BY t.Index2 LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]]]"); - } - } - - - { + WHERE t.Index2 = 1002 + ORDER BY t.Index2 LIMIT 2; + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2];[\"Value2\"]]]"); + } + } + + + { const TString query(Q1_(R"( SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t - WHERE t.Index2 > 1001 - ORDER BY t.Index2 LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2]];[[1003];[3]]]"); - } - } - - { - // Request by Key but using Index table, expect correct result + WHERE t.Index2 > 1001 + ORDER BY t.Index2 LIMIT 2; + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2]];[[1003];[3]]]"); + } + } + + { + // Request by Key but using Index table, expect correct result const TString query(Q1_(R"( SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t - WHERE t.Key > 1 - ORDER BY t.Index2 LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - if (UseNewEngine) { - UNIT_ASSERT_C(!result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(!result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2]];[[1003];[3]]]"); - } - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy2, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); - auto serverSettings = TKikimrSettings() + WHERE t.Key > 1 + ORDER BY t.Index2 LIMIT 2; + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + if (UseNewEngine) { + UNIT_ASSERT_C(!result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(!result.GetAst().Contains("('\"ItemsLimit\""), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1002];[2]];[[1003];[3]]]"); + } + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy2, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) - .SetKqpSettings({setting}); - TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("id", EPrimitiveType::Uint64) - .AddNullableColumn("customer", EPrimitiveType::Utf8) - .AddNullableColumn("created", EPrimitiveType::Datetime) - .AddNullableColumn("processed", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"id"}); - tableBuilder.AddSecondaryIndex("ix_cust", TVector<TString>{"customer"}); - tableBuilder.AddSecondaryIndex("ix_cust2", TVector<TString>{"customer", "created"}); - tableBuilder.AddSecondaryIndex("ix_cust3", TVector<TString>{"customer", "created"}, TVector<TString>{"processed"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("id", EPrimitiveType::Uint64) + .AddNullableColumn("customer", EPrimitiveType::Utf8) + .AddNullableColumn("created", EPrimitiveType::Datetime) + .AddNullableColumn("processed", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"id"}); + tableBuilder.AddSecondaryIndex("ix_cust", TVector<TString>{"customer"}); + tableBuilder.AddSecondaryIndex("ix_cust2", TVector<TString>{"customer", "created"}); + tableBuilder.AddSecondaryIndex("ix_cust3", TVector<TString>{"customer", "created"}, TVector<TString>{"processed"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (id, customer, created, processed) VALUES - (1, "Vasya", CAST('2020-01-01T00:00:01Z' as DATETIME), "Value1"), - (2, "Vova", CAST('2020-01-01T00:00:02Z' as DATETIME), "Value2"), - (3, "Petya", CAST('2020-01-01T00:00:03Z' as DATETIME), "Value3"), - (4, "Vasya", CAST('2020-01-01T00:00:04Z' as DATETIME), "Value4"), - (5, "Vasya", CAST('2020-01-01T00:00:05Z' as DATETIME), "Value5"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - { + (1, "Vasya", CAST('2020-01-01T00:00:01Z' as DATETIME), "Value1"), + (2, "Vova", CAST('2020-01-01T00:00:02Z' as DATETIME), "Value2"), + (3, "Petya", CAST('2020-01-01T00:00:03Z' as DATETIME), "Value3"), + (4, "Vasya", CAST('2020-01-01T00:00:04Z' as DATETIME), "Value4"), + (5, "Vasya", CAST('2020-01-01T00:00:05Z' as DATETIME), "Value5"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW ix_cust as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]];[[1577836801u];[\"Vasya\"];[1u];[\"Value1\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 0; - int tablePhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 1; - tablePhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 3); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]];[[1577836801u];[\"Vasya\"];[1u];[\"Value1\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 0; + int tablePhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 1; + tablePhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 3); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW ix_cust2 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 0; - int tablePhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 1; - tablePhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).name(), "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).reads().rows(), 2); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust2/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - } - - { + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 0; + int tablePhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 1; + tablePhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).name(), "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(tablePhaseId).table_access(0).reads().rows(), 2); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust2/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + } + + { const TString query(Q1_(R"( SELECT * FROM `/Root/TestTable` VIEW ix_cust3 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2; - )")); - - { - auto result = session.ExplainDataQuery( - query) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); - - if (UseNewEngine) { - UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); - } - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 0; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - indexPhaseId = 1; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust3/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexReplace, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + { + auto result = session.ExplainDataQuery( + query) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString().c_str()); + + if (UseNewEngine) { + UNIT_ASSERT_C(result.GetAst().Contains("'('\"ItemsLimit"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); + } else { + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + UNIT_ASSERT_C(result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("PartialSort"), result.GetAst()); + } + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString().c_str()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[1577836805u];[\"Vasya\"];[5u];[\"Value5\"]];[[1577836804u];[\"Vasya\"];[4u];[\"Value4\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 0; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); + indexPhaseId = 1; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable/ix_cust3/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexReplace, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("KeyA", EPrimitiveType::Uint64) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Index2A", EPrimitiveType::Uint8) - .AddNullableColumn("Value", EPrimitiveType::Utf8); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2", "Index2A"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("KeyA", EPrimitiveType::Uint64) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Index2A", EPrimitiveType::Uint8) + .AddNullableColumn("Value", EPrimitiveType::Utf8); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2", "Index2A"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, KeyA, Index2, Index2A, Value) VALUES - ("Primary1", 41, "Secondary1", 1, "Value1"), - ("Primary2", 42, "Secondary2", 2, "Value2"), - ("Primary3", 43, "Secondary3", 3, "Value3"); - )")); - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = -R"([[["Primary1"];[41u];["Secondary1"];[1u];["Value1"]];[["Primary2"];[42u];["Secondary2"];[2u];["Value2"]];[["Primary3"];[43u];["Secondary3"];[3u];["Value3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = -R"([[["Secondary1"];[1u];["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Secondary3"];[3u];["Primary3"];[43u]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query(Q_(R"( + ("Primary1", 41, "Secondary1", 1, "Value1"), + ("Primary2", 42, "Secondary2", 2, "Value2"), + ("Primary3", 43, "Secondary3", 3, "Value3"); + )")); + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = +R"([[["Primary1"];[41u];["Secondary1"];[1u];["Value1"]];[["Primary2"];[42u];["Secondary2"];[2u];["Value2"]];[["Primary3"];[43u];["Secondary3"];[3u];["Value3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = +R"([[["Secondary1"];[1u];["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Secondary3"];[3u];["Primary3"];[43u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, KeyA, Value) VALUES - ("Primary1", 41, "Value1_1"); - )")); - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = -R"([[["Primary1"];[41u];#;#;["Value1_1"]];[["Primary2"];[42u];["Secondary2"];[2u];["Value2"]];[["Primary3"];[43u];["Secondary3"];[3u];["Value3"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = -R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Secondary3"];[3u];["Primary3"];[43u]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexInsert1, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + ("Primary1", 41, "Value1_1"); + )")); + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = +R"([[["Primary1"];[41u];#;#;["Value1_1"]];[["Primary2"];[42u];["Secondary2"];[2u];["Value2"]];[["Primary3"];[43u];["Secondary3"];[3u];["Value3"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = +R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Secondary3"];[3u];["Primary3"];[43u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexInsert1, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query(Q_(R"( + } + + { + const TString query(Q_(R"( INSERT INTO `/Root/TestTable` (Key, Value) VALUES - ("Primary1", "Value1"), - ("Primary2", "Value2"), - ("Primary3", "Value3"); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + ("Primary1", "Value1"), + ("Primary2", "Value2"), + ("Primary3", "Value3"); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - const TString expected = R"([[#;["Primary1"]];[#;["Primary2"]];[#;["Primary3"]]])"; + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + const TString expected = R"([[#;["Primary1"]];[#;["Primary2"]];[#;["Primary3"]]])"; UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(MultipleSecondaryIndex, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + } + } + + Y_UNIT_TEST_QUAD(MultipleSecondaryIndex, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value1", EPrimitiveType::String) - .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", "Value1"); - tableBuilder.AddSecondaryIndex("Index2", "Value2"); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value1", EPrimitiveType::String) + .AddNullableColumn("Value2", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", "Value1"); + tableBuilder.AddSecondaryIndex("Index2", "Value2"); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const TString query1(Q_(R"( + } + + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Value1, Value2) VALUES - ("Primary1", "Val1", "Val2"); - )")); - + ("Primary1", "Val1", "Val2"); + )")); + auto explainResult = session.ExplainDataQuery(query1).ExtractValueSync(); UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); @@ -3173,1271 +3173,1271 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index1/indexImplTable"); UNIT_ASSERT(tables.at(2).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index2/indexImplTable"); - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2"];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1"];["Val2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST_QUAD(MultipleSecondaryIndexWithSameComulns, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2"];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1"];["Val2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST_QUAD(MultipleSecondaryIndexWithSameComulns, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value1", EPrimitiveType::String) - .AddNullableColumn("Value2", EPrimitiveType::String) - .AddNullableColumn("Value3", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", TVector<TString>{"Value1", "Value3"}); - tableBuilder.AddSecondaryIndex("Index2", TVector<TString>{"Value2", "Value3"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value1", EPrimitiveType::String) + .AddNullableColumn("Value2", EPrimitiveType::String) + .AddNullableColumn("Value3", EPrimitiveType::Int64); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", TVector<TString>{"Value1", "Value3"}); + tableBuilder.AddSecondaryIndex("Index2", TVector<TString>{"Value2", "Value3"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES - ("Primary1", "Val1", "Val2", 42); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1"];["Val2"];[42]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { + ("Primary1", "Val1", "Val2", 42); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1"];["Val2"];[42]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { const TString query(Q1_(R"( SELECT Value2 FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Val2\"]]]"); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES - ("Primary1", "Val1_1", "Val2_1", 43); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_1"];[43];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];[43];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[43]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + ("Primary1", "Val1_1", "Val2_1", 43); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_1"];[43];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];[43];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[43]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, Value1, Value2) VALUES - ("Primary1", "Val1_1", "Val2_1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_1"];#;["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];#;["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + ("Primary1", "Val1_1", "Val2_1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_1"];#;["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];#;["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( UPDATE `/Root/TestTable` SET 'Value3' = 35; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_1"];[35];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];[35];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[35]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_1"];[35];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];[35];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[35]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( UPDATE `/Root/TestTable` ON (Key, Value3) VALUES ('Primary1', 36); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[36]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_1"];["Val2_1"];[36]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( UPDATE `/Root/TestTable` ON (Key, Value1) VALUES ('Primary1', 'Val1_2'); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_2"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_2"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( INSERT INTO `/Root/TestTable` (Key, Value2) VALUES ('Primary2', 'Record2'); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[#;#;["Primary2"]];[["Val1_2"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[#;#;["Primary2"]];[["Val1_2"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( INSERT INTO `/Root/TestTable` (Key, Value3) VALUES ('Primary3', 37); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[#;#;["Primary2"]];[#;[37];["Primary3"]];[["Val1_2"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[#;[37];["Primary3"]];[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#];[["Primary3"];#;#;[37]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[#;#;["Primary2"]];[#;[37];["Primary3"]];[["Val1_2"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[#;[37];["Primary3"]];[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#];[["Primary3"];#;#;[37]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( DELETE FROM `/Root/TestTable` WHERE Key = 'Primary3'; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[#;#;["Primary2"]];[["Val1_2"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[#;#;["Primary2"]];[["Val1_2"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Record2"];#;["Primary2"]];[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]];[["Primary2"];#;["Record2"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary2'); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_2"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_2"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Val2_1"];[36];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];["Val1_2"];["Val2_1"];[36]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( DELETE FROM `/Root/TestTable`; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexWithPrimaryKeySameComulns, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexWithPrimaryKeySameComulns, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("KeyA", EPrimitiveType::Int64) - .AddNullableColumn("Value1", EPrimitiveType::String) - .AddNullableColumn("Payload", EPrimitiveType::Utf8); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index1", TVector<TString>{"Value1", "KeyA"}); - tableBuilder.AddSecondaryIndex("Index2", TVector<TString>{"Key", "Value1"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("KeyA", EPrimitiveType::Int64) + .AddNullableColumn("Value1", EPrimitiveType::String) + .AddNullableColumn("Payload", EPrimitiveType::Utf8); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index1", TVector<TString>{"Value1", "KeyA"}); + tableBuilder.AddSecondaryIndex("Index2", TVector<TString>{"Key", "Value1"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES - ("Primary1", 42, "Val1", "SomeData"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Primary1"];["Val1"];[42]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1"];["SomeData"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { + ("Primary1", 42, "Val1", "SomeData"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Primary1"];["Val1"];[42]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1"];["SomeData"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { const TString query(Q1_(R"( SELECT Key FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Primary1\"]]]"); - } - - { + } + + { const TString query(Q1_(R"( SELECT Key, Value1, Payload FROM `/Root/TestTable` VIEW Index2 WHERE Key = 'Primary1' ORDER BY Key, Value1; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Primary1\"];[\"Val1\"];[\"SomeData\"]]]"); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Primary1\"];[\"Val1\"];[\"SomeData\"]]]"); + } + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES - ("Primary1", 42, "Val1_0", "SomeData2"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_0"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Primary1"];["Val1_0"];[42]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_0"];["SomeData2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + ("Primary1", 42, "Val1_0", "SomeData2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_0"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Primary1"];["Val1_0"];[42]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_0"];["SomeData2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES - ("Primary1", 42, "Val1_1"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_1"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); - const TString expected = R"([[["Primary1"];["Val1_1"];[42]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_1"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + ("Primary1", 42, "Val1_1"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_1"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index2/indexImplTable"); + const TString expected = R"([[["Primary1"];["Val1_1"];[42]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_1"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( REPLACE INTO `/Root/TestTable` (Key, KeyA) VALUES - ("Primary1", 42); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[#;[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];#;#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + ("Primary1", 42); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[#;[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];#;#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( UPDATE `/Root/TestTable` SET 'Value1' = 'Val1_2'; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_2"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_2"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_2"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_2"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( UPDATE `/Root/TestTable` ON (Key, KeyA, Value1) VALUES ('Primary1', 42, 'Val1_3'); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_3"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_3"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_3"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_3"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( INSERT INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES ('Primary2', 43, 'Val2'); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_3"];[42];["Primary1"]];[["Val2"];[43];["Primary2"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_3"];#];[["Primary2"];[43];["Val2"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_3"];[42];["Primary1"]];[["Val2"];[43];["Primary2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_3"];#];[["Primary2"];[43];["Val2"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2'; - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - const TString expected = R"([[["Val1_3"];[42];["Primary1"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - const TString expected = R"([[["Primary1"];[42];["Val1_3"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - { - const TString query1(Q_(R"( + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + const TString expected = R"([[["Val1_3"];[42];["Primary1"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + const TString expected = R"([[["Primary1"];[42];["Val1_3"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + { + const TString query1(Q_(R"( DELETE FROM `/Root/TestTable` ON (Key, KeyA) VALUES ('Primary1', 42); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - { - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - } - + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + } + Y_UNIT_TEST_TWIN(DeleteOnWithSubquery, WithMvcc) { - auto setting = NKikimrKqp::TKqpSetting(); + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - const TString query(R"( + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + const TString query(R"( --!syntax_v1 PRAGMA Kikimr.UseNewEngine = 'false'; DECLARE $keys AS List<Tuple<Int32, String>>; - $to_delete = ( + $to_delete = ( SELECT Key FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) in $keys - ); - DELETE FROM `/Root/SecondaryComplexKeys` ON - SELECT * FROM $to_delete; - )"); - - auto params = TParamsBuilder().AddParam("$keys").BeginList() - .AddListItem().BeginTuple().AddElement().Int32(1).AddElement().String("Fk1").EndTuple() - .EndList().Build().Build(); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, execSettings).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - const auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - // In theory we can optimize and remove this read access - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/SecondaryComplexKeys"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).reads().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access().size(), 2); - - // No guarantee effects will be in the same order - if (stats.query_phases(3).table_access(0).name() == "/Root/SecondaryComplexKeys") { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); - } else if (stats.query_phases(3).table_access(0).name() == "/Root/SecondaryComplexKeys/Index/indexImplTable") { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).name(), "/Root/SecondaryComplexKeys"); - } else { - Y_FAIL("unexpected table name"); - } - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(0).deletes().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).deletes().rows(), 1); - } - - Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + ); + DELETE FROM `/Root/SecondaryComplexKeys` ON + SELECT * FROM $to_delete; + )"); + + auto params = TParamsBuilder().AddParam("$keys").BeginList() + .AddListItem().BeginTuple().AddElement().Int32(1).AddElement().String("Fk1").EndTuple() + .EndList().Build().Build(); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, execSettings).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + const auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); + // In theory we can optimize and remove this read access + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).name(), "/Root/SecondaryComplexKeys"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).reads().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access().size(), 2); + + // No guarantee effects will be in the same order + if (stats.query_phases(3).table_access(0).name() == "/Root/SecondaryComplexKeys") { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); + } else if (stats.query_phases(3).table_access(0).name() == "/Root/SecondaryComplexKeys/Index/indexImplTable") { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).name(), "/Root/SecondaryComplexKeys"); + } else { + Y_FAIL("unexpected table name"); + } + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(0).deletes().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(3).table_access(1).deletes().rows(), 1); + } + + Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", "Value"); - auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::Int64); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", "Value"); + auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", "Value"); - auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); + } + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::Int64); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", "Value"); + auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - { - const TString query1(Q_(R"( + } + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES - ("Table1Primary3", 3), - ("Table1Primary4", 4), - ("Table1Primary55", 55); - + ("Table1Primary3", 3), + ("Table1Primary4", 4), + ("Table1Primary55", 55); + UPSERT INTO `/Root/TestTable2` (Key, Value) VALUES - ("Table2Primary1", 1), - ("Table2Primary2", 2), - ("Table2Primary3", 3), - ("Table2Primary4", 4), - ("Table2Primary5", 5); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - { + ("Table2Primary1", 1), + ("Table2Primary2", 2), + ("Table2Primary3", 3), + ("Table2Primary4", 4), + ("Table2Primary5", 5); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + { const TString query(Q1_(R"( SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1` INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + + { const TString query(Q1_(R"( SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1` INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + + { const TString query(Q1_(R"( SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1` LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]];[[\"Table1Primary55\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]];[[\"Table1Primary55\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + + { const TString query(Q1_(R"( SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1` LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary55\"];[55]];[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - } - - Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin2, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary55\"];[55]];[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 3); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + } + + Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin2, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::Int64); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", "Value"); - auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::Int64); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", "Value"); + auto result = session.CreateTable("/Root/TestTable1", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::Int64) - .AddNullableColumn("Value2", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index1", "Value"); - auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); + } + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::Int64) + .AddNullableColumn("Value2", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index1", "Value"); + auto result = session.CreateTable("/Root/TestTable2", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - { - const TString query1(Q_(R"( + } + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES - ("Table1Primary3", 3), - ("Table1Primary4", 4), - ("Table1Primary55", 55); - + ("Table1Primary3", 3), + ("Table1Primary4", 4), + ("Table1Primary55", 55); + UPSERT INTO `/Root/TestTable2` (Key, Value, Value2) VALUES - ("Table2Primary1", 1, "aa"), - ("Table2Primary2", 2, "bb"), - ("Table2Primary3", 3, "cc"), - ("Table2Primary4", 4, "dd"), - ("Table2Primary5", 5, "ee"); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - { + ("Table2Primary1", 1, "aa"), + ("Table2Primary2", 2, "bb"), + ("Table2Primary3", 3, "cc"), + ("Table2Primary4", 4, "dd"), + ("Table2Primary5", 5, "ee"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { const TString query(Q1_(R"( SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1 INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - - indexPhaseId++; - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - - { + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + + indexPhaseId++; + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + + { const TString query(Q1_(R"( SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1 LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - execSettings) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), - "[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]];[[\"Table1Primary55\"];#]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - - int indexPhaseId = 1; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); - indexPhaseId = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - - indexPhaseId++; - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); - } - } - - Y_UNIT_TEST_QUAD(ForbidViewModification, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), + "[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]];[[\"Table1Primary55\"];#]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + + int indexPhaseId = 1; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); + indexPhaseId = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TestTable1"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3); + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2/Index1/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + + indexPhaseId++; + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).name(), "/Root/TestTable2"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(indexPhaseId).table_access(0).reads().rows(), 2); + } + } + + Y_UNIT_TEST_QUAD(ForbidViewModification, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - const TString query1(Q_(R"( + + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"); - )")); - - auto result1 = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result1.IsSuccess()); - } - - { + ("Primary1", "Secondary1", "Value1"); + )")); + + auto result1 = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result1.IsSuccess()); + } + + { const TString query(Q1_(R"( INSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - - { + } + + { const TString query(Q1_(R"( UPSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - - { + } + + { const TString query(Q1_(R"( UPDATE `/Root/TestTable` VIEW Index ON (Index2, Key) VALUES ('Primary1', 'Secondary1'); - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - - { + } + + { const TString query(Q1_(R"( DELETE FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1'; - )")); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); UNIT_ASSERT(result.GetIssues().ToString().Contains("Unexpected token")); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - - } - + } + + } + Y_UNIT_TEST_TWIN(ForbidDirectIndexTableCreation, WithMvcc) { - auto setting = NKikimrKqp::TKqpSetting(); + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto scheme = kikimr.GetSchemeClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto scheme = kikimr.GetSchemeClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Value", EPrimitiveType::String) - .AddNullableColumn("Key", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Value"}); - auto result = session.CreateTable("/Root/TestTable/Index", tableBuilder.Build()).ExtractValueSync(); + } + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Value", EPrimitiveType::String) + .AddNullableColumn("Key", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Value"}); + auto result = session.CreateTable("/Root/TestTable/Index", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - - { - auto result = scheme.MakeDirectory("/Root/TestTable/Index").ExtractValueSync(); + } + + { + auto result = scheme.MakeDirectory("/Root/TestTable/Index").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - - { - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Value", EPrimitiveType::String) - .AddNullableColumn("Key", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Value"}); - auto result = session.CreateTable("/Root/TestTable/Index/indexImplTable", tableBuilder.Build()).ExtractValueSync(); + } + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Value", EPrimitiveType::String) + .AddNullableColumn("Key", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Value"}); + auto result = session.CreateTable("/Root/TestTable/Index/indexImplTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - } - - Y_UNIT_TEST_QUAD(DuplicateUpsert, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + } + } + + Y_UNIT_TEST_QUAD(DuplicateUpsert, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - { - const TString query(Q_(R"( + + { + const TString query(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"), - ("Primary1", "Secondary2", "Value2"), - ("Primary1", "Secondary3", "Value3"); - )")); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - CompareYson(R"([[["Secondary3"];["Primary1"]]])", - ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); - } - } - - Y_UNIT_TEST_QUAD(DuplicateUpsertInterleave, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + ("Primary1", "Secondary1", "Value1"), + ("Primary1", "Secondary2", "Value2"), + ("Primary1", "Secondary3", "Value3"); + )")); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + CompareYson(R"([[["Secondary3"];["Primary1"]]])", + ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); + } + } + + Y_UNIT_TEST_QUAD(DuplicateUpsertInterleave, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - { - const TString query(Q_(R"( + + { + const TString query(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"), - ("Primary2", "Secondary2", "Value2"), - ("Primary1", "Secondary11", "Value3"), - ("Primary2", "Secondary22", "Value3"); - )")); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([[["Secondary11"];["Primary1"]];[["Secondary22"];["Primary2"]]])", - ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); - } - } - - Y_UNIT_TEST_QUAD(DuplicateUpsertInterleaveParams, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + ("Primary1", "Secondary1", "Value1"), + ("Primary2", "Secondary2", "Value2"), + ("Primary1", "Secondary11", "Value3"), + ("Primary2", "Secondary22", "Value3"); + )")); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([[["Secondary11"];["Primary1"]];[["Secondary22"];["Primary2"]]])", + ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); + } + } + + Y_UNIT_TEST_QUAD(DuplicateUpsertInterleaveParams, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - { + + { const TString query(Q1_(R"( - DECLARE $rows AS + DECLARE $rows AS List<Struct< - Key: String?, - Index2: String?, + Key: String?, + Index2: String?, Value: String?>>; - + UPSERT INTO `/Root/TestTable` - SELECT Key, Index2, Value FROM AS_TABLE($rows); - )")); - + SELECT Key, Index2, Value FROM AS_TABLE($rows); + )")); + auto explainResult = session.ExplainDataQuery(query).ExtractValueSync(); UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); @@ -4450,96 +4450,96 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda UNIT_ASSERT(tables.at(0).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable"); UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index/indexImplTable"); - auto qId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery(); - - auto params = qId.GetParamsBuilder() - .AddParam("$rows") - .BeginList() - .AddListItem() - .BeginStruct() - .AddMember("Key").OptionalString("Primary1") - .AddMember("Index2").OptionalString("Secondary1") - .AddMember("Value").OptionalString("Value1") - .EndStruct() - .AddListItem() - .BeginStruct() - .AddMember("Key").OptionalString("Primary2") - .AddMember("Index2").OptionalString("Secondary2") - .AddMember("Value").OptionalString("Value2") - .EndStruct() - .AddListItem() - .BeginStruct() - .AddMember("Key").OptionalString("Primary1") - .AddMember("Index2").OptionalString("Secondary11") - .AddMember("Value").OptionalString("Value1") - .EndStruct() - .AddListItem() - .BeginStruct() - .AddMember("Key").OptionalString("Primary2") - .AddMember("Index2").OptionalString("Secondary22") - .AddMember("Value").OptionalString("Value2") - .EndStruct() - .EndList() - .Build() - .Build(); - - auto result = qId.Execute( - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - std::move(params)).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([[["Secondary11"];["Primary1"]];[["Secondary22"];["Primary2"]]])", - ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); - } - } - - Y_UNIT_TEST_QUAD(MultipleModifications, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); + auto qId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery(); + + auto params = qId.GetParamsBuilder() + .AddParam("$rows") + .BeginList() + .AddListItem() + .BeginStruct() + .AddMember("Key").OptionalString("Primary1") + .AddMember("Index2").OptionalString("Secondary1") + .AddMember("Value").OptionalString("Value1") + .EndStruct() + .AddListItem() + .BeginStruct() + .AddMember("Key").OptionalString("Primary2") + .AddMember("Index2").OptionalString("Secondary2") + .AddMember("Value").OptionalString("Value2") + .EndStruct() + .AddListItem() + .BeginStruct() + .AddMember("Key").OptionalString("Primary1") + .AddMember("Index2").OptionalString("Secondary11") + .AddMember("Value").OptionalString("Value1") + .EndStruct() + .AddListItem() + .BeginStruct() + .AddMember("Key").OptionalString("Primary2") + .AddMember("Index2").OptionalString("Secondary22") + .AddMember("Value").OptionalString("Value2") + .EndStruct() + .EndList() + .Build() + .Build(); + + auto result = qId.Execute( + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + std::move(params)).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([[["Secondary11"];["Primary1"]];[["Secondary22"];["Primary2"]]])", + ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable")); + } + } + + Y_UNIT_TEST_QUAD(MultipleModifications, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto tableBuilder = db.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::String) - .AddNullableColumn("Index2", EPrimitiveType::String) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); - tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); - auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("Index2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + tableBuilder.AddSecondaryIndex("Index", TVector<TString>{"Index2"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - { - const TString query1(Q_(R"( + + { + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES - ("Primary1", "Secondary1", "Value1"), - ("Primary2", "Secondary2", "Value2"); - )")); - - auto result = session.ExecuteDataQuery(query1, TTxControl::BeginTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - auto tx = result.GetTransaction(); - - - const TString query2(Q_(R"( + ("Primary1", "Secondary1", "Value1"), + ("Primary2", "Secondary2", "Value2"); + )")); + + auto result = session.ExecuteDataQuery(query1, TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto tx = result.GetTransaction(); + + + const TString query2(Q_(R"( DELETE FROM `/Root/TestTable` ON (Key) VALUES - ("Primary1"), - ("Primary2"); - )")); - - result = session.ExecuteDataQuery(query2, TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - - const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, ""); - } - } + ("Primary1"), + ("Primary2"); + )")); + + result = session.ExecuteDataQuery(query2, TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, ""); + } + } template <bool WithMvcc, bool UseNewEngine> void CreateTableWithIndexSQL(EIndexTypeSql type, bool enableAsyncIndexes = false) { @@ -4553,52 +4553,52 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetEnableAsyncIndexes(enableAsyncIndexes) .SetKqpSettings({kqpSetting}); TKikimrRunner kikimr(settings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + const auto typeStr = IndexTypeSqlString(type); const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::GENERIC_ERROR) : EStatus::SUCCESS; const TString createTableSql = Sprintf("CREATE TABLE `/Root/TestTable` (" - " Key Int32, IndexA Int32, IndexB Int32, IndexC String, Value String," - " PRIMARY KEY (Key)," + " Key Int32, IndexA Int32, IndexB Int32, IndexC String, Value String," + " PRIMARY KEY (Key)," " INDEX SecondaryIndex1 %s ON (IndexA, IndexB)," " INDEX SecondaryIndex2 %s ON (IndexC)" ")", typeStr.data(), typeStr.data()); - { - auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); + { + auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); - } - - // Multiple create table requests with same scheme should be OK - { - - auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); + } + + // Multiple create table requests with same scheme should be OK + { + + auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); - } + } if (type == EIndexTypeSql::GlobalAsync && !enableAsyncIndexes) { return; } - // Check we can use index (metadata is correct) + // Check we can use index (metadata is correct) const TString query1(Q_(R"( - UPSERT INTO `/Root/TestTable` (Key, IndexA, IndexB, IndexC, Value) VALUES - (1, 11, 111, "a", "Value1"), - (2, 22, 222, "b", "Value2"), - (3, 33, 333, "c", "Value3"); + UPSERT INTO `/Root/TestTable` (Key, IndexA, IndexB, IndexC, Value) VALUES + (1, 11, 111, "a", "Value1"), + (2, 22, 222, "b", "Value2"), + (3, 33, 333, "c", "Value3"); )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); auto waitForAsyncIndexContent = [&session](const TString& indexImplTable, const TString& expected) { while (true) { @@ -4611,7 +4611,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda } }; - { + { const TString indexImplTable = "/Root/TestTable/SecondaryIndex1/indexImplTable"; const TString expected = R"([[[11];[111];[1]];[[22];[222];[2]];[[33];[333];[3]]])"; if (!enableAsyncIndexes) { @@ -4619,8 +4619,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda } else { waitForAsyncIndexContent(indexImplTable, expected); } - } - { + } + { const TString indexImplTable = "/Root/TestTable/SecondaryIndex2/indexImplTable"; const TString expected = R"([[["a"];[1]];[["b"];[2]];[["c"];[3]]])"; if (!enableAsyncIndexes) { @@ -4628,9 +4628,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda } else { waitForAsyncIndexContent(indexImplTable, expected); } - } - } - + } + } + Y_UNIT_TEST_QUAD(CreateTableWithImplicitSyncIndexSQL, WithMvcc, UseNewEngine) { CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::Global); } @@ -4708,269 +4708,269 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda SelectFromAsyncIndexedTable<WithMvcc, UseNewEngine>(); } - Y_UNIT_TEST_QUAD(InnerJoinWithNonIndexWherePredicate, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_KqpYqlSyntaxVersion"); - setting.SetValue("1"); + Y_UNIT_TEST_QUAD(InnerJoinWithNonIndexWherePredicate, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_KqpYqlSyntaxVersion"); + setting.SetValue("1"); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - { - const TString query1(Q_(R"( - DECLARE $mongo_key AS Utf8?; - DECLARE $targets AS List<Struct<fk1:Int32>>; - DECLARE $limit AS Uint64; - - SELECT t.Value AS value - FROM AS_TABLE($targets) AS k - INNER JOIN `/Root/SecondaryComplexKeys` VIEW Index AS t - ON t.Fk1 = k.fk1 - WHERE Fk2 > $mongo_key - LIMIT $limit; - )")); - - auto result = session.ExplainDataQuery( - query1) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - UNIT_ASSERT_C(!result.GetAst().Contains("EquiJoin"), result.GetAst()); - - auto params = TParamsBuilder() - .AddParam("$targets") - .BeginList() - .AddListItem() - .BeginStruct() - .AddMember("fk1").Int32(1) - .EndStruct() - .EndList().Build() - .AddParam("$mongo_key") - .OptionalUtf8("") - .Build() - .AddParam("$limit") - .Uint64(2) - .Build() - .Build(); - - - auto result2 = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx().CommitTx(), - params, - execSettings).ExtractValueSync(); - - UNIT_ASSERT_C(result2.IsSuccess(), result2.GetIssues().ToString()); - UNIT_ASSERT(result2.GetIssues().Empty()); - - UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result2.GetResultSet(0)), "[[[\"Payload1\"]]]"); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result2.GetStats()); - - int readPhase = 0; - if (UseNewEngine) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); - readPhase = 2; - } - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).reads().rows(), 1); - - readPhase++; - - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).name(), "/Root/SecondaryComplexKeys"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).reads().rows(), 1); - } - } - - //KIKIMR-8144 - Y_UNIT_TEST_QUAD(InnerJoinSecondaryIndexLookupAndRightTablePredicateNonIndexColumn, WithMvcc, UseNewEngine) { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_KqpYqlSyntaxVersion"); - setting.SetValue("1"); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + { + const TString query1(Q_(R"( + DECLARE $mongo_key AS Utf8?; + DECLARE $targets AS List<Struct<fk1:Int32>>; + DECLARE $limit AS Uint64; + + SELECT t.Value AS value + FROM AS_TABLE($targets) AS k + INNER JOIN `/Root/SecondaryComplexKeys` VIEW Index AS t + ON t.Fk1 = k.fk1 + WHERE Fk2 > $mongo_key + LIMIT $limit; + )")); + + auto result = session.ExplainDataQuery( + query1) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + UNIT_ASSERT_C(!result.GetAst().Contains("EquiJoin"), result.GetAst()); + + auto params = TParamsBuilder() + .AddParam("$targets") + .BeginList() + .AddListItem() + .BeginStruct() + .AddMember("fk1").Int32(1) + .EndStruct() + .EndList().Build() + .AddParam("$mongo_key") + .OptionalUtf8("") + .Build() + .AddParam("$limit") + .Uint64(2) + .Build() + .Build(); + + + auto result2 = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx().CommitTx(), + params, + execSettings).ExtractValueSync(); + + UNIT_ASSERT_C(result2.IsSuccess(), result2.GetIssues().ToString()); + UNIT_ASSERT(result2.GetIssues().Empty()); + + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result2.GetResultSet(0)), "[[[\"Payload1\"]]]"); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result2.GetStats()); + + int readPhase = 0; + if (UseNewEngine) { + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 4); + readPhase = 2; + } + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).name(), "/Root/SecondaryComplexKeys/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).reads().rows(), 1); + + readPhase++; + + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).name(), "/Root/SecondaryComplexKeys"); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(readPhase).table_access(0).reads().rows(), 1); + } + } + + //KIKIMR-8144 + Y_UNIT_TEST_QUAD(InnerJoinSecondaryIndexLookupAndRightTablePredicateNonIndexColumn, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_KqpYqlSyntaxVersion"); + setting.SetValue("1"); auto serverSettings = TKikimrSettings() .SetEnableMvcc(WithMvcc) .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - const TString createTableSql = R"(CREATE TABLE `/Root/user` ( - id Uint64, - uid Uint64, - yandexuid Utf8, - PRIMARY KEY (id), - INDEX uid_index GLOBAL ON (uid), - INDEX yandexuid_index GLOBAL ON (yandexuid) - );)"; - { - auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT(result.GetIssues().Empty()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - const TString query1(Q_(R"( - UPSERT INTO `/Root/user` (id, yandexuid, uid) VALUES - (2, "def", 222), - (1, "abc", NULL), - (NULL, "ghi", 333); - )")); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/user"); - const TString expected = R"([[#;[333u];["ghi"]];[[1u];#;["abc"]];[[2u];[222u];["def"]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - - auto execQuery = [&session](const TString& query) { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - return NYdb::FormatResultSetYson(result.GetResultSet(0)); - }; - - const TString query_template(Q_(R"( - $yandexuids = (AsList(AsStruct(CAST("abc" as Utf8) as yandexuid), AsStruct(CAST("def" as Utf8) as yandexuid))); - SELECT t.id as id, t.yandexuid as yandexuid, t.uid as uid - FROM AS_TABLE($yandexuids) AS k - %s JOIN `/Root/user` VIEW yandexuid_index AS t - ON t.yandexuid = k.yandexuid - %s - ;)")); - - { - const TString query = Sprintf(query_template.data(), "INNER", "WHERE uid IS NULL"); - const TString expected = R"([[[1u];["abc"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "INNER", "WHERE uid IS NOT NULL"); - const TString expected = R"([[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "LEFT", "WHERE uid IS NULL"); - const TString expected = R"([[[1u];["abc"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "LEFT", "WHERE uid IS NOT NULL"); - const TString expected = R"([[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NULL"); - const TString expected = R"([[[1u];["abc"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY id DESC"); - const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY id"); - const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY yandexuid"); - const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY yandexuid DESC"); - const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY uid"); - const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY uid DESC"); - const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL"); - const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT ONLY", "WHERE uid IS NULL"); - const TString expected = R"([])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT ONLY", "WHERE uid IS NOT NULL"); - const TString expected = R"([[#;["ghi"];[333u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT SEMI", "WHERE uid IS NULL"); - const TString expected = R"([[[1u];["abc"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - { - const TString query = Sprintf(query_template.data(), "RIGHT SEMI", "WHERE uid IS NOT NULL"); - const TString expected = R"([[[2u];["def"];[222u]]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); - } - - // Without using index by pk directly - { - const TString query1(Q_(R"( - $ids = (AsList(AsStruct(CAST(1 as Uint64) as id), AsStruct(CAST(2 as Uint64) as id))); - - SELECT t.id as id, t.yandexuid as yandexuid, t.uid as uid - FROM AS_TABLE($ids) AS k - INNER JOIN `/Root/user` AS t - ON t.id = k.id - WHERE uid IS NULL - ;)")); - const TString expected = R"([[[1u];["abc"];#]])"; - UNIT_ASSERT_VALUES_EQUAL(execQuery(query1), expected); - } - } + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + const TString createTableSql = R"(CREATE TABLE `/Root/user` ( + id Uint64, + uid Uint64, + yandexuid Utf8, + PRIMARY KEY (id), + INDEX uid_index GLOBAL ON (uid), + INDEX yandexuid_index GLOBAL ON (yandexuid) + );)"; + { + auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT(result.GetIssues().Empty()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + const TString query1(Q_(R"( + UPSERT INTO `/Root/user` (id, yandexuid, uid) VALUES + (2, "def", 222), + (1, "abc", NULL), + (NULL, "ghi", 333); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/user"); + const TString expected = R"([[#;[333u];["ghi"]];[[1u];#;["abc"]];[[2u];[222u];["def"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + auto execQuery = [&session](const TString& query) { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + return NYdb::FormatResultSetYson(result.GetResultSet(0)); + }; + + const TString query_template(Q_(R"( + $yandexuids = (AsList(AsStruct(CAST("abc" as Utf8) as yandexuid), AsStruct(CAST("def" as Utf8) as yandexuid))); + SELECT t.id as id, t.yandexuid as yandexuid, t.uid as uid + FROM AS_TABLE($yandexuids) AS k + %s JOIN `/Root/user` VIEW yandexuid_index AS t + ON t.yandexuid = k.yandexuid + %s + ;)")); + + { + const TString query = Sprintf(query_template.data(), "INNER", "WHERE uid IS NULL"); + const TString expected = R"([[[1u];["abc"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "INNER", "WHERE uid IS NOT NULL"); + const TString expected = R"([[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "LEFT", "WHERE uid IS NULL"); + const TString expected = R"([[[1u];["abc"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "LEFT", "WHERE uid IS NOT NULL"); + const TString expected = R"([[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NULL"); + const TString expected = R"([[[1u];["abc"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY id DESC"); + const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY id"); + const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY yandexuid"); + const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY yandexuid DESC"); + const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY uid"); + const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL ORDER BY uid DESC"); + const TString expected = R"([[#;["ghi"];[333u]];[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT", "WHERE uid IS NOT NULL"); + const TString expected = R"([[[2u];["def"];[222u]];[#;["ghi"];[333u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT ONLY", "WHERE uid IS NULL"); + const TString expected = R"([])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT ONLY", "WHERE uid IS NOT NULL"); + const TString expected = R"([[#;["ghi"];[333u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT SEMI", "WHERE uid IS NULL"); + const TString expected = R"([[[1u];["abc"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + { + const TString query = Sprintf(query_template.data(), "RIGHT SEMI", "WHERE uid IS NOT NULL"); + const TString expected = R"([[[2u];["def"];[222u]]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query), expected); + } + + // Without using index by pk directly + { + const TString query1(Q_(R"( + $ids = (AsList(AsStruct(CAST(1 as Uint64) as id), AsStruct(CAST(2 as Uint64) as id))); + + SELECT t.id as id, t.yandexuid as yandexuid, t.uid as uid + FROM AS_TABLE($ids) AS k + INNER JOIN `/Root/user` AS t + ON t.id = k.id + WHERE uid IS NULL + ;)")); + const TString expected = R"([[[1u];["abc"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(execQuery(query1), expected); + } + } Y_UNIT_TEST_QUAD(DeleteByIndex, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); @@ -5172,7 +5172,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda CompareYson(R"([[#];[[3]];[[7]]])", FormatResultSetYson(result.GetResultSet(1))); CompareYson(R"([[[2]]])", FormatResultSetYson(result.GetResultSet(2))); } -} - -} -} +} + +} +} diff --git a/ydb/core/kqp/ut/kqp_join_ut.cpp b/ydb/core/kqp/ut/kqp_join_ut.cpp index bcbf8607fa..8fa59d31b3 100644 --- a/ydb/core/kqp/ut/kqp_join_ut.cpp +++ b/ydb/core/kqp/ut/kqp_join_ut.cpp @@ -39,21 +39,21 @@ static TParams BuildPureTableParams(TTableClient& client) { static void CreateSampleTables(TSession session) { UNIT_ASSERT(session.ExecuteSchemeQuery(R"( - CREATE TABLE `/Root/Join1_1` ( + CREATE TABLE `/Root/Join1_1` ( Key Int32, Fk21 Int32, Fk22 String, Value String, PRIMARY KEY (Key) ); - CREATE TABLE `/Root/Join1_2` ( + CREATE TABLE `/Root/Join1_2` ( Key1 Int32, Key2 String, Fk3 String, Value String, PRIMARY KEY (Key1, Key2) ); - CREATE TABLE `/Root/Join1_3` ( + CREATE TABLE `/Root/Join1_3` ( Key String, Value Int32, PRIMARY KEY (Key) @@ -63,7 +63,7 @@ static void CreateSampleTables(TSession session) { UNIT_ASSERT(session.ExecuteDataQuery(R"( PRAGMA kikimr.UseNewEngine = "true"; - REPLACE INTO `/Root/Join1_1` (Key, Fk21, Fk22, Value) VALUES + REPLACE INTO `/Root/Join1_1` (Key, Fk21, Fk22, Value) VALUES (1, 101, "One", "Value1"), (2, 102, "Two", "Value1"), (3, 103, "One", "Value2"), @@ -73,7 +73,7 @@ static void CreateSampleTables(TSession session) { (7, 107, "One", "Value4"), (8, 108, "One", "Value5"); - REPLACE INTO `/Root/Join1_2` (Key1, Key2, Fk3, Value) VALUES + REPLACE INTO `/Root/Join1_2` (Key1, Key2, Fk3, Value) VALUES (101, "One", "Name1", "Value21"), (101, "Two", "Name1", "Value22"), (101, "Three", "Name3", "Value23"), @@ -86,7 +86,7 @@ static void CreateSampleTables(TSession session) { (108, "One", NULL, "Value31"), (109, "Four", NULL, "Value41"); - REPLACE INTO `/Root/Join1_3` (Key, Value) VALUES + REPLACE INTO `/Root/Join1_3` (Key, Value) VALUES ("Name1", 1001), ("Name2", 1002), ("Name4", 1004); @@ -264,46 +264,46 @@ Y_UNIT_TEST_SUITE(KqpJoin) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 3); } - Y_UNIT_TEST_NEW_ENGINE(IdxLookupPartialWithTempTable) { - TKikimrRunner kikimr(SyntaxV1Settings()); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTables(session); - - auto params = TParamsBuilder() - .AddParam("$in") - .BeginList() - .AddListItem() - .BeginStruct() - .AddMember("k").Int32(101) - .EndStruct() - .EndList().Build() - .Build(); - - - const TString query = Q_(R"( - DECLARE $in AS List<Struct<k: Int32>>; - SELECT * FROM AS_TABLE($in) AS t1 - INNER JOIN `/Root/Join1_2` AS t2 - ON t1.k == t2.Key1; - )"); - - const TString expected = R"( - [ - [["Name1"];[101];["One"];["Value21"];101]; - [["Name3"];[101];["Three"];["Value23"];101]; - [["Name1"];[101];["Two"];["Value22"];101] - ] - )"; - - auto result = ExecQuery(session, query, params, expected, false); - AssertTableReads(result, "/Root/Join1_2", 3); - } - + Y_UNIT_TEST_NEW_ENGINE(IdxLookupPartialWithTempTable) { + TKikimrRunner kikimr(SyntaxV1Settings()); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTables(session); + + auto params = TParamsBuilder() + .AddParam("$in") + .BeginList() + .AddListItem() + .BeginStruct() + .AddMember("k").Int32(101) + .EndStruct() + .EndList().Build() + .Build(); + + + const TString query = Q_(R"( + DECLARE $in AS List<Struct<k: Int32>>; + SELECT * FROM AS_TABLE($in) AS t1 + INNER JOIN `/Root/Join1_2` AS t2 + ON t1.k == t2.Key1; + )"); + + const TString expected = R"( + [ + [["Name1"];[101];["One"];["Value21"];101]; + [["Name3"];[101];["Three"];["Value23"];101]; + [["Name1"];[101];["Two"];["Value22"];101] + ] + )"; + + auto result = ExecQuery(session, query, params, expected, false); + AssertTableReads(result, "/Root/Join1_2", 3); + } + Y_UNIT_TEST_NEW_ENGINE(LeftJoinWithNull) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -501,7 +501,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { } // join on secondary index => index-lookup - Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_SecondaryIndex) { + Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_SecondaryIndex) { TKikimrRunner kikimr(SyntaxV1Settings()); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -527,13 +527,13 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with parameters */ { - const TString query = Q_(R"( + const TString query = Q_(R"( DECLARE $in AS List<Struct<v: String?>>; SELECT * FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r ON l.v = r.SubKey ORDER BY Key - )"); + )"); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct() @@ -551,12 +551,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with real table */ { - const TString query = Q_(R"( + const TString query = Q_(R"( SELECT * FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r ON l.Value = r.SubKey ORDER BY Key - )"); + )"); auto result = ExecQuery(session, query, NoParams, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])"); AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */); @@ -567,7 +567,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { } // join on complex secondary index => index-lookup - Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndex) { + Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndex) { TKikimrRunner kikimr(SyntaxV1Settings()); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -576,13 +576,13 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with parameters */ { - const TString query = Q_(R"( + const TString query = Q_(R"( DECLARE $in AS List<Struct<k: Int32?, v: String?>>; SELECT * FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r ON l.k = r.SubKey1 AND l.v = r.SubKey2 ORDER BY Key - )"); + )"); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1) @@ -605,12 +605,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with real table */ { - const TString query = Q_(R"( + const TString query = Q_(R"( SELECT * FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r ON l.Key = r.SubKey1 AND l.Value = r.SubKey2 ORDER BY Key - )"); + )"); auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])"); AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */); @@ -622,7 +622,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { } // join on secondary index prefix => index-lookup - Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndexPrefix) { + Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndexPrefix) { TKikimrRunner kikimr(SyntaxV1Settings()); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -631,13 +631,13 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with parameters */ { - const TString query = Q_(R"( + const TString query = Q_(R"( DECLARE $in AS List<Struct<k: Int32?>>; SELECT * FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r ON l.k = r.SubKey1 ORDER BY Key - )"); + )"); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() @@ -655,13 +655,13 @@ Y_UNIT_TEST_SUITE(KqpJoin) { /* join with real table */ { - const TString query = Q_(R"( + const TString query = Q_(R"( SELECT * FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r ON l.Key = r.SubKey1 -- WHERE r.Key > 1 ORDER BY Key - )"); + )"); auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])"); AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */); @@ -672,7 +672,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { } template<bool UseNewEngine = false> - void TestInnerJoinWithPredicate(const TString& predicate, const TString& expected) { + void TestInnerJoinWithPredicate(const TString& predicate, const TString& expected) { TKikimrRunner kikimr(SyntaxV1Settings()); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -695,36 +695,36 @@ Y_UNIT_TEST_SUITE(KqpJoin) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - const TString query = Sprintf(R"( + const TString query = Sprintf(R"( DECLARE $in AS List<Struct<k: Int32?>>; SELECT * FROM AS_TABLE($in) AS l INNER JOIN `/Root/SecondaryKeys` VIEW Index AS r ON l.k = r.Fk - WHERE %s + WHERE %s ORDER BY Key - )", predicate.c_str()); + )", predicate.c_str()); auto params = TParamsBuilder().AddParam("$in").BeginList() - .AddListItem().BeginStruct().AddMember("k").OptionalInt32(105).EndStruct() + .AddListItem().BeginStruct().AddMember("k").OptionalInt32(105).EndStruct() .EndList().Build().Build(); result = session.ExecuteDataQuery(Q_(query), TTxControl::BeginTx().CommitTx(), params) .ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - CompareYson(expected, FormatResultSetYson(result.GetResultSet(0))); + CompareYson(expected, FormatResultSetYson(result.GetResultSet(0))); } - + Y_UNIT_TEST_NEW_ENGINE(RightTableKeyPredicate) { TestInnerJoinWithPredicate<UseNewEngine>("r.Key > 1", "[[[105];[5];[\"Payload2\"];[105]]]"); - } - + } + Y_UNIT_TEST_NEW_ENGINE(RightTableIndexPredicate) { TestInnerJoinWithPredicate<UseNewEngine>("r.Fk > 1", "[[[105];[5];[\"Payload2\"];[105]]]"); - } - + } + Y_UNIT_TEST_NEW_ENGINE(RightTableValuePredicate) { TestInnerJoinWithPredicate<UseNewEngine>("r.Value = \"Payload2\"", "[[[105];[5];[\"Payload2\"];[105]]]"); - } + } Y_UNIT_TEST_NEW_ENGINE(JoinAggregateSingleRow) { TKikimrRunner kikimr; @@ -768,37 +768,37 @@ Y_UNIT_TEST_SUITE(KqpJoin) { auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); - { + { auto result = session.ExecuteDataQuery(Q_(R"( - SELECT t1.Value, SUM(t3.Value) + SELECT t1.Value, SUM(t3.Value) FROM `/Root/Join1_1` AS t1 INNER JOIN `/Root/Join1_2` AS t2 - ON t1.Fk21 == t2.Key1 + ON t1.Fk21 == t2.Key1 LEFT JOIN `/Root/Join1_3` AS t3 - ON t2.Fk3 = t3.Key - GROUP BY t1.Value; + ON t2.Fk3 = t3.Key + GROUP BY t1.Value; )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]];[["Value5"];#]])", - FormatResultSetYson(result.GetResultSet(0))); - } - - { + CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]];[["Value5"];#]])", + FormatResultSetYson(result.GetResultSet(0))); + } + + { auto result = session.ExecuteDataQuery(Q_(R"( - SELECT t1.Value, SUM(t3.Value) + SELECT t1.Value, SUM(t3.Value) FROM `/Root/Join1_1` AS t1 INNER JOIN `/Root/Join1_2` AS t2 - ON t1.Fk21 == t2.Key1 + ON t1.Fk21 == t2.Key1 LEFT JOIN `/Root/Join1_3` AS t3 - ON t2.Fk3 = t3.Key - GROUP BY t1.Value LIMIT 3; + ON t2.Fk3 = t3.Key + GROUP BY t1.Value LIMIT 3; )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]]])", - FormatResultSetYson(result.GetResultSet(0))); - } + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]]])", + FormatResultSetYson(result.GetResultSet(0))); + } } Y_UNIT_TEST_NEW_ENGINE(JoinConvert) { diff --git a/ydb/core/kqp/ut/kqp_limits_ut.cpp b/ydb/core/kqp/ut/kqp_limits_ut.cpp index 43c8f41dbd..0e2478dad7 100644 --- a/ydb/core/kqp/ut/kqp_limits_ut.cpp +++ b/ydb/core/kqp/ut/kqp_limits_ut.cpp @@ -72,7 +72,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR); - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto paramsBuilder = db.GetParamsBuilder(); diff --git a/ydb/core/kqp/ut/kqp_locks_ut.cpp b/ydb/core/kqp/ut/kqp_locks_ut.cpp index 2cfb11c3f9..bd9d9ede5b 100644 --- a/ydb/core/kqp/ut/kqp_locks_ut.cpp +++ b/ydb/core/kqp/ut/kqp_locks_ut.cpp @@ -9,7 +9,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpLocks) { Y_UNIT_TEST_NEW_ENGINE(Invalidate) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session1 = db.CreateSession().GetValueSync().GetSession(); auto session2 = db.CreateSession().GetValueSync().GetSession(); @@ -90,7 +90,7 @@ Y_UNIT_TEST_SUITE(KqpLocks) { Y_UNIT_TEST_NEW_ENGINE(DifferentKeyUpdate) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session1 = db.CreateSession().GetValueSync().GetSession(); auto session2 = db.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp b/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp index 9d15306a18..8ba8607165 100644 --- a/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp +++ b/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp @@ -41,28 +41,28 @@ void CreateSimpleDataTypes(TKikimrRunner& kikimr) { REPLACE INTO `/Root/SimpleDataTypes` (col_bool, col_uint64, col_int32, col_double, col_float, col_string, col_utf8, col_date, col_datetime, col_timestamp, col_interval, col_decimal) VALUES - (NULL, NULL, -1, 1.0, 1.0f, "Value-001", "値-001", + (NULL, NULL, -1, 1.0, 1.0f, "Value-001", "値-001", Date("2021-01-01"), Datetime("2021-01-01T01:01:01Z"), Timestamp("2021-01-01T01:01:01.111111Z"), Interval("P1DT1H1M1.111111S"), Decimal("1.11", 22, 9)), - (false, 2u, NULL, NULL, 2.0f, "Value-002", "値-002", + (false, 2u, NULL, NULL, 2.0f, "Value-002", "値-002", Date("2021-02-02"), Datetime("2021-07-31T02:02:02Z"), Timestamp("2021-02-02T02:02:02.222222Z"), Interval("P2DT2H2M2.222S"), Decimal("2.22", 22, 9)), - (false, 101u, -101, 101.101, NULL, NULL, "値-101", + (false, 101u, -101, 101.101, NULL, NULL, "値-101", Date("2021-02-02"), Datetime("2021-05-31T10:10:10Z"), Timestamp("2021-10-10T10:10:10.101101Z"), Interval("P101DT10H10M10.101101S"), Decimal("101.101", 22, 9)), (true, 102u, -102, 102.102, 102.0f, "Value-102", NULL, Date("2021-12-12"), Datetime("2021-12-12T10:10:10Z"), Timestamp("2021-12-12T10:10:10.102102Z"), Interval("P102DT10H10M10.102102S"), Decimal("102.102", 22, 9)), - (false, 201u, -201, 201.201, 201.201f, "Value-201", "値-201", + (false, 201u, -201, 201.201, 201.201f, "Value-201", "値-201", NULL, NULL, Timestamp("2021-12-21T10:10:10.201201Z"), Interval("P201DT10H10M10.201201S"), Decimal("201.201", 22, 9)), - (true, 202u, -202, 202.202, 202.202f, "Value-202", "値-202", + (true, 202u, -202, 202.202, 202.202f, "Value-202", "値-202", Date("2021-12-22"), Datetime("2021-12-22T10:10:10Z"), NULL, NULL, Decimal("202.202", 22, 9)), - (true, 301u, -301, 301.301, 301.301f, "Value-301", "値-301", + (true, 301u, -301, 301.301, 301.301f, "Value-301", "値-301", Date("2021-05-31"), Datetime("2021-10-10T10:31:31Z"), Timestamp("2021-05-31T10:31:31.301301Z"), Interval("P301DT10H10M10.301301S"), NULL), - (false, 302u, -302, 302.302, 302.302f, "Value-302", "値-302", + (false, 302u, -302, 302.302, 302.302f, "Value-302", "値-302", Date("2021-06-30"), Datetime("2021-05-31T10:32:32Z"), Timestamp("2021-06-30T10:32:32.302302Z"), Interval("P302DT10H10M10.302302S"), Decimal("302.302", 22, 9)) )", TTxControl::BeginTx().CommitTx()).GetValueSync(); @@ -251,7 +251,7 @@ Y_UNIT_TEST_SUITE(KqpMergeCn) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); CompareYson(R"([ [#]; - [["値-001"]]])", StreamResultToYson(result)); + [["値-001"]]])", StreamResultToYson(result)); } Y_UNIT_TEST(TopSortBy_Date_Limit4) { diff --git a/ydb/core/kqp/ut/kqp_newengine_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_ut.cpp index a18266ecd9..3737687d8d 100644 --- a/ydb/core/kqp/ut/kqp_newengine_ut.cpp +++ b/ydb/core/kqp/ut/kqp_newengine_ut.cpp @@ -21,7 +21,7 @@ TKikimrRunner KikimrRunnerWithSessionActor() { Y_UNIT_TEST_SUITE(KqpNewEngine) { void TestSimpleSelect(const TKikimrRunner& kikimr) { - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(MultiSelect) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -333,7 +333,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(InShardsWrite) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -363,7 +363,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(ShuffleWrite) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -391,88 +391,88 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { )", FormatResultSetYson(result.GetResultSet(0))); } - Y_UNIT_TEST(KeyColumnOrder) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - // Please do not change the name of columns here - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("index_0", EPrimitiveType::Utf8) - .AddNullableColumn("value", EPrimitiveType::Uint32) - .SetPrimaryKeyColumns({"key","index_0"}); - - auto result = session.CreateTable("/Root/Test1", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - { - const TString query1(R"( - --!syntax_v1 - PRAGMA Kikimr.UseNewEngine = "true"; - DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; - UPSERT INTO `/Root/Test1` - SELECT * FROM AS_TABLE($items); - )"); - - TParamsBuilder builder; - builder.AddParam("$items").BeginList() - .AddListItem() - .BeginStruct() - .AddMember("key").Uint64(646464646464) - .AddMember("index_0").Utf8("SomeUtf8Data") - .AddMember("value").Uint32(323232) - .EndStruct() - .EndList() - .Build(); - - static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); - auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - { - const auto& yson = ReadTablePartToYson(session, "/Root/Test1"); - const TString expected = R"([[[646464646464u];["SomeUtf8Data"];[323232u]]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - - Y_UNIT_TEST(KeyColumnOrder2) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - { - const TString query1(R"( - --!syntax_v1 - PRAGMA Kikimr.UseNewEngine = "true"; - DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; - SELECT * FROM AS_TABLE($items); - )"); - - TParamsBuilder builder; - builder.AddParam("$items").BeginList() - .AddListItem() - .BeginStruct() - .AddMember("key").Uint64(646464646464) - .AddMember("index_0").Utf8("SomeUtf8Data") - .AddMember("value").Uint32(323232) - .EndStruct() - .EndList() - .Build(); - - static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); - auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - const auto yson = FormatResultSetYson(result.GetResultSet(0)); - const auto expected = R"([["SomeUtf8Data";646464646464u;323232u]])"; - UNIT_ASSERT_VALUES_EQUAL(yson, expected); - } - } - + Y_UNIT_TEST(KeyColumnOrder) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + // Please do not change the name of columns here + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("index_0", EPrimitiveType::Utf8) + .AddNullableColumn("value", EPrimitiveType::Uint32) + .SetPrimaryKeyColumns({"key","index_0"}); + + auto result = session.CreateTable("/Root/Test1", builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + const TString query1(R"( + --!syntax_v1 + PRAGMA Kikimr.UseNewEngine = "true"; + DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; + UPSERT INTO `/Root/Test1` + SELECT * FROM AS_TABLE($items); + )"); + + TParamsBuilder builder; + builder.AddParam("$items").BeginList() + .AddListItem() + .BeginStruct() + .AddMember("key").Uint64(646464646464) + .AddMember("index_0").Utf8("SomeUtf8Data") + .AddMember("value").Uint32(323232) + .EndStruct() + .EndList() + .Build(); + + static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); + auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + { + const auto& yson = ReadTablePartToYson(session, "/Root/Test1"); + const TString expected = R"([[[646464646464u];["SomeUtf8Data"];[323232u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST(KeyColumnOrder2) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + { + const TString query1(R"( + --!syntax_v1 + PRAGMA Kikimr.UseNewEngine = "true"; + DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; + SELECT * FROM AS_TABLE($items); + )"); + + TParamsBuilder builder; + builder.AddParam("$items").BeginList() + .AddListItem() + .BeginStruct() + .AddMember("key").Uint64(646464646464) + .AddMember("index_0").Utf8("SomeUtf8Data") + .AddMember("value").Uint32(323232) + .EndStruct() + .EndList() + .Build(); + + static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); + auto result = session.ExecuteDataQuery(query1, txControl, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + const auto yson = FormatResultSetYson(result.GetResultSet(0)); + const auto expected = R"([["SomeUtf8Data";646464646464u;323232u]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + Y_UNIT_TEST(BlindWrite) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -500,7 +500,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(BlindWriteParameters) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto params = db.GetParamsBuilder() @@ -550,7 +550,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(BlindWriteListParameter) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto params = db.GetParamsBuilder() @@ -600,7 +600,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(BatchUpload) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto queryText = R"( @@ -677,7 +677,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { #if !defined(_ubsan_enabled_) Y_UNIT_TEST(Aggregate) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( PRAGMA kikimr.UseNewEngine = "true"; @@ -693,7 +693,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(AggregateTuple) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( PRAGMA kikimr.UseNewEngine = "true"; @@ -774,7 +774,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(LocksSingleShard) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session1 = db.CreateSession().GetValueSync().GetSession(); auto result = session1.ExecuteDataQuery(R"( @@ -809,7 +809,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(LocksMultiShard) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session1 = db.CreateSession().GetValueSync().GetSession(); auto result = session1.ExecuteDataQuery(R"( @@ -844,7 +844,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(LocksMultiShardOk) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -864,7 +864,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { Y_UNIT_TEST(LocksEffects) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session1 = db.CreateSession().GetValueSync().GetSession(); auto result = session1.ExecuteDataQuery(R"( diff --git a/ydb/core/kqp/ut/kqp_params_ut.cpp b/ydb/core/kqp/ut/kqp_params_ut.cpp index 0b3c020d75..1eee111329 100644 --- a/ydb/core/kqp/ut/kqp_params_ut.cpp +++ b/ydb/core/kqp/ut/kqp_params_ut.cpp @@ -9,7 +9,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpParams) { Y_UNIT_TEST_NEW_ENGINE(RowsList) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto query = session.PrepareDataQuery(Q1_(R"( diff --git a/ydb/core/kqp/ut/kqp_pragma_ut.cpp b/ydb/core/kqp/ut/kqp_pragma_ut.cpp index 00a592b5e3..cdf99e4043 100644 --- a/ydb/core/kqp/ut/kqp_pragma_ut.cpp +++ b/ydb/core/kqp/ut/kqp_pragma_ut.cpp @@ -11,7 +11,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpPragma) { Y_UNIT_TEST(Static) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -25,7 +25,7 @@ Y_UNIT_TEST_SUITE(KqpPragma) { Y_UNIT_TEST_NEW_ENGINE(Runtime) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(Q_(R"( @@ -39,7 +39,7 @@ Y_UNIT_TEST_SUITE(KqpPragma) { Y_UNIT_TEST_NEW_ENGINE(Auth) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(Q_(R"( @@ -53,7 +53,7 @@ Y_UNIT_TEST_SUITE(KqpPragma) { Y_UNIT_TEST(ResetPerQuery) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( diff --git a/ydb/core/kqp/ut/kqp_query_ut.cpp b/ydb/core/kqp/ut/kqp_query_ut.cpp index fec0952e32..5ef6d18f38 100644 --- a/ydb/core/kqp/ut/kqp_query_ut.cpp +++ b/ydb/core/kqp/ut/kqp_query_ut.cpp @@ -20,7 +20,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpQuery) { Y_UNIT_TEST_NEW_ENGINE(PreparedQueryInvalidate) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto prepareResult = session.PrepareDataQuery(Q_(R"( @@ -58,7 +58,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { Y_UNIT_TEST_NEW_ENGINE(QueryCache) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto query = Q_(R"( @@ -217,7 +217,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { Y_UNIT_TEST_NEW_ENGINE(QueryTimeout) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto query = Q_(R"( @@ -276,63 +276,63 @@ Y_UNIT_TEST_SUITE(KqpQuery) { } Y_UNIT_TEST_NEW_ENGINE(QueryClientTimeout) { - TStringStream logStream; + TStringStream logStream; + + { + TKikimrRunner kikimr( + TKikimrSettings() + .SetLogStream(&logStream)); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); - { - TKikimrRunner kikimr( - TKikimrSettings() - .SetLogStream(&logStream)); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); - auto query = Q_(R"( - SELECT * FROM `/Root/TwoShard`; - )"); + auto query = Q_(R"( + SELECT * FROM `/Root/TwoShard`; + )"); - auto txControl = TTxControl::BeginTx().CommitTx(); + auto txControl = TTxControl::BeginTx().CommitTx(); NDataShard::gSkipRepliesFailPoint.Enable(-1, -1, 1); - auto result = session.ExecuteDataQuery( - query, - txControl, - TExecDataQuerySettings() - .UseClientTimeoutForOperation(false) - .ClientTimeout(TDuration::Seconds(3)) - ).ExtractValueSync(); - - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::CLIENT_DEADLINE_EXCEEDED); - + auto result = session.ExecuteDataQuery( + query, + txControl, + TExecDataQuerySettings() + .UseClientTimeoutForOperation(false) + .ClientTimeout(TDuration::Seconds(3)) + ).ExtractValueSync(); + + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::CLIENT_DEADLINE_EXCEEDED); + NDataShard::gSkipRepliesFailPoint.Disable(); - } - - TString line; - int v = 0; - while (logStream.ReadLine(line)) { + } + + TString line; + int v = 0; + while (logStream.ReadLine(line)) { if (line.find("ExecuteDataQuery") == line.npos) continue; - const TString pattern("timeout# "); - auto start = line.find(pattern); - if (start == line.npos) - continue; - // We just want integer part - auto end = line.find('.', start + pattern.size()); - UNIT_ASSERT(end > start + pattern.size()); //something wrong with string - UNIT_ASSERT(end != line.npos); - TString val = line.substr(start + pattern.size(), end - start - pattern.size()); - v = std::stoi(val); - } - - UNIT_ASSERT(v); + const TString pattern("timeout# "); + auto start = line.find(pattern); + if (start == line.npos) + continue; + // We just want integer part + auto end = line.find('.', start + pattern.size()); + UNIT_ASSERT(end > start + pattern.size()); //something wrong with string + UNIT_ASSERT(end != line.npos); + TString val = line.substr(start + pattern.size(), end - start - pattern.size()); + v = std::stoi(val); + } + + UNIT_ASSERT(v); UNIT_ASSERT_C(v <= 3, "got " << v); } Y_UNIT_TEST_NEW_ENGINE(QueryCancel) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto query = Q_(R"( @@ -611,7 +611,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); } - + Y_UNIT_TEST_NEW_ENGINE(YqlTableSample) { auto setting = NKikimrKqp::TKqpSetting(); setting.SetName("_KqpYqlSyntaxVersion"); @@ -893,7 +893,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - + Y_UNIT_TEST_NEW_ENGINE(UnsafeTimestampCastV1) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -1002,15 +1002,15 @@ Y_UNIT_TEST_SUITE(KqpQuery) { UNIT_ASSERT(!phase1.table_access(1).has_updates()); UNIT_ASSERT(!phase1.table_access(1).has_deletes()); } - UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).name(), "/Root/EightShard"); + UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).name(), "/Root/EightShard"); UNIT_ASSERT(!phase1.table_access(0).has_reads()); - UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).updates().rows(), 3); - UNIT_ASSERT(phase1.table_access(0).updates().bytes() > 0); + UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).updates().rows(), 3); + UNIT_ASSERT(phase1.table_access(0).updates().bytes() > 0); UNIT_ASSERT(!phase1.table_access(0).has_deletes()); UNIT_ASSERT_VALUES_EQUAL(stats.total_duration_us(), totalDurationUs); UNIT_ASSERT_VALUES_EQUAL(stats.total_cpu_time_us(), totalCpuTimeUs); - } + } Y_UNIT_TEST_NEW_ENGINE(RowsLimit) { auto setting = NKikimrKqp::TKqpSetting(); @@ -1275,6 +1275,6 @@ Y_UNIT_TEST_SUITE(KqpQuery) { } } - + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index 6295329356..347ca35441 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -273,33 +273,33 @@ Y_UNIT_TEST_SUITE(KqpScan) { [["ydb"];["ydb-1000"];["some very very very very long string"];[0]]])", StreamResultToYson(it)); } - Y_UNIT_TEST(Order) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - - auto params = db.GetParamsBuilder() - .AddParam("$items").BeginList() - .AddListItem() - .BeginStruct() - .AddMember("key").Uint64(646464646464) - .AddMember("index_0").Utf8("SomeUtf8Data") - .AddMember("value").Uint32(323232) - .EndStruct() - .EndList() - .Build().Build(); - - auto it = db.StreamExecuteScanQuery(R"( - DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; - SELECT * FROM AS_TABLE($items); - )", params).GetValueSync(); - - UNIT_ASSERT(it.IsSuccess()); - - CompareYson(R"([ - ["SomeUtf8Data";646464646464u;323232u] - ])", StreamResultToYson(it)); - } - + Y_UNIT_TEST(Order) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + + auto params = db.GetParamsBuilder() + .AddParam("$items").BeginList() + .AddListItem() + .BeginStruct() + .AddMember("key").Uint64(646464646464) + .AddMember("index_0").Utf8("SomeUtf8Data") + .AddMember("value").Uint32(323232) + .EndStruct() + .EndList() + .Build().Build(); + + auto it = db.StreamExecuteScanQuery(R"( + DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>; + SELECT * FROM AS_TABLE($items); + )", params).GetValueSync(); + + UNIT_ASSERT(it.IsSuccess()); + + CompareYson(R"([ + ["SomeUtf8Data";646464646464u;323232u] + ])", StreamResultToYson(it)); + } + Y_UNIT_TEST(GrepRange) { TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); @@ -1675,7 +1675,7 @@ Y_UNIT_TEST_SUITE(KqpScan) { break; } } - UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::BAD_REQUEST, "ScanQuery with explicit index should fail"); + UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::BAD_REQUEST, "ScanQuery with explicit index should fail"); } Y_UNIT_TEST(BoolFlag) { diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp index 4ead3b5f17..ca7ffa42a4 100644 --- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp @@ -14,7 +14,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpScheme) { Y_UNIT_TEST(UseUnauthorizedTable) { TKikimrRunner kikimr("test_user@builtin"); - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto& tableSettings = kikimr.GetTestServer().GetSettings().AppConfig.GetTableServiceConfig(); @@ -37,7 +37,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { Y_UNIT_TEST(UseNonexistentTable) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -54,7 +54,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { Y_UNIT_TEST(UseDroppedTable) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(R"( @@ -76,7 +76,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { Y_UNIT_TEST(CreateDroppedTable) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto schemeResult = session.ExecuteSchemeQuery(R"( @@ -99,98 +99,98 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - Y_UNIT_TEST(CreateDropTableMultipleTime) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - - const size_t inflight = 4; - const size_t limit = 1000; - - const static TString createTableQuery = R"( + Y_UNIT_TEST(CreateDropTableMultipleTime) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + + const size_t inflight = 4; + const size_t limit = 1000; + + const static TString createTableQuery = R"( CREATE TABLE [/Root/Test1234/KeyValue] ( - Key Uint32, - Value String, - PRIMARY KEY(Key) - ); - )"; - - const static TString dropTableQuery = R"( + Key Uint32, + Value String, + PRIMARY KEY(Key) + ); + )"; + + const static TString dropTableQuery = R"( DROP TABLE [/Root/Test1234/KeyValue]; - )"; - - NPar::LocalExecutor().RunAdditionalThreads(inflight); + )"; + + NPar::LocalExecutor().RunAdditionalThreads(inflight); NPar::LocalExecutor().ExecRange([=, &db](int /*id*/) mutable { - size_t i = limit; - while (--i) { - auto session = db.GetSession().GetValueSync().GetSession(); - { - auto result = session.ExecuteSchemeQuery(dropTableQuery) - .ExtractValueSync(); - if (!(result.GetStatus() == EStatus::SUCCESS || - result.GetStatus() == EStatus::SCHEME_ERROR || - result.GetStatus() == EStatus::OVERLOADED)) { - UNIT_ASSERT_C(false, "status: " << result.GetStatus() - << " issues: " << result.GetIssues().ToString()); - } - } - { - auto result = session.ExecuteSchemeQuery(createTableQuery) - .ExtractValueSync(); - if (!(result.GetStatus() == EStatus::SUCCESS || - result.GetStatus() == EStatus::SCHEME_ERROR || - result.GetStatus() == EStatus::OVERLOADED)) { - UNIT_ASSERT_C(false, "status: " << result.GetStatus() - << " issues: " << result.GetIssues().ToString()); - } - } - - } - }, 0, inflight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); - } - - Y_UNIT_TEST(CreateDropTableViaApiMultipleTime) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - - const size_t inflight = 4; - const size_t limit = 1000; + size_t i = limit; + while (--i) { + auto session = db.GetSession().GetValueSync().GetSession(); + { + auto result = session.ExecuteSchemeQuery(dropTableQuery) + .ExtractValueSync(); + if (!(result.GetStatus() == EStatus::SUCCESS || + result.GetStatus() == EStatus::SCHEME_ERROR || + result.GetStatus() == EStatus::OVERLOADED)) { + UNIT_ASSERT_C(false, "status: " << result.GetStatus() + << " issues: " << result.GetIssues().ToString()); + } + } + { + auto result = session.ExecuteSchemeQuery(createTableQuery) + .ExtractValueSync(); + if (!(result.GetStatus() == EStatus::SUCCESS || + result.GetStatus() == EStatus::SCHEME_ERROR || + result.GetStatus() == EStatus::OVERLOADED)) { + UNIT_ASSERT_C(false, "status: " << result.GetStatus() + << " issues: " << result.GetIssues().ToString()); + } + } + + } + }, 0, inflight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); + } + + Y_UNIT_TEST(CreateDropTableViaApiMultipleTime) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + + const size_t inflight = 4; + const size_t limit = 1000; const static TString tableName = "/Root/Test1234/KeyValue"; - - NPar::LocalExecutor().RunAdditionalThreads(inflight); + + NPar::LocalExecutor().RunAdditionalThreads(inflight); NPar::LocalExecutor().ExecRange([=, &db](int /*id*/) mutable { - size_t i = limit; - while (--i) { - auto session = db.GetSession().GetValueSync().GetSession(); - { - auto result = session.DropTable(tableName) - .ExtractValueSync(); - if (!(result.GetStatus() == EStatus::SUCCESS || - result.GetStatus() == EStatus::SCHEME_ERROR || - result.GetStatus() == EStatus::OVERLOADED)) { - UNIT_ASSERT_C(false, "status: " << result.GetStatus() - << " issues: " << result.GetIssues().ToString()); - } - } - { - auto desc = TTableBuilder() - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .AddNullableColumn("Value", EPrimitiveType::String) - .SetPrimaryKeyColumn("Key") - .Build(); - auto result = session.CreateTable(tableName, std::move(desc)) - .ExtractValueSync(); - if (!(result.GetStatus() == EStatus::SUCCESS || - result.GetStatus() == EStatus::SCHEME_ERROR || - result.GetStatus() == EStatus::OVERLOADED)) { - UNIT_ASSERT_C(false, "status: " << result.GetStatus() - << " issues: " << result.GetIssues().ToString()); - } - } - - } - }, 0, inflight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); - } - + size_t i = limit; + while (--i) { + auto session = db.GetSession().GetValueSync().GetSession(); + { + auto result = session.DropTable(tableName) + .ExtractValueSync(); + if (!(result.GetStatus() == EStatus::SUCCESS || + result.GetStatus() == EStatus::SCHEME_ERROR || + result.GetStatus() == EStatus::OVERLOADED)) { + UNIT_ASSERT_C(false, "status: " << result.GetStatus() + << " issues: " << result.GetIssues().ToString()); + } + } + { + auto desc = TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumn("Key") + .Build(); + auto result = session.CreateTable(tableName, std::move(desc)) + .ExtractValueSync(); + if (!(result.GetStatus() == EStatus::SUCCESS || + result.GetStatus() == EStatus::SCHEME_ERROR || + result.GetStatus() == EStatus::OVERLOADED)) { + UNIT_ASSERT_C(false, "status: " << result.GetStatus() + << " issues: " << result.GetIssues().ToString()); + } + } + + } + }, 0, inflight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); + } + Y_UNIT_TEST(QueryWithAlter) { auto kikimr = std::make_shared<TKikimrRunner>(); auto db = kikimr->GetTableClient(); @@ -240,13 +240,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } }, 0, Inflight + 1, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); } - + Y_UNIT_TEST_NEW_ENGINE(SchemaVersionMissmatch) { TKikimrRunner kikimr; - - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + const TString query = Q_(R"( SELECT * FROM [/Root/KeyValue] WHERE Value = "New"; )"); @@ -255,14 +255,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { execSettings.KeepInQueryCache(true); execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - { + { auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), TExecDataQuerySettings().KeepInQueryCache(true)).ExtractValueSync(); - + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } + } - { + { auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -272,46 +272,46 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } { - auto type = TTypeBuilder() - .BeginOptional() - .Primitive(EPrimitiveType::Utf8) - .EndOptional() - .Build(); - + auto type = TTypeBuilder() + .BeginOptional() + .Primitive(EPrimitiveType::Utf8) + .EndOptional() + .Build(); + auto result = session.AlterTable("/Root/KeyValue", TAlterTableSettings() - .AppendAddColumns(TColumn{"NewColumn", type})).ExtractValueSync(); + .AppendAddColumns(TColumn{"NewColumn", type})).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto result = session.ExecuteDataQuery(query, + } + + { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - + auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); - } - } - + } + } + void CheckInvalidationAfterDropCreateTable(bool withCompatSchema) { TKikimrRunner kikimr; - - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); const TString sql = "UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES(1, \"One\")"; - + NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.KeepInQueryCache(true); execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - { + { auto result = session.ExecuteDataQuery(sql, TTxControl::BeginTx().CommitTx(), TExecDataQuerySettings().KeepInQueryCache(true)).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = session.ExecuteDataQuery(sql, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -322,141 +322,141 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto result = session.DropTable("/Root/KeyValue").ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto desc = withCompatSchema - ? TTableBuilder() - .AddNullableColumn("Key", EPrimitiveType::Uint64) - .AddNullableColumn("Value", EPrimitiveType::String) - .AddNullableColumn("Value2", EPrimitiveType::String) - .SetPrimaryKeyColumns({"Key"}) - .Build() - : TTableBuilder() - .AddNullableColumn("Key", EPrimitiveType::Uint64) - .AddNullableColumn("Value", EPrimitiveType::String) - .SetPrimaryKeyColumns({"Key", "Value"}) - .Build(); - + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto desc = withCompatSchema + ? TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .AddNullableColumn("Value2", EPrimitiveType::String) + .SetPrimaryKeyColumns({"Key"}) + .Build() + : TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumns({"Key", "Value"}) + .Build(); + auto result = session.CreateTable("/Root/KeyValue", - std::move(desc)).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + std::move(desc)).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = session.ExecuteDataQuery(sql, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); - + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - + auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); - } - - { - // New session - auto session2 = db.CreateSession().GetValueSync().GetSession(); + } + + { + // New session + auto session2 = db.CreateSession().GetValueSync().GetSession(); auto result = session2.ExecuteDataQuery(sql, TTxControl::BeginTx().CommitTx(), TExecDataQuerySettings().KeepInQueryCache(true)).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - - Y_UNIT_TEST(InvalidationAfterDropCreate) { + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + + Y_UNIT_TEST(InvalidationAfterDropCreate) { CheckInvalidationAfterDropCreateTable(false); - } - - Y_UNIT_TEST(InvalidationAfterDropCreateCompatSchema) { + } + + Y_UNIT_TEST(InvalidationAfterDropCreateCompatSchema) { CheckInvalidationAfterDropCreateTable(true); - } - - void CheckInvalidationAfterDropCreateTable2(bool multistageTx, bool select) { - TKikimrRunner kikimr(TString(), KikimrDefaultUtDomainRoot, 1); - - auto db = kikimr.GetTableClient(); - const TString sql = select + } + + void CheckInvalidationAfterDropCreateTable2(bool multistageTx, bool select) { + TKikimrRunner kikimr(TString(), KikimrDefaultUtDomainRoot, 1); + + auto db = kikimr.GetTableClient(); + const TString sql = select ? "SELECT * FROM [/Root/KeyValue];" : "UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES(1, \"One\")"; - - auto action = [db, sql, multistageTx]() mutable { - return db.RetryOperationSync( - [&](NYdb::NTable::TSession session) -> NYdb::TStatus { - auto prepareResult = session.PrepareDataQuery(sql).GetValueSync(); - - if (!prepareResult.IsSuccess()) { - return prepareResult; - } - auto dataQuery = prepareResult.GetQuery(); - - auto transaction = multistageTx - ? NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()) - : NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); - auto result = dataQuery.Execute(transaction).GetValueSync(); - - if (multistageTx) { - if (!result.IsSuccess()) { - return result; - } - return result.GetTransaction()->Commit().GetValueSync(); - } else { - return result; - } - } - ); - }; - - { - auto result = action(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + + auto action = [db, sql, multistageTx]() mutable { + return db.RetryOperationSync( + [&](NYdb::NTable::TSession session) -> NYdb::TStatus { + auto prepareResult = session.PrepareDataQuery(sql).GetValueSync(); + + if (!prepareResult.IsSuccess()) { + return prepareResult; + } + auto dataQuery = prepareResult.GetQuery(); + + auto transaction = multistageTx + ? NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()) + : NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); + auto result = dataQuery.Execute(transaction).GetValueSync(); + + if (multistageTx) { + if (!result.IsSuccess()) { + return result; + } + return result.GetTransaction()->Commit().GetValueSync(); + } else { + return result; + } + } + ); + }; + + { + auto result = action(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = db.GetSession().GetValueSync().GetSession().DropTable("/Root/KeyValue").ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto desc = TTableBuilder() - .AddNullableColumn("Key", EPrimitiveType::Uint64) - .AddNullableColumn("Value", EPrimitiveType::String) - .AddNullableColumn("Value2", EPrimitiveType::String) - .SetPrimaryKeyColumns({"Key"}) - .Build(); - + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto desc = TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .AddNullableColumn("Value2", EPrimitiveType::String) + .SetPrimaryKeyColumns({"Key"}) + .Build(); + auto result = db.GetSession().GetValueSync().GetSession().CreateTable("/Root/KeyValue", - std::move(desc)).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = action(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - - Y_UNIT_TEST(InvalidationAfterDropCreateTable2) { - CheckInvalidationAfterDropCreateTable2(false, false); - } - - Y_UNIT_TEST(InvalidationAfterDropCreateTable2MultiStageTx) { - CheckInvalidationAfterDropCreateTable2(true, false); - } - - Y_UNIT_TEST(InvalidationAfterDropCreateTable2NoEffects) { - CheckInvalidationAfterDropCreateTable2(false, true); - } - - Y_UNIT_TEST(InvalidationAfterDropCreateTable2MultiStageTxNoEffects) { - CheckInvalidationAfterDropCreateTable2(true, true); - } + std::move(desc)).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = action(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + + Y_UNIT_TEST(InvalidationAfterDropCreateTable2) { + CheckInvalidationAfterDropCreateTable2(false, false); + } + + Y_UNIT_TEST(InvalidationAfterDropCreateTable2MultiStageTx) { + CheckInvalidationAfterDropCreateTable2(true, false); + } + + Y_UNIT_TEST(InvalidationAfterDropCreateTable2NoEffects) { + CheckInvalidationAfterDropCreateTable2(false, true); + } + + Y_UNIT_TEST(InvalidationAfterDropCreateTable2MultiStageTxNoEffects) { + CheckInvalidationAfterDropCreateTable2(true, true); + } Y_UNIT_TEST(CreateTableWithDefaultSettings) { TKikimrRunner kikimr; @@ -1645,24 +1645,24 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR)); } - Y_UNIT_TEST(DropIndexDataColumn) { - auto setting = NKikimrKqp::TKqpSetting(); - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); - - const TString query1(R"( - ALTER TABLE `/Root/SecondaryWithDataColumns` DROP COLUMN Value; - )"); - - auto result = session.ExecuteSchemeQuery( - query1) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); - } - + Y_UNIT_TEST(DropIndexDataColumn) { + auto setting = NKikimrKqp::TKqpSetting(); + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + const TString query1(R"( + ALTER TABLE `/Root/SecondaryWithDataColumns` DROP COLUMN Value; + )"); + + auto result = session.ExecuteSchemeQuery( + query1) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); + } + Y_UNIT_TEST(PathWithNoRoot) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -1742,75 +1742,75 @@ Y_UNIT_TEST_SUITE(KqpScheme) { void AlterTableAddIndex(EIndexTypeSql type, bool enableAsyncIndexes = false) { TKikimrRunner kikimr(TKikimrSettings().SetEnableAsyncIndexes(enableAsyncIndexes)); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); const auto typeStr = IndexTypeSqlString(type); const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::UNSUPPORTED) : EStatus::SUCCESS; - - { + + { auto status = session.ExecuteSchemeQuery(Sprintf(R"( - --!syntax_v1 + --!syntax_v1 ALTER TABLE `/Root/Test` ADD INDEX NameIndex %s ON (Name); )", typeStr.data())).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), expectedStatus, status.GetIssues().ToString()); - } - - { + } + + { TDescribeTableResult describe = session.DescribeTable("/Root/Test").GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); if (expectedStatus != EStatus::SUCCESS) { UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 0); return; } - UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 0); - } - - { - auto status = session.ExecuteSchemeQuery(R"( - --!syntax_v1 + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 0); + } + + { + auto status = session.ExecuteSchemeQuery(R"( + --!syntax_v1 ALTER TABLE `/Root/Test` DROP INDEX NameIndex; - )").ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); - } - - { + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); + } + + { TDescribeTableResult describe = session.DescribeTable("/Root/Test").GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 0); - } - - { + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 0); + } + + { auto status = session.ExecuteSchemeQuery(Sprintf(R"( - --!syntax_v1 + --!syntax_v1 ALTER TABLE `/Root/Test` ADD INDEX NameIndex %s ON (Name) COVER (Amount); )", typeStr.data())).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); - } - - { + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); + } + + { TDescribeTableResult describe = session.DescribeTable("/Root/Test").GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 1); - } - } - + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 1); + } + } + Y_UNIT_TEST(AlterTableAddImplicitSyncIndex) { AlterTableAddIndex(EIndexTypeSql::Global); } diff --git a/ydb/core/kqp/ut/kqp_service_ut.cpp b/ydb/core/kqp/ut/kqp_service_ut.cpp index 36ce0ceedc..37e178c810 100644 --- a/ydb/core/kqp/ut/kqp_service_ut.cpp +++ b/ydb/core/kqp/ut/kqp_service_ut.cpp @@ -19,8 +19,8 @@ Y_UNIT_TEST_SUITE(KqpService) { NPar::LocalExecutor().RunAdditionalThreads(Inflight); auto driverConfig = kikimr->GetDriverConfig(); - NYdb::TDriver driver(driverConfig); - NPar::LocalExecutor().ExecRange([driver](int id) { + NYdb::TDriver driver(driverConfig); + NPar::LocalExecutor().ExecRange([driver](int id) { NYdb::NTable::TTableClient db(driver); auto sessionResult = db.CreateSession().GetValueSync(); @@ -62,7 +62,7 @@ Y_UNIT_TEST_SUITE(KqpService) { Sleep(WaitDuration); kikimr.Reset(); Sleep(WaitDuration); - driver.Stop(true); + driver.Stop(true); } Y_UNIT_TEST(CloseSessionsWithLoad) { @@ -128,19 +128,19 @@ Y_UNIT_TEST_SUITE(KqpService) { }, 0, SessionsCount + 1, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); } - TVector<TAsyncDataQueryResult> simulateSessionBusy(ui32 count, TSession& session) { - TVector<TAsyncDataQueryResult> futures; - for (ui32 i = 0; i < count; ++i) { - auto query = Sprintf(R"( + TVector<TAsyncDataQueryResult> simulateSessionBusy(ui32 count, TSession& session) { + TVector<TAsyncDataQueryResult> futures; + for (ui32 i = 0; i < count; ++i) { + auto query = Sprintf(R"( SELECT * FROM [/Root/EightShard] WHERE Key=%1$d; - )", i); - - auto future = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()); - futures.push_back(future); - } - return futures; - } - + )", i); + + auto future = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()); + futures.push_back(future); + } + return futures; + } + Y_UNIT_TEST(SessionBusy) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetUseSessionBusyStatus(true); @@ -149,7 +149,7 @@ Y_UNIT_TEST_SUITE(KqpService) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto futures = simulateSessionBusy(10, session); + auto futures = simulateSessionBusy(10, session); NThreading::WaitExceptionOrAll(futures).GetValueSync(); @@ -160,71 +160,71 @@ Y_UNIT_TEST_SUITE(KqpService) { } } } - - Y_UNIT_TEST(SessionBusyRetryOperation) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetUseSessionBusyStatus(true); - - TKikimrRunner kikimr(appConfig); - auto db = kikimr.GetTableClient(); - - ui32 queriesCount = 10; - ui32 busyResultCount = 0; - auto status = db.RetryOperation([&queriesCount, &busyResultCount](TSession session) { - UNIT_ASSERT(queriesCount); - UNIT_ASSERT(session.GetId()); - - auto futures = simulateSessionBusy(queriesCount, session); - + + Y_UNIT_TEST(SessionBusyRetryOperation) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetUseSessionBusyStatus(true); + + TKikimrRunner kikimr(appConfig); + auto db = kikimr.GetTableClient(); + + ui32 queriesCount = 10; + ui32 busyResultCount = 0; + auto status = db.RetryOperation([&queriesCount, &busyResultCount](TSession session) { + UNIT_ASSERT(queriesCount); + UNIT_ASSERT(session.GetId()); + + auto futures = simulateSessionBusy(queriesCount, session); + NThreading::WaitExceptionOrAll(futures).GetValueSync(); - - for (auto& future : futures) { - auto result = future.GetValue(); - if (!result.IsSuccess()) { - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SESSION_BUSY, result.GetIssues().ToString()); - queriesCount--; - busyResultCount++; - return NThreading::MakeFuture<TStatus>(result); - } - } - return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - }).GetValueSync(); - // Result should be SUCCESS in case of SESSION_BUSY - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); - } - - Y_UNIT_TEST(SessionBusyRetryOperationSync) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetUseSessionBusyStatus(true); - - TKikimrRunner kikimr(appConfig); - auto db = kikimr.GetTableClient(); - - ui32 queriesCount = 10; - ui32 busyResultCount = 0; - auto status = db.RetryOperationSync([&queriesCount, &busyResultCount](TSession session) { - UNIT_ASSERT(queriesCount); - UNIT_ASSERT(session.GetId()); - - auto futures = simulateSessionBusy(queriesCount, session); - + + for (auto& future : futures) { + auto result = future.GetValue(); + if (!result.IsSuccess()) { + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SESSION_BUSY, result.GetIssues().ToString()); + queriesCount--; + busyResultCount++; + return NThreading::MakeFuture<TStatus>(result); + } + } + return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + }).GetValueSync(); + // Result should be SUCCESS in case of SESSION_BUSY + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); + } + + Y_UNIT_TEST(SessionBusyRetryOperationSync) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetUseSessionBusyStatus(true); + + TKikimrRunner kikimr(appConfig); + auto db = kikimr.GetTableClient(); + + ui32 queriesCount = 10; + ui32 busyResultCount = 0; + auto status = db.RetryOperationSync([&queriesCount, &busyResultCount](TSession session) { + UNIT_ASSERT(queriesCount); + UNIT_ASSERT(session.GetId()); + + auto futures = simulateSessionBusy(queriesCount, session); + NThreading::WaitExceptionOrAll(futures).GetValueSync(); - - for (auto& future : futures) { - auto result = future.GetValue(); - if (!result.IsSuccess()) { - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SESSION_BUSY, result.GetIssues().ToString()); - queriesCount--; - busyResultCount++; - return (TStatus)result; - } - } - return TStatus(EStatus::SUCCESS, NYql::TIssues()); - }); - // Result should be SUCCESS in case of SESSION_BUSY - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); - } - + + for (auto& future : futures) { + auto result = future.GetValue(); + if (!result.IsSuccess()) { + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SESSION_BUSY, result.GetIssues().ToString()); + queriesCount--; + busyResultCount++; + return (TStatus)result; + } + } + return TStatus(EStatus::SUCCESS, NYql::TIssues()); + }); + // Result should be SUCCESS in case of SESSION_BUSY + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); + } + } } // namspace NKqp diff --git a/ydb/core/kqp/ut/kqp_sort_ut.cpp b/ydb/core/kqp/ut/kqp_sort_ut.cpp index a85bfae1d0..0b7c65a3d5 100644 --- a/ydb/core/kqp/ut/kqp_sort_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sort_ut.cpp @@ -60,7 +60,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - UNIT_ASSERT_C(!result.GetAst().Contains("Sort"), result.GetAst()); + UNIT_ASSERT_C(!result.GetAst().Contains("Sort"), result.GetAst()); } { @@ -75,24 +75,24 @@ Y_UNIT_TEST_SUITE(KqpSort) { } Y_UNIT_TEST_NEW_ENGINE(ReverseOptimizedWithPredicate) { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_AllowReverseRange"); - setting.SetValue("True"); - - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_AllowReverseRange"); + setting.SetValue("True"); + + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString query = Q_(R"( - SELECT Group, Name, Amount, Comment + SELECT Group, Name, Amount, Comment FROM `/Root/Test` - WHERE Group < 2u - ORDER BY Group DESC, Name DESC; + WHERE Group < 2u + ORDER BY Group DESC, Name DESC; )"); - + auto result = session.ExplainDataQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - + if (UseNewEngine) { UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst()); @@ -104,19 +104,19 @@ Y_UNIT_TEST_SUITE(KqpSort) { UNIT_ASSERT(read.IsDefined()); UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { - UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); - } - - { - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - CompareYson(R"([ - [[1u];["Paul"];[300u];["None"]]; - [[1u];["Anna"];[3500u];["None"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - + UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); + } + + { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + CompareYson(R"([ + [[1u];["Paul"];[300u];["None"]]; + [[1u];["Anna"];[3500u];["None"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + Y_UNIT_TEST_NEW_ENGINE(ReverseFirstKeyOptimized) { auto setting = NKikimrKqp::TKqpSetting(); setting.SetName("_AllowReverseRange"); @@ -568,218 +568,218 @@ Y_UNIT_TEST_SUITE(KqpSort) { } } - Y_UNIT_TEST_NEW_ENGINE(ComplexPkExclusiveSecondOptionalPredicate) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - TString query = Q_(R"( - DECLARE $x AS "Uint32"; - DECLARE $y AS "String?"; - - SELECT * + Y_UNIT_TEST_NEW_ENGINE(ComplexPkExclusiveSecondOptionalPredicate) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + TString query = Q_(R"( + DECLARE $x AS "Uint32"; + DECLARE $y AS "String?"; + + SELECT * FROM `/Root/Join2` - WHERE Key1 = $x AND Key2 > $y - ORDER BY Key1, Key2 - LIMIT 10; - )"); - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .OptionalString("One") - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - [[105u];["Two"];["Name4"];["Value28"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .EmptyOptional(EPrimitiveType::String) - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - - { - TString query = Q_(R"( - DECLARE $x AS "Uint32"; - DECLARE $y AS "String?"; - - SELECT * + WHERE Key1 = $x AND Key2 > $y + ORDER BY Key1, Key2 + LIMIT 10; + )"); + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .OptionalString("One") + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + [[105u];["Two"];["Name4"];["Value28"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .EmptyOptional(EPrimitiveType::String) + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + { + TString query = Q_(R"( + DECLARE $x AS "Uint32"; + DECLARE $y AS "String?"; + + SELECT * FROM `/Root/Join2` - WHERE Key1 = $x AND Key2 < $y - ORDER BY Key1, Key2 - LIMIT 10; - )"); - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .OptionalString("Two") - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - [[105u];["One"];["Name2"];["Value27"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .EmptyOptional(EPrimitiveType::String) - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - } - - Y_UNIT_TEST_NEW_ENGINE(ComplexPkInclusiveSecondOptionalPredicate) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - TString query = Q_(R"( - DECLARE $x AS "Uint32"; - DECLARE $y AS "String?"; - - SELECT * + WHERE Key1 = $x AND Key2 < $y + ORDER BY Key1, Key2 + LIMIT 10; + )"); + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .OptionalString("Two") + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + [[105u];["One"];["Name2"];["Value27"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .EmptyOptional(EPrimitiveType::String) + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + } + + Y_UNIT_TEST_NEW_ENGINE(ComplexPkInclusiveSecondOptionalPredicate) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + TString query = Q_(R"( + DECLARE $x AS "Uint32"; + DECLARE $y AS "String?"; + + SELECT * FROM `/Root/Join2` - WHERE Key1 = $x AND Key2 >= $y - ORDER BY Key1, Key2 - LIMIT 10; - )"); - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .OptionalString("One") - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - [[105u];["One"];["Name2"];["Value27"]];[[105u];["Two"];["Name4"];["Value28"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .EmptyOptional(EPrimitiveType::String) - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - - { - TString query = Q_(R"( - DECLARE $x AS "Uint32"; - DECLARE $y AS "String?"; - - SELECT * + WHERE Key1 = $x AND Key2 >= $y + ORDER BY Key1, Key2 + LIMIT 10; + )"); + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .OptionalString("One") + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + [[105u];["One"];["Name2"];["Value27"]];[[105u];["Two"];["Name4"];["Value28"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .EmptyOptional(EPrimitiveType::String) + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + { + TString query = Q_(R"( + DECLARE $x AS "Uint32"; + DECLARE $y AS "String?"; + + SELECT * FROM `/Root/Join2` - WHERE Key1 = $x AND Key2 <= $y - ORDER BY Key1, Key2 - LIMIT 10; - )"); - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .OptionalString("Two") - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - [[105u];["One"];["Name2"];["Value27"]];[[105u];["Two"];["Name4"];["Value28"]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } - - { - auto params = db.GetParamsBuilder() - .AddParam("$x") - .Uint32(105) - .Build() - .AddParam("$y") - .EmptyOptional(EPrimitiveType::String) - .Build() - - .Build(); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([ - ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - } - + WHERE Key1 = $x AND Key2 <= $y + ORDER BY Key1, Key2 + LIMIT 10; + )"); + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .OptionalString("Two") + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + [[105u];["One"];["Name2"];["Value27"]];[[105u];["Two"];["Name4"];["Value28"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto params = db.GetParamsBuilder() + .AddParam("$x") + .Uint32(105) + .Build() + .AddParam("$y") + .EmptyOptional(EPrimitiveType::String) + .Build() + + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([ + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + } + Y_UNIT_TEST_NEW_ENGINE(TopSortTableExpr) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -1082,7 +1082,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { } } - // https://st.yandex-team.ru/KIKIMR-11523 + // https://st.yandex-team.ru/KIKIMR-11523 Y_UNIT_TEST_NEW_ENGINE(PassLimit) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_sqlin_ut.cpp b/ydb/core/kqp/ut/kqp_sqlin_ut.cpp index 3357605adc..b41df7c39e 100644 --- a/ydb/core/kqp/ut/kqp_sqlin_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sqlin_ut.cpp @@ -29,7 +29,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { TString query = TStringBuilder() @@ -69,7 +69,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() << "--!syntax_v1" << Endl @@ -108,7 +108,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() @@ -161,7 +161,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() << "--!syntax_v1" << Endl @@ -187,7 +187,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() @@ -240,7 +240,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() << "--!syntax_v1" << Endl @@ -270,7 +270,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() << "--!syntax_v1" << Endl @@ -298,7 +298,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() << "--!syntax_v1" << Endl @@ -378,7 +378,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto params = TParamsBuilder() .AddParam("$keys").BeginList() .AddListItem().Utf8("1") - .AddListItem().Utf8("¿cómo estás?") + .AddListItem().Utf8("¿cómo estás?") .EndList().Build() .Build(); @@ -391,7 +391,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() @@ -429,7 +429,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() @@ -471,7 +471,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() @@ -506,11 +506,11 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() - << Q1_(Sprintf(R"( + << Q1_(Sprintf(R"( PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s"; -- not depends on `Kikimr.OptDisableSqlInToJoin` pragma DECLARE $in AS List<Struct<k: Uint32, v: String>>; @@ -548,7 +548,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); const TString query = Q1_(R"( DECLARE $in AS List<Int32?>; @@ -577,20 +577,20 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { AssertTableReads(result, "/Root/SecondaryKeys", 2); } - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey) { + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( + const TString query = Q1_(Sprintf(R"( %s DECLARE $in AS List<Int32?>; SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IN $in ORDER BY Value - )", FormatPragma(disableOpt))); + )", FormatPragma(disableOpt))); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().OptionalInt32(1) @@ -604,205 +604,205 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { }; test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryKeys", 5); + }); + + test(EnableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 2); + AssertTableReads(result, "/Root/SecondaryKeys", 2); + }); + } + + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey_In_And) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTablesWithIndex(session); + + auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { + const TString query = Q1_(Sprintf(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + %s + DECLARE $in AS List<Int32?>; + SELECT * FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IN $in + AND Value == "Payload1" + )", FormatPragma(disableOpt))); + + auto params = TParamsBuilder().AddParam("$in").BeginList() + .AddListItem().OptionalInt32(1) + .AddListItem().OptionalInt32(2) + .AddListItem().OptionalInt32(42) + .AddListItem().OptionalInt32(Nothing()) + .EndList().Build().Build(); + + auto result = ExecQueryAndTestResult(session, query, params, R"([[[1];[1];["Payload1"]]])"); + assertFn(result); + }; + + test(DisableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 5); AssertTableReads(result, "/Root/SecondaryKeys", 5); + UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), + result.GetIssues().ToString()); }); test(EnableOpt, [](const TDataQueryResult& result) { AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 2); AssertTableReads(result, "/Root/SecondaryKeys", 2); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + }); + } + + Y_UNIT_TEST_NEW_ENGINE(SimpleKey_In_And_In) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTablesWithIndex(session); + + auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { + const TString query = Q1_(Sprintf(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + %s + DECLARE $in1 AS List<Int32>; + SELECT Key, FROM `/Root/SecondaryComplexKeys` + WHERE Key IN $in1 AND Value IN ("active", "Payload2"); + )", FormatPragma(disableOpt))); + + auto params = TParamsBuilder() + .AddParam("$in1") + .BeginList() + .AddListItem().Int32(1) + .AddListItem().Int32(2) + .AddListItem().Int32(42) + .EndList().Build() + .Build(); + + auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); + assertFn(result); + }; + + test(DisableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); + }); + + test(EnableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + }); + } + + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey_In_And_In) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTablesWithIndex(session); + + auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { + const TString query = Q1_(Sprintf(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + %s + DECLARE $in1 AS List<Int32>; + SELECT Key, FROM `/Root/SecondaryComplexKeys` VIEW Index + WHERE Fk1 IN $in1 AND Value IN ("active", "Payload2"); + )", FormatPragma(disableOpt))); + + auto params = TParamsBuilder() + .AddParam("$in1") + .BeginList() + .AddListItem().Int32(1) + .AddListItem().Int32(2) + .AddListItem().Int32(42) + .EndList().Build() + .Build(); + + auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); + assertFn(result); + }; + + test(DisableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); + UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), + result.GetIssues().ToString()); + }); + + test(EnableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2); + AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); + }); + } + + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_ComplexKey_In_And_In) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTablesWithIndex(session); + + auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { + const TString query = Q1_(Sprintf(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + %s + DECLARE $in1 AS List<Int32>; + DECLARE $in2 AS List<String>; + SELECT Key, FROM `/Root/SecondaryComplexKeys` VIEW Index + WHERE Fk1 IN $in1 AND Fk2 IN $in2 AND Value IN ("active", "Payload2"); + )", FormatPragma(disableOpt))); + + auto params = TParamsBuilder() + .AddParam("$in1") + .BeginList() + .AddListItem().Int32(1) + .AddListItem().Int32(2) + .AddListItem().Int32(42) + .EndList().Build() + .AddParam("$in2") + .BeginList() + .AddListItem().String("Fk2") + .AddListItem().String("qq") + .EndList().Build() + + .Build(); + + auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); + assertFn(result); + }; + + test(DisableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); + UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), + result.GetIssues().ToString()); + }); + + test(EnableOpt, [](const TDataQueryResult& result) { + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2); + AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); + UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); }); } - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey_In_And) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTablesWithIndex(session); - - auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - %s - DECLARE $in AS List<Int32?>; - SELECT * FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IN $in - AND Value == "Payload1" - )", FormatPragma(disableOpt))); - - auto params = TParamsBuilder().AddParam("$in").BeginList() - .AddListItem().OptionalInt32(1) - .AddListItem().OptionalInt32(2) - .AddListItem().OptionalInt32(42) - .AddListItem().OptionalInt32(Nothing()) - .EndList().Build().Build(); - - auto result = ExecQueryAndTestResult(session, query, params, R"([[[1];[1];["Payload1"]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 5); - AssertTableReads(result, "/Root/SecondaryKeys", 5); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), - result.GetIssues().ToString()); - }); - - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 2); - AssertTableReads(result, "/Root/SecondaryKeys", 2); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - }); - } - - Y_UNIT_TEST_NEW_ENGINE(SimpleKey_In_And_In) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTablesWithIndex(session); - - auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - %s - DECLARE $in1 AS List<Int32>; - SELECT Key, FROM `/Root/SecondaryComplexKeys` - WHERE Key IN $in1 AND Value IN ("active", "Payload2"); - )", FormatPragma(disableOpt))); - - auto params = TParamsBuilder() - .AddParam("$in1") - .BeginList() - .AddListItem().Int32(1) - .AddListItem().Int32(2) - .AddListItem().Int32(42) - .EndList().Build() - .Build(); - - auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); - }); - - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - }); - } - - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey_In_And_In) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTablesWithIndex(session); - - auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - %s - DECLARE $in1 AS List<Int32>; - SELECT Key, FROM `/Root/SecondaryComplexKeys` VIEW Index - WHERE Fk1 IN $in1 AND Value IN ("active", "Payload2"); - )", FormatPragma(disableOpt))); - - auto params = TParamsBuilder() - .AddParam("$in1") - .BeginList() - .AddListItem().Int32(1) - .AddListItem().Int32(2) - .AddListItem().Int32(42) - .EndList().Build() - .Build(); - - auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), - result.GetIssues().ToString()); - }); - - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - }); - } - - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_ComplexKey_In_And_In) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTablesWithIndex(session); - - auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - %s - DECLARE $in1 AS List<Int32>; - DECLARE $in2 AS List<String>; - SELECT Key, FROM `/Root/SecondaryComplexKeys` VIEW Index - WHERE Fk1 IN $in1 AND Fk2 IN $in2 AND Value IN ("active", "Payload2"); - )", FormatPragma(disableOpt))); - - auto params = TParamsBuilder() - .AddParam("$in1") - .BeginList() - .AddListItem().Int32(1) - .AddListItem().Int32(2) - .AddListItem().Int32(42) - .EndList().Build() - .AddParam("$in2") - .BeginList() - .AddListItem().String("Fk2") - .AddListItem().String("qq") - .EndList().Build() - - .Build(); - - auto result = ExecQueryAndTestResult(session, query, params, R"([[[2]]])"); - assertFn(result); - }; - - test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); - UNIT_ASSERT_C(result.GetIssues().ToString().Contains("Given predicate is not suitable"), - result.GetIssues().ToString()); - }); - - test(EnableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2); - AssertTableReads(result, "/Root/SecondaryComplexKeys", 2); - UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString()); - }); - } - - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleParameter) { + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleParameter) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( + const TString query = Q1_(Sprintf(R"( %s DECLARE $in AS List<Tuple<Int32?, String>>; SELECT Value FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) IN $in ORDER BY Value - )", FormatPragma(disableOpt))); + )", FormatPragma(disableOpt))); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().BeginTuple().AddElement().OptionalInt32(1).AddElement().String("Fk1").EndTuple() @@ -816,7 +816,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { }; test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); }); @@ -826,28 +826,28 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { }); } - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleLiteral) { + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleLiteral) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( + const TString query = Q1_(Sprintf(R"( %s SELECT Value FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull")) ORDER BY Value - )", FormatPragma(disableOpt))); + )", FormatPragma(disableOpt))); auto result = ExecQueryAndTestResult(session, query, R"([[["Payload1"]];[["Payload2"]]])"); assertFn(result); }; test(DisableOpt, [](const TDataQueryResult& result) { - AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); + AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5); AssertTableReads(result, "/Root/SecondaryComplexKeys", 5); }); @@ -857,15 +857,15 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { }); } - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleSelect) { + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleSelect) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { - const TString query = Q1_(Sprintf(R"( + const TString query = Q1_(Sprintf(R"( PRAGMA Kikimr.OptDisableSqlInToJoin = "True"; PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s"; DECLARE $in AS List<Struct<k: Int32?, v: String>>; @@ -873,7 +873,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) IN (SELECT (k, v) FROM AS_TABLE($in)) ORDER BY Value - )", disableOpt ? "True" : "False")); + )", disableOpt ? "True" : "False")); auto params = TParamsBuilder().AddParam("$in").BeginList() .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct() @@ -902,11 +902,11 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) { auto query = TStringBuilder() - << Q1_(Sprintf(R"( + << Q1_(Sprintf(R"( %s DECLARE $in AS List<Tuple<String, Uint32>>; SELECT Group AS g, Name as n, Amount AS a @@ -952,10 +952,10 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTablesWithIndex(session); + CreateSampleTablesWithIndex(session); auto query = TStringBuilder() - << Q1_(R"( + << Q1_(R"( DECLARE $in AS List<Uint64>; DELETE FROM `/Root/KeyValue` WHERE Key IN $in )"); diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp index a40fc33a07..f7815c016e 100644 --- a/ydb/core/kqp/ut/kqp_stats_ut.cpp +++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp @@ -4,8 +4,8 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> -#include <cstdlib> - +#include <cstdlib> + namespace NKikimr { namespace NKqp { @@ -174,11 +174,11 @@ Y_UNIT_TEST(DeferredEffects) { NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); UNIT_ASSERT_VALUES_EQUAL(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 4); - - auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); - UNIT_ASSERT(ru != result.GetResponseMetadata().end()); - - UNIT_ASSERT(std::atoi(ru->second.c_str()) > 1); + + auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); + UNIT_ASSERT(ru != result.GetResponseMetadata().end()); + + UNIT_ASSERT(std::atoi(ru->second.c_str()) > 1); } Y_UNIT_TEST(DataQueryWithEffects) { @@ -248,58 +248,58 @@ Y_UNIT_TEST(DataQueryMulti) { } Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteDataQuery(Q_(R"( - INCORRECT_STMT + INCORRECT_STMT )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), TExecDataQuerySettings().ReportCostInfo(true)) - .ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - - auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); - UNIT_ASSERT(ru != result.GetResponseMetadata().end()); - UNIT_ASSERT_VALUES_EQUAL(ru->second, "1"); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - UNIT_ASSERT(result.GetConsumedRu() > 0); -} - + .ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + + auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); + UNIT_ASSERT(ru != result.GetResponseMetadata().end()); + UNIT_ASSERT_VALUES_EQUAL(ru->second, "1"); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT(result.GetConsumedRu() > 0); +} + Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExplicitPrepare) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.PrepareDataQuery(Q_(R"( - INCORRECT_STMT + INCORRECT_STMT )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - - auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); - UNIT_ASSERT(ru != result.GetResponseMetadata().end()); - UNIT_ASSERT_VALUES_EQUAL(ru->second, "1"); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - UNIT_ASSERT(result.GetConsumedRu() > 0); -} - + result.GetIssues().PrintTo(Cerr); + + auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); + UNIT_ASSERT(ru != result.GetResponseMetadata().end()); + UNIT_ASSERT_VALUES_EQUAL(ru->second, "1"); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT(result.GetConsumedRu() > 0); +} + Y_UNIT_TEST_NEW_ENGINE(RequestUnitForSuccessExplicitPrepare) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.PrepareDataQuery(Q_(R"( - SELECT 0; SELECT 1; SELECT 2; SELECT 3; SELECT 4; - SELECT 5; SELECT 6; SELECT 7; SELECT 8; SELECT 9; + SELECT 0; SELECT 1; SELECT 2; SELECT 3; SELECT 4; + SELECT 5; SELECT 6; SELECT 7; SELECT 8; SELECT 9; )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - - auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); - UNIT_ASSERT(ru != result.GetResponseMetadata().end()); - UNIT_ASSERT(atoi(ru->second.c_str()) > 1); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(result.GetConsumedRu() > 1); -} - + result.GetIssues().PrintTo(Cerr); + + auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); + UNIT_ASSERT(ru != result.GetResponseMetadata().end()); + UNIT_ASSERT(atoi(ru->second.c_str()) > 1); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(result.GetConsumedRu() > 1); +} + } // suite } // namespace NKqp diff --git a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp index 2926c2b1ff..1f8988517c 100644 --- a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp @@ -81,12 +81,12 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { [[72057594046644480u];[8u];["/Root/BatchUpload"];[5u]]; [[72057594046644480u];[9u];["/Root/BatchUpload"];[5u]]; [[72057594046644480u];[0u];["/Root/KeyValue"];[6u]]; - [[72057594046644480u];[0u];["/Root/KeyValue2"];[7u]]; - [[72057594046644480u];[0u];["/Root/Test"];[8u]]; - [[72057594046644480u];[0u];["/Root/Join1"];[9u]]; - [[72057594046644480u];[1u];["/Root/Join1"];[9u]]; - [[72057594046644480u];[0u];["/Root/Join2"];[10u]]; - [[72057594046644480u];[1u];["/Root/Join2"];[10u]] + [[72057594046644480u];[0u];["/Root/KeyValue2"];[7u]]; + [[72057594046644480u];[0u];["/Root/Test"];[8u]]; + [[72057594046644480u];[0u];["/Root/Join1"];[9u]]; + [[72057594046644480u];[1u];["/Root/Join1"];[9u]]; + [[72057594046644480u];[0u];["/Root/Join2"];[10u]]; + [[72057594046644480u];[1u];["/Root/Join2"];[10u]] ])", StreamResultToYson(it)); } @@ -101,16 +101,16 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { TString query = R"( SELECT OwnerId, PathId, PartIdx, Path FROM `/Root/.sys/partition_stats` - WHERE OwnerId = 72057594046644480ul AND PathId > 5u AND PathId <= 9u + WHERE OwnerId = 72057594046644480ul AND PathId > 5u AND PathId <= 9u ORDER BY PathId, PartIdx; )"; TString expectedYson = R"([ [[72057594046644480u];[6u];[0u];["/Root/KeyValue"]]; - [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; - [[72057594046644480u];[8u];[0u];["/Root/Test"]]; - [[72057594046644480u];[9u];[0u];["/Root/Join1"]]; - [[72057594046644480u];[9u];[1u];["/Root/Join1"]] + [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; + [[72057594046644480u];[8u];[0u];["/Root/Test"]]; + [[72057594046644480u];[9u];[0u];["/Root/Join1"]]; + [[72057594046644480u];[9u];[1u];["/Root/Join1"]] ])"; auto it = client.StreamExecuteScanQuery(query).GetValueSync(); @@ -132,14 +132,14 @@ Y_UNIT_TEST_SUITE(KqpSystemView) { TString query = R"( SELECT OwnerId, PathId, PartIdx, Path FROM `/Root/.sys/partition_stats` - WHERE OwnerId = 72057594046644480ul AND PathId >= 6u AND PathId < 9u + WHERE OwnerId = 72057594046644480ul AND PathId >= 6u AND PathId < 9u ORDER BY PathId, PartIdx; )"; TString expectedYson = R"([ [[72057594046644480u];[6u];[0u];["/Root/KeyValue"]]; - [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; - [[72057594046644480u];[8u];[0u];["/Root/Test"]] + [[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]]; + [[72057594046644480u];[8u];[0u];["/Root/Test"]] ])"; auto it = client.StreamExecuteScanQuery(query).GetValueSync(); diff --git a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp index b1090fad12..5ab5108968 100644 --- a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp +++ b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp @@ -6,9 +6,9 @@ #include <ydb/library/yql/ast/yql_ast.h> #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/yql_expr_optimize.h> - + #include <library/cpp/json/json_reader.h> - + namespace NKikimr { namespace NKqp { @@ -42,7 +42,7 @@ static void CreateSampleTables(TSession session) { (3u, 400u, "Thirteen"), (3u, 500u, "Fourteen"); )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync().IsSuccess()); - + UNIT_ASSERT(session.ExecuteSchemeQuery(R"( CREATE TABLE [/Root/TestDate] ( Key Date, @@ -64,7 +64,7 @@ static void CreateSampleTables(TSession session) { .AddNullableColumn("Value", EPrimitiveType::Utf8) .AddNullableColumn("ValueInt", EPrimitiveType::Int32) .SetPrimaryKeyColumn("Key"); - + UNIT_ASSERT(session.CreateTable("/Root/MultiShardTable", builder.Build(), TCreateTableSettings() @@ -73,7 +73,7 @@ static void CreateSampleTables(TSession session) { .UniformPartitions(5) ) ).GetValueSync().IsSuccess()); - + UNIT_ASSERT(session.ExecuteDataQuery(R"( REPLACE INTO [/Root/MultiShardTable] (Key, Value) VALUES (1, "One"), @@ -82,20 +82,20 @@ static void CreateSampleTables(TSession session) { (4, "Four"), (4294967295, "LastOne"); )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync().IsSuccess()); - + UNIT_ASSERT(session.ExecuteDataQuery(R"( REPLACE INTO [/Root/MultiShardTable] (Key, ValueInt) VALUES (10, 10); )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync().IsSuccess()); } - + { auto builder = TTableBuilder() .AddNullableColumn("Key1", EPrimitiveType::Uint32) .AddNullableColumn("Key2", EPrimitiveType::String) .AddNullableColumn("ValueInt", EPrimitiveType::Int32) .SetPrimaryKeyColumns({"Key1", "Key2"}); - + UNIT_ASSERT(session.CreateTable("/Root/MultiShardTableCk", builder.Build(), TCreateTableSettings() @@ -345,7 +345,7 @@ void RunPredicateTest(const std::vector<TString>& predicates, bool withNulls) { Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(IsNull) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -361,7 +361,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(IsNullPartial) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -378,7 +378,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(NullInKey) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -395,7 +395,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(NullInKeySuffix) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -413,7 +413,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(NullInPredicate) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -438,7 +438,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST_NEW_ENGINE(NullInPredicateRow) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -463,7 +463,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { Y_UNIT_TEST(AllowNullCompareInIndex) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); @@ -511,12 +511,12 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0))); } - + Y_UNIT_TEST_NEW_ENGINE(UpdateMulti) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + { auto query = Q_(R"(UPDATE `/Root/KeyValue` SET Value = Value || "_updated" WHERE Key IN (1, 2, 3, 4))"); @@ -540,87 +540,87 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateSampleTables(session); - + CreateSampleTables(session); + const TString query = Q_("UPDATE [/Root/MultiShardTable] SET Value = 'aaaaa' WHERE Key IN (1, 500)"); - + if (!UseNewEngine) { - auto result = session.ExplainDataQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - auto astRes = NYql::ParseAst(result.GetAst()); - UNIT_ASSERT(astRes.IsOk()); - NYql::TExprContext exprCtx; - NYql::TExprNode::TPtr exprRoot; - UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); - - UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); - UNIT_ASSERT(!NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { - return NYql::NNodes::TMaybeNode<NYql::NNodes::TKiSelectRange>(node).IsValid(); - })); - } - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(result.IsSuccess()); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + auto result = session.ExplainDataQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + auto astRes = NYql::ParseAst(result.GetAst()); + UNIT_ASSERT(astRes.IsOk()); + NYql::TExprContext exprCtx; + NYql::TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); + + UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); + UNIT_ASSERT(!NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { + return NYql::NNodes::TMaybeNode<NYql::NNodes::TKiSelectRange>(node).IsValid(); + })); + } + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT(result.IsSuccess()); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/MultiShardTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1); - + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/MultiShardTable"); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1); - } - } - + } + } + Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInWithNull) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE); - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE); - - CreateSampleTables(session); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE); + + CreateSampleTables(session); + const TString query = Q_(R"( UPDATE [/Root/MultiShardTable] SET ValueInt = ValueInt + 1 WHERE Key IN (1,2,3,4,NULL))"); - { - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync(); - Cerr << result.GetIssues().ToString() << Endl; - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync(); + Cerr << result.GetIssues().ToString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT Key, ValueInt FROM [/Root/MultiShardTable] ORDER BY Key; - + )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[10]];[[4294967295u];#]])", - FormatResultSetYson(result.GetResultSet(0))); - } - } - + UNIT_ASSERT(result.IsSuccess()); + + CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[10]];[[4294967295u];#]])", + FormatResultSetYson(result.GetResultSet(0))); + } + } + Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralList) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTables(session); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTables(session); + auto result = session.ExecuteDataQuery(Q_(R"( UPDATE `/Root/MultiShardTable` SET ValueInt = ValueInt + 1 WHERE Key IN (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, @@ -630,7 +630,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { )"), TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - { + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT Key, ValueInt FROM `/Root/MultiShardTable` ORDER BY Key )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); @@ -638,9 +638,9 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])", FormatResultSetYson(result.GetResultSet(0))); - } + } } - + Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralListPrefix) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -658,52 +658,52 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - { + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT Key1, ValueInt FROM `/Root/MultiShardTableCk` ORDER BY Key1; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])", - FormatResultSetYson(result.GetResultSet(0))); - } - } - + + CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])", + FormatResultSetYson(result.GetResultSet(0))); + } + } + Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInMultipleUpdate) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE); - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE); - - CreateSampleTables(session); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE); + + CreateSampleTables(session); + const TString query = Q_(R"( UPDATE `/Root/MultiShardTable` SET ValueInt = ValueInt + 1 WHERE Key IN (1,2,10); UPDATE `/Root/TestNulls` SET Value = 'qq' WHERE Key1 IN (1,2); )"); - { - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync(); + { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { + } + + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT Key, ValueInt FROM `/Root/MultiShardTable` ORDER BY Key; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])", - FormatResultSetYson(result.GetResultSet(0))); - } + UNIT_ASSERT(result.IsSuccess()); - { + CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])", + FormatResultSetYson(result.GetResultSet(0))); + } + + { auto result = session.ExecuteDataQuery(Q_(R"( SELECT Key1, Value FROM `/Root/TestNulls` ORDER BY Key1; )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - + UNIT_ASSERT(result.IsSuccess()); + CompareYson(R"([ [#;["One"]]; [#;["Two"]]; @@ -720,75 +720,75 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { [[3u];["Thirteen"]]; [[3u];["Fourteen"]] ])", FormatResultSetYson(result.GetResultSet(0))); - } - } - - Y_UNIT_TEST(SelectFullScanUnionAll) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTables(session); - + } + } + + Y_UNIT_TEST(SelectFullScanUnionAll) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTables(session); + const TString query = "PRAGMA Kikimr.UseNewEngine = 'false'; " "SELECT Key, ValueInt FROM [/Root/MultiShardTable] WHERE ValueInt = 1 " "UNION ALL SELECT Key, ValueInt FROM [/Root/MultiShardTable] WHERE ValueInt = 1;"; - - auto result = session.ExplainDataQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - auto astRes = NYql::ParseAst(result.GetAst()); - UNIT_ASSERT(astRes.IsOk()); - NYql::TExprContext exprCtx; - NYql::TExprNode::TPtr exprRoot; - UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); - - UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); - auto extend = NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { - return NYql::NNodes::TMaybeNode<NYql::NNodes::TCoExtend>(node).IsValid(); - }); - UNIT_ASSERT(extend); - - for (const auto& item : NYql::NNodes::TExprBase(extend).Maybe<NYql::NNodes::TCoExtend>().Cast()) { - UNIT_ASSERT(item.Maybe<NYql::NNodes::TCoFilter>()); - auto filter = item.Maybe<NYql::NNodes::TCoFilter>().Cast(); - UNIT_ASSERT(filter.Input().Maybe<NYql::NNodes::TKiSelectRange>()); - } - } - + + auto result = session.ExplainDataQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + auto astRes = NYql::ParseAst(result.GetAst()); + UNIT_ASSERT(astRes.IsOk()); + NYql::TExprContext exprCtx; + NYql::TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); + + UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); + auto extend = NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { + return NYql::NNodes::TMaybeNode<NYql::NNodes::TCoExtend>(node).IsValid(); + }); + UNIT_ASSERT(extend); + + for (const auto& item : NYql::NNodes::TExprBase(extend).Maybe<NYql::NNodes::TCoExtend>().Cast()) { + UNIT_ASSERT(item.Maybe<NYql::NNodes::TCoFilter>()); + auto filter = item.Maybe<NYql::NNodes::TCoFilter>().Cast(); + UNIT_ASSERT(filter.Input().Maybe<NYql::NNodes::TKiSelectRange>()); + } + } + Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInFullScan) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTables(session); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTables(session); + const TString query = Q_("UPDATE `/Root/MultiShardTable` SET Value = 'aaaaa' WHERE Value IN ('One', 'www')"); if (!UseNewEngine) { - auto result = session.ExplainDataQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - auto astRes = NYql::ParseAst(result.GetAst()); - UNIT_ASSERT(astRes.IsOk()); - NYql::TExprContext exprCtx; - NYql::TExprNode::TPtr exprRoot; - UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); - - UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); - UNIT_ASSERT(NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { - return NYql::NNodes::TMaybeNode<NYql::NNodes::TKiSelectRange>(node).IsValid(); - })); - } - - { - NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(result.IsSuccess()); - - auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + auto result = session.ExplainDataQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + auto astRes = NYql::ParseAst(result.GetAst()); + UNIT_ASSERT(astRes.IsOk()); + NYql::TExprContext exprCtx; + NYql::TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); + + UNIT_ASSERT(NYql::NNodes::TMaybeNode<NYql::NNodes::TKiProgram>(exprRoot)); + UNIT_ASSERT(NYql::FindNode(exprRoot, [](const NYql::TExprNode::TPtr& node) { + return NYql::NNodes::TMaybeNode<NYql::NNodes::TKiSelectRange>(node).IsValid(); + })); + } + + { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT(result.IsSuccess()); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 5); @@ -806,8 +806,8 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { } else { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 5); } - } - } + } + } Y_UNIT_TEST_NEW_ENGINE(DateKeyPredicate) { TKikimrRunner kikimr; diff --git a/ydb/core/kqp/ut/kqp_tx_ut.cpp b/ydb/core/kqp/ut/kqp_tx_ut.cpp index ff3147a3e2..5837331604 100644 --- a/ydb/core/kqp/ut/kqp_tx_ut.cpp +++ b/ydb/core/kqp/ut/kqp_tx_ut.cpp @@ -48,7 +48,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(ExplicitTcl) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto tx = session.BeginTransaction(TTxSettings::SerializableRW()) @@ -119,7 +119,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST(BeginTransactionBadMode) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.BeginTransaction(TTxSettings::OnlineRO()).ExtractValueSync(); @@ -131,7 +131,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(CommitRequired) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(Q_(R"( @@ -147,7 +147,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(RollbackTx) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); // with effects, without locks @@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(EmptyTxOnCommit) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(Q_(R"( @@ -294,7 +294,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { setting.SetValue("2"); TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.BeginTransaction(TTxSettings::SerializableRW()).ExtractValueSync(); @@ -323,7 +323,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { settings.push_back(setting); TKikimrRunner kikimr(settings); - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.BeginTransaction(TTxSettings::SerializableRW()).ExtractValueSync(); @@ -347,7 +347,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(RollbackInvalidated) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto result = session.ExecuteDataQuery(Q_(R"( @@ -372,7 +372,7 @@ Y_UNIT_TEST_SUITE(KqpTx) { Y_UNIT_TEST_NEW_ENGINE(CommitPrepared) { TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); auto query = session.PrepareDataQuery(Q_(R"( diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make index 0e988992ea..609ec3237f 100644 --- a/ydb/core/kqp/ut/ya.make +++ b/ydb/core/kqp/ut/ya.make @@ -26,8 +26,8 @@ SRCS( kqp_explain_ut.cpp kqp_flip_join_ut.cpp kqp_index_lookup_join_ut.cpp - kqp_indexes_ut.cpp - kqp_indexes_multishard_ut.cpp + kqp_indexes_ut.cpp + kqp_indexes_multishard_ut.cpp kqp_join_ut.cpp kqp_limits_ut.cpp kqp_locks_ut.cpp @@ -70,7 +70,7 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_proto ) -YQL_LAST_ABI_VERSION() +YQL_LAST_ABI_VERSION() REQUIREMENTS(ram:12) diff --git a/ydb/core/mind/dynamic_nameserver.cpp b/ydb/core/mind/dynamic_nameserver.cpp index 4fc7e4556d..6f4f147982 100644 --- a/ydb/core/mind/dynamic_nameserver.cpp +++ b/ydb/core/mind/dynamic_nameserver.cpp @@ -33,7 +33,7 @@ void TDynamicNodeResolverBase::Bootstrap(const TActorContext &ctx) ui32 group = dinfo->GetDefaultStateStorageGroup(domain); auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeNodeBrokerID(group), NTabletPipe::TClientConfig(retryPolicy)); - NodeBrokerPipe = ctx.RegisterWithSameMailbox(pipe); + NodeBrokerPipe = ctx.RegisterWithSameMailbox(pipe); TAutoPtr<TEvNodeBroker::TEvResolveNode> request = new TEvNodeBroker::TEvResolveNode; request->Record.SetNodeId(NodeId); @@ -157,7 +157,7 @@ void TDynamicNameserver::OpenPipe(ui32 domain, if (!NodeBrokerPipes[domain]) { auto pipe = NTabletPipe::CreateClient(ctx.SelfID, MakeNodeBrokerID(group)); - NodeBrokerPipes[domain] = ctx.RegisterWithSameMailbox(pipe); + NodeBrokerPipes[domain] = ctx.RegisterWithSameMailbox(pipe); } } @@ -184,7 +184,7 @@ void TDynamicNameserver::ResolveStaticNode(ui32 nodeId, TActorId sender, TInstan return; } - RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, sender, SelfId(), deadline)); + RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, sender, SelfId(), deadline)); } void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId, @@ -198,14 +198,14 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId, if (it != DynamicConfigs[domain]->DynamicNodes.end() && it->second.Expire > ctx.Now()) { - RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, ev->Sender, SelfId(), deadline)); + RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, ev->Sender, SelfId(), deadline)); } else if (DynamicConfigs[domain]->ExpiredNodes.contains(nodeId) && ctx.Now() < DynamicConfigs[domain]->Epoch.End) { auto reply = new TEvLocalNodeInfo; reply->NodeId = nodeId; ctx.Send(ev->Sender, reply); } else { - ctx.RegisterWithSameMailbox(new TDynamicNodeResolver(SelfId(), nodeId, DynamicConfigs[domain], ev, deadline)); + ctx.RegisterWithSameMailbox(new TDynamicNodeResolver(SelfId(), nodeId, DynamicConfigs[domain], ev, deadline)); } } @@ -330,7 +330,7 @@ void TDynamicNameserver::Handle(TEvResolveAddress::TPtr &ev, const TActorContext const TEvResolveAddress* request = ev->Get(); - RegisterWithSameMailbox(CreateResolveActor(request->Address, request->Port, ev->Sender, SelfId(), TInstant::Max())); + RegisterWithSameMailbox(CreateResolveActor(request->Address, request->Port, ev->Sender, SelfId(), TInstant::Max())); } void TDynamicNameserver::Handle(TEvInterconnect::TEvListNodes::TPtr &ev, @@ -379,7 +379,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc ctx.Send(ev->Sender, reply.Release()); } else { const TInstant deadline = ev->Get()->Deadline; - ctx.RegisterWithSameMailbox(new TDynamicNodeSearcher(SelfId(), nodeId, DynamicConfigs[domain], ev.Release(), + ctx.RegisterWithSameMailbox(new TDynamicNodeSearcher(SelfId(), nodeId, DynamicConfigs[domain], ev.Release(), deadline)); } } diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index c444f7ade0..58ca2ac3e4 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -845,7 +845,7 @@ void THive::OnActivateExecutor(const TActorContext&) { Execute(CreateInitScheme()); if (!ResponsivenessPinger) { ResponsivenessPinger = new TTabletResponsivenessPinger(TabletCounters->Simple()[NHive::COUNTER_RESPONSE_TIME_USEC], TDuration::Seconds(1)); - ResponsivenessActorID = RegisterWithSameMailbox(ResponsivenessPinger); + ResponsivenessActorID = RegisterWithSameMailbox(ResponsivenessPinger); } } diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index ef76147f3a..7e9a83b354 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -2190,7 +2190,7 @@ public: TReassignTabletWaitActor* waitActor = nullptr; if (Wait) { waitActor = new TReassignTabletWaitActor(Source, Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); } for (TLeaderTabletInfo* tablet : tablets) { @@ -2306,7 +2306,7 @@ public: TInitMigrationWaitActor* waitActor = nullptr; if (Wait) { waitActor = new TInitMigrationWaitActor(Source, Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); } // TODO: pass arguments as post data json @@ -2380,7 +2380,7 @@ public: TActorId waitActorId; TQueryMigrationWaitActor* waitActor = nullptr; waitActor = new TQueryMigrationWaitActor(Source, Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); ctx.Send(new IEventHandle(Self->SelfId(), waitActorId, new TEvHive::TEvQueryMigration())); return true; @@ -2453,7 +2453,7 @@ public: TStopTabletWaitActor* waitActor = nullptr; if (Wait) { waitActor = new TStopTabletWaitActor(Source, Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); } Self->Execute(Self->CreateStopTablet(TabletId, waitActorId)); @@ -2532,7 +2532,7 @@ public: TResumeTabletWaitActor* waitActor = nullptr; if (Wait) { waitActor = new TResumeTabletWaitActor(Source, Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); } Self->Execute(Self->CreateResumeTablet(TabletId, waitActorId)); @@ -3400,10 +3400,10 @@ void THive::CreateEvMonitoring(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorCo if (cgi.Has("owner") && cgi.Has("owner_idx")) { ui64 owner = FromStringWithDefault<ui64>(cgi.Get("owner"), 0); ui64 ownerIdx = FromStringWithDefault<ui64>(cgi.Get("owner_idx"), 0); - ctx.RegisterWithSameMailbox(new TDeleteTabletActor(ev->Sender, owner, ownerIdx, this)); + ctx.RegisterWithSameMailbox(new TDeleteTabletActor(ev->Sender, owner, ownerIdx, this)); } else if (cgi.Has("tablet")) { TTabletId tabletId = FromStringWithDefault<TTabletId>(cgi.Get("tablet"), 0); - ctx.RegisterWithSameMailbox(new TDeleteTabletActor(ev->Sender, tabletId, this)); + ctx.RegisterWithSameMailbox(new TDeleteTabletActor(ev->Sender, tabletId, this)); } else { ctx.Send(ev->Sender, new NMon::TEvRemoteJsonInfoRes("{\"Error\": \"tablet or (owner, owner_idx) params must be specified\"}")); } diff --git a/ydb/core/mind/hive/tx__release_tablets_reply.cpp b/ydb/core/mind/hive/tx__release_tablets_reply.cpp index eec6a62d08..8589258f32 100644 --- a/ydb/core/mind/hive/tx__release_tablets_reply.cpp +++ b/ydb/core/mind/hive/tx__release_tablets_reply.cpp @@ -79,7 +79,7 @@ public: TReleaseTabletsWaitActor* waitActor = nullptr; if (Self->MigrationFilter.GetWaitForTabletsToRise()) { waitActor = new TReleaseTabletsWaitActor(Self); - waitActorId = ctx.RegisterWithSameMailbox(waitActor); + waitActorId = ctx.RegisterWithSameMailbox(waitActor); Self->SubActors.emplace_back(waitActor); } diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index b908a77e45..09fc15f76d 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -180,7 +180,7 @@ class TLocalNodeRegistrar : public TActorBootstrapped<TLocalNodeRegistrar> { .BackoffMultiplier = 2, .DoFirstRetryInstantly = true }; - HivePipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, HiveId, pipeConfig)); + HivePipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, HiveId, pipeConfig)); THolder<TEvLocal::TEvRegisterNode> request = MakeHolder<TEvLocal::TEvRegisterNode>(HiveId); for (auto &domain: ServicedDomains) { @@ -974,7 +974,7 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { void OpenPipe(const TActorContext &ctx) { ClosePipe(ctx); - SchemeShardPipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, SchemeRoot, PipeConfig)); + SchemeShardPipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, SchemeRoot, PipeConfig)); } void ClosePipe(const TActorContext &ctx) diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp index 2f8c3af741..a2ef938369 100644 --- a/ydb/core/mind/node_broker.cpp +++ b/ydb/core/mind/node_broker.cpp @@ -898,7 +898,7 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev, CFunc(TEvents::TSystem::Undelivered, HandleUndelivered) }) }; - ctx.RegisterWithSameMailbox(new TRegisterNodeActor(ev, this)); + ctx.RegisterWithSameMailbox(new TRegisterNodeActor(ev, this)); } void TNodeBroker::Handle(TEvNodeBroker::TEvExtendLeaseRequest::TPtr &ev, diff --git a/ydb/core/mind/tenant_pool.cpp b/ydb/core/mind/tenant_pool.cpp index f6c37e9791..80236570b9 100644 --- a/ydb/core/mind/tenant_pool.cpp +++ b/ydb/core/mind/tenant_pool.cpp @@ -911,7 +911,7 @@ public: for (auto &pr : domainConfigs) { auto *domain = domains->GetDomainByName(pr.first); Y_VERIFY(domain, "unknown domain %s in Tenant Pool config", pr.first.data()); - auto aid = ctx.RegisterWithSameMailbox(new TDomainTenantPool(pr.first, LocalID, pr.second)); + auto aid = ctx.RegisterWithSameMailbox(new TDomainTenantPool(pr.first, LocalID, pr.second)); DomainTenantPools[pr.first] = aid; auto serviceId = MakeTenantPoolID(SelfId().NodeId(), domain->DomainUid); ctx.ExecutorThread.ActorSystem->RegisterLocalService(serviceId, aid); diff --git a/ydb/core/persqueue/codecs/pqv1.cpp b/ydb/core/persqueue/codecs/pqv1.cpp index d1b3085b91..8037b0ac03 100644 --- a/ydb/core/persqueue/codecs/pqv1.cpp +++ b/ydb/core/persqueue/codecs/pqv1.cpp @@ -1,35 +1,35 @@ -#include "pqv1.h" - -namespace NKikimr::NPQ { - -Ydb::PersQueue::V1::Codec ToV1Codec(const NPersQueueCommon::ECodec codec) { - switch (codec) { - case NPersQueueCommon::RAW: - return Ydb::PersQueue::V1::CODEC_RAW; - case NPersQueueCommon::GZIP: - return Ydb::PersQueue::V1::CODEC_GZIP; - case NPersQueueCommon::LZOP: - return Ydb::PersQueue::V1::CODEC_LZOP; - case NPersQueueCommon::ZSTD: - return Ydb::PersQueue::V1::CODEC_ZSTD; - default: - return Ydb::PersQueue::V1::CODEC_UNSPECIFIED; - } -} - -std::optional<NPersQueueCommon::ECodec> FromV1Codec(const NYdb::NPersQueue::ECodec codec) { - switch (codec) { - case NYdb::NPersQueue::ECodec::RAW: - return NPersQueueCommon::RAW; - case NYdb::NPersQueue::ECodec::GZIP: - return NPersQueueCommon::GZIP; - case NYdb::NPersQueue::ECodec::LZOP: - return NPersQueueCommon::LZOP; - case NYdb::NPersQueue::ECodec::ZSTD: - return NPersQueueCommon::ZSTD; - default: - return std::nullopt; - } -} - -} // NKikimr::NPQ +#include "pqv1.h" + +namespace NKikimr::NPQ { + +Ydb::PersQueue::V1::Codec ToV1Codec(const NPersQueueCommon::ECodec codec) { + switch (codec) { + case NPersQueueCommon::RAW: + return Ydb::PersQueue::V1::CODEC_RAW; + case NPersQueueCommon::GZIP: + return Ydb::PersQueue::V1::CODEC_GZIP; + case NPersQueueCommon::LZOP: + return Ydb::PersQueue::V1::CODEC_LZOP; + case NPersQueueCommon::ZSTD: + return Ydb::PersQueue::V1::CODEC_ZSTD; + default: + return Ydb::PersQueue::V1::CODEC_UNSPECIFIED; + } +} + +std::optional<NPersQueueCommon::ECodec> FromV1Codec(const NYdb::NPersQueue::ECodec codec) { + switch (codec) { + case NYdb::NPersQueue::ECodec::RAW: + return NPersQueueCommon::RAW; + case NYdb::NPersQueue::ECodec::GZIP: + return NPersQueueCommon::GZIP; + case NYdb::NPersQueue::ECodec::LZOP: + return NPersQueueCommon::LZOP; + case NYdb::NPersQueue::ECodec::ZSTD: + return NPersQueueCommon::ZSTD; + default: + return std::nullopt; + } +} + +} // NKikimr::NPQ diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp index 27035d3298..4eb9c7206d 100644 --- a/ydb/core/persqueue/read_balancer.cpp +++ b/ydb/core/persqueue/read_balancer.cpp @@ -620,7 +620,7 @@ void TPersQueueReadBalancer::RequestTabletIfNeeded(const ui64 tabletId, const TA TActorId pipeClient; if (it == TabletPipes.end()) { NTabletPipe::TClientConfig clientConfig; - pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); + pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); TabletPipes[tabletId] = pipeClient; } else { pipeClient = it->second; diff --git a/ydb/core/persqueue/type_codecs_defs.cpp b/ydb/core/persqueue/type_codecs_defs.cpp index bbf12cfef3..f333be5cfc 100644 --- a/ydb/core/persqueue/type_codecs_defs.cpp +++ b/ydb/core/persqueue/type_codecs_defs.cpp @@ -111,7 +111,7 @@ TTypeCodecs::TTypeCodecs(TTypeId typeId) { case NTypeIds::String4k: case NTypeIds::String2m: case NTypeIds::Utf8: - case NTypeIds::Json: + case NTypeIds::Json: case NTypeIds::JsonDocument: case NTypeIds::DyNumber: InitDefaults(this, TCodecType::VarLen); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 47d7584602..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -566,11 +566,11 @@ message TGRpcConfig { optional uint64 MaxMessageSize = 6; // default = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT optional uint32 MaxInFlight = 7; // 0 == unlimited [default] optional NKikimrStream.TStreamingConfig StreamingConfig = 8; - // Ssl part - optional uint32 SslPort = 9; - optional string CA = 10; - optional string Cert = 11; - optional string Key = 12; + // Ssl part + optional uint32 SslPort = 9; + optional string CA = 10; + optional string Cert = 11; + optional string Key = 12; // public host/port for publishing optional string PublicHost = 13; @@ -622,7 +622,7 @@ message TFeatureFlags { optional bool EnableChunkGraceForPDisk = 7 [default = true]; optional bool AllowConsistentOperationsForSchemeShard = 8 [default = true]; optional bool EnableSchemeBoard = 9 [default = true]; // deprecated: always true - optional bool AllowYdbRequestsWithoutDatabase = 10 [default = true]; + optional bool AllowYdbRequestsWithoutDatabase = 10 [default = true]; optional bool EnableExternalSubdomains = 11 [default = true]; optional bool AllowRecursiveMkDir = 12 [default = true]; // deprecated: always true optional bool AllowHugeKeyValueDeletes = 13 [default = true]; // delete when all clients limit deletes per request @@ -631,11 +631,11 @@ message TFeatureFlags { optional bool EnableSystemViews = 16 [default = true]; optional bool EnableExternalHive = 17 [default = true]; optional bool UseSchemeBoardCacheForSchemeRequests = 18 [default = true]; // deprecated: always true - optional bool CompileMinikqlWithVersion = 19 [default = true]; // deprecated: always true + optional bool CompileMinikqlWithVersion = 19 [default = true]; // deprecated: always true optional bool ReadTableWithSnapshot = 20 [default = true]; optional bool ImportantTabletsUseSystemPool = 21 [default = true]; optional bool EnableOfflineSlaves = 22 [default = true]; // deprecated: always true - optional bool CheckDatabaseAccessPermission = 23 [default = false]; + optional bool CheckDatabaseAccessPermission = 23 [default = false]; optional bool AllowOnlineIndexBuild = 24 [default = true]; // deprecated: always true optional bool EnablePersistentQueryStats = 25 [default = false]; optional bool DisableDataShardBarrier = 26 [default = false]; @@ -646,13 +646,13 @@ message TFeatureFlags { optional bool EnableTtlOnIndexedTables = 31 [default = true]; // deprecated: always true optional bool AllowUpdateChannelsBindingOfSolomonPartitions = 32 [default = false]; optional bool DisableLegacyYql = 33 [default = true]; // deprecated: always true - optional bool EnableDataColumnForIndexTable = 34 [default = true]; + optional bool EnableDataColumnForIndexTable = 34 [default = true]; optional bool AllowServerlessStorageBillingForSchemeShard = 35 [default = false]; optional bool EnableGracefulShutdown = 36 [default = true]; optional bool EnableDrainOnShutdown = 37 [default = true]; optional bool EnableConfigurationCache = 38 [default = false]; optional bool EnableDbCounters = 39 [default = false]; - optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false]; + optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false]; optional bool EnableAsyncIndexes = 41 [default = true]; optional bool AllowStreamExecuteYqlScript = 42 [default = true]; optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true diff --git a/ydb/core/protos/counters_datashard.proto b/ydb/core/protos/counters_datashard.proto index 1529f3f72d..956a136937 100644 --- a/ydb/core/protos/counters_datashard.proto +++ b/ydb/core/protos/counters_datashard.proto @@ -386,7 +386,7 @@ enum ETxTypes { TXTYPE_STORE_TABLE_PATH = 37 [(TxTypeOpts) = {Name: "TTxStoreTablePath"}]; TXTYPE_READ_COLUMNS = 38 [(TxTypeOpts) = {Name: "TTxReadColumns"}]; TXTYPE_PROPOSE = 39 [(TxTypeOpts) = {Name: "TxPropose"}]; - TXTYPE_STORE_SCAN_STATE = 40 [(TxTypeOpts) = {Name: "TxStoreScanState"}]; + TXTYPE_STORE_SCAN_STATE = 40 [(TxTypeOpts) = {Name: "TxStoreScanState"}]; TXTYPE_INIT_SCHEMA_DEFAULTS = 41 [(TxTypeOpts) = {Name: "TxInitSchemaDefaults"}]; TXTYPE_REFRESH_VOLATILE_SNAPSHOT = 42 [(TxTypeOpts) = {Name: "TxRefreshVolatileSnapshot"}]; TXTYPE_DISCARD_VOLATILE_SNAPSHOT = 43 [(TxTypeOpts) = {Name: "TxDiscardVolatileSnapshot"}]; diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index d72f1e589c..a839915c98 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -72,7 +72,7 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxMergeTablePartition = 58 [(CounterOpts) = {Name: "InFlightOps/MergeTablePartition"}]; COUNTER_IN_FLIGHT_OPS_TxAlterExtSubDomain = 59 [(CounterOpts) = {Name: "InFlightOps/AlterExtSubDomain"}]; COUNTER_IN_FLIGHT_OPS_TxForceDropExtSubDomain = 60 [(CounterOpts) = {Name: "InFlightOps/ForceDropExtSubDomain"}]; - COUNTER_IN_FLIGHT_OPS_TxFillIndex = 61 [(CounterOpts) = {Name: "InFlightOps/FillIndex"}]; + COUNTER_IN_FLIGHT_OPS_TxFillIndex = 61 [(CounterOpts) = {Name: "InFlightOps/FillIndex"}]; COUNTER_IN_FLIGHT_OPS_TxUpgradeSubDomain = 62 [(CounterOpts) = {Name: "InFlightOps/UpgradeSubDomain"}]; COUNTER_IN_FLIGHT_OPS_TxUpgradeSubDomainDecision = 63 [(CounterOpts) = {Name: "InFlightOps/UpgradeSubDomainDecision"}]; COUNTER_SUB_DOMAIN_HIVE_COUNT = 64 [(CounterOpts) = {Name: "SubDomainsHives"}]; @@ -189,7 +189,7 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxMergeTablePartition = 33 [(CounterOpts) = {Name: "FinishedOps/MergeTablePartition"}]; COUNTER_FINISHED_OPS_TxAlterExtSubDomain = 34 [(CounterOpts) = {Name: "FinishedOps/AlterExtSubDomain"}]; COUNTER_FINISHED_OPS_TxForceDropExtSubDomain = 35 [(CounterOpts) = {Name: "FinishedOps/ForceDropExtSubDomain"}]; - COUNTER_FINISHED_OPS_TxFillIndex = 36 [(CounterOpts) = {Name: "FinishedOps/FillIndex"}]; + COUNTER_FINISHED_OPS_TxFillIndex = 36 [(CounterOpts) = {Name: "FinishedOps/FillIndex"}]; COUNTER_FINISHED_OPS_TxUpgradeSubDomain = 37 [(CounterOpts) = {Name: "FinishedOps/UpgradeSubDomain"}]; COUNTER_FINISHED_OPS_TxUpgradeSubDomainDecision = 38 [(CounterOpts) = {Name: "FinishedOps/UpgradeSubDomainDecision"}]; COUNTER_FINISHED_OPS_TxInitializeBuildIndex = 39 [(CounterOpts) = {Name: "FinishedOps/InitializeBuildIndex"}]; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 7fa52a0123..33e598c1c2 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -58,12 +58,12 @@ enum EColumnStorage { ColumnStorageTest_1_2_1k = 999; } -enum EFreezeState { - Unspecified = 0; - Freeze = 1; - Unfreeze = 2; -} - +enum EFreezeState { + Unspecified = 0; + Freeze = 1; + Unfreeze = 2; +} + enum EMvccState { MvccUnspecified = 0; MvccEnabled = 1; @@ -268,7 +268,7 @@ message TPartitionConfig { optional bool EnableEraseCache = 18 [default = true]; // Use erase cache for faster iteration over erased rows optional uint32 EraseCacheMinRows = 19; // Minimum number of erased rows worth caching (default 16) optional uint32 EraseCacheMaxBytes = 20; // Maximum number of bytes to use for cached rows (default 1MB) - optional EFreezeState FreezeState = 21; + optional EFreezeState FreezeState = 21; optional bool ShadowData = 22; // Table has separate shadow data for later merging repeated NKikimrHive.TFollowerGroup FollowerGroups = 23; reserved 24; // EMvccState MvccState = 24; no longer used @@ -342,8 +342,8 @@ message TTableDescription { repeated TSplitBoundary SplitBoundary = 31; // Boundaries for non-uniform split repeated TIndexDescription TableIndexes = 32; - - optional uint64 TableSchemaVersion = 33; + + optional uint64 TableSchemaVersion = 33; optional NKikimrProto.TPathID PathId = 34; // TPathId, owerride Id = 2 @@ -749,24 +749,24 @@ enum EIndexType { enum EIndexState { EIndexStateInvalid = 0; EIndexStateReady = 1; - EIndexStateNotReady = 2; - EIndexStateWriteOnly = 3; + EIndexStateNotReady = 2; + EIndexStateWriteOnly = 3; } message TIndexDescription { optional string Name = 1; - optional uint64 LocalPathId = 2; + optional uint64 LocalPathId = 2; optional EIndexType Type = 3; optional EIndexState State = 4; repeated string KeyColumnNames = 5; - - optional uint64 SchemaVersion = 6; // Schema version of abstract index (not impl table) - - optional uint64 PathOwnerId = 7; - - repeated string DataColumnNames = 8; + + optional uint64 SchemaVersion = 6; // Schema version of abstract index (not impl table) + + optional uint64 PathOwnerId = 7; + + repeated string DataColumnNames = 8; // DataSize + IndexSize of indexImplTable optional uint64 DataSize = 9; } @@ -775,9 +775,9 @@ message TIndexCreationConfig { optional string Name = 1; repeated string KeyColumnNames = 2; optional EIndexType Type = 3; - optional TTableDescription IndexImplTableDescription = 4; //description for index impl table - optional EIndexState State = 5; //state of index at the creation time - repeated string DataColumnNames = 6; //columns to be denormalized to read data just from index + optional TTableDescription IndexImplTableDescription = 4; //description for index impl table + optional EIndexState State = 5; //state of index at the creation time + repeated string DataColumnNames = 6; //columns to be denormalized to read data just from index } message TIndexAlteringConfig { @@ -1318,7 +1318,7 @@ enum EPathType { EPathTypeBlockStoreVolume = 6; EPathTypeKesus = 7; EPathTypeSolomonVolume = 8; - EPathTypeTableIndex = 9; // Used for abstract index + EPathTypeTableIndex = 9; // Used for abstract index EPathTypeExtSubDomain = 10; EPathTypeFileStore = 11; EPathTypeColumnStore = 12; @@ -1349,22 +1349,22 @@ enum EPathState { EPathStateMoving = 11; } -message TPathVersion { - optional uint64 GeneralVersion = 1; - - optional uint64 ACLVersion = 2; - optional uint64 EffectiveACLVersion = 3; - optional uint64 UserAttrsVersion = 4; - optional uint64 ChildrenVersion = 5; - optional uint64 SubDomainVersion = 6; - optional uint64 TableSchemaVersion = 7; - optional uint64 TablePartitionVersion = 8; - optional uint64 TableIndexVersion = 9; - optional uint64 PQVersion = 10; - optional uint64 BSVVersion = 11; - optional uint64 KesusVersion = 12; - optional uint64 RTMRVersion = 13; - optional uint64 SolomonVersion = 14; +message TPathVersion { + optional uint64 GeneralVersion = 1; + + optional uint64 ACLVersion = 2; + optional uint64 EffectiveACLVersion = 3; + optional uint64 UserAttrsVersion = 4; + optional uint64 ChildrenVersion = 5; + optional uint64 SubDomainVersion = 6; + optional uint64 TableSchemaVersion = 7; + optional uint64 TablePartitionVersion = 8; + optional uint64 TableIndexVersion = 9; + optional uint64 PQVersion = 10; + optional uint64 BSVVersion = 11; + optional uint64 KesusVersion = 12; + optional uint64 RTMRVersion = 13; + optional uint64 SolomonVersion = 14; optional uint64 FileStoreVersion = 15; optional uint64 ColumnStoreVersion = 16; optional uint64 ColumnTableVersion = 17; @@ -1376,8 +1376,8 @@ message TPathVersion { optional uint64 SecurityStateVersion = 23; optional uint64 SequenceVersion = 24; optional uint64 ReplicationVersion = 25; -} - +} + // Describes single path message TDirEntry { optional string Name = 1; @@ -1394,7 +1394,7 @@ message TDirEntry { optional bytes EffectiveACL = 12; optional uint64 PathVersion = 13; optional EPathSubType PathSubType = 14; - optional TPathVersion Version = 15; + optional TPathVersion Version = 15; optional uint64 BalancerTabletID = 999; //temporary optimization for old PQ read/write protocol. Must be removed later } diff --git a/ydb/core/protos/index_builder.proto b/ydb/core/protos/index_builder.proto index 213c4d2bf6..f6ca5f9da6 100644 --- a/ydb/core/protos/index_builder.proto +++ b/ydb/core/protos/index_builder.proto @@ -10,8 +10,8 @@ message TIndexBuildSettings { optional string source_path = 1; optional Ydb.Table.TableIndex index = 2; - optional uint32 max_batch_rows = 3 [ default = 500 ]; - optional uint64 max_batch_bytes = 4 [ default = 8388608 ]; + optional uint32 max_batch_rows = 3 [ default = 500 ]; + optional uint64 max_batch_bytes = 4 [ default = 8388608 ]; optional uint32 max_shards_in_flight = 5 [ default = 32 ]; optional uint32 max_retries_upload_batch = 6 [ default = 50 ]; } @@ -19,9 +19,9 @@ message TIndexBuildSettings { message TIndexBuild { optional uint64 Id = 1; repeated Ydb.Issue.IssueMessage Issues = 2; - optional Ydb.Table.IndexBuildState.State State = 3; + optional Ydb.Table.IndexBuildState.State State = 3; optional TIndexBuildSettings Settings = 4; - optional float Progress = 5 [default = 0]; + optional float Progress = 5 [default = 0]; } message TEvCreateRequest { diff --git a/ydb/core/protos/issue_id.proto b/ydb/core/protos/issue_id.proto index 455547b15b..1c82c0d95e 100644 --- a/ydb/core/protos/issue_id.proto +++ b/ydb/core/protos/issue_id.proto @@ -41,11 +41,11 @@ message TIssuesIds { SCOPE_TXPROXY_RESOLVE = 200701; SCOPE_TXPROXY_PREPARE = 200702; SCOPE_TXPROXY_EXECUTE = 200703; - - YDB_API_VALIDATION_ERROR = 200800; - YDB_AUTH_UNAVAILABLE = 200801; - YDB_DB_NOT_READY = 200802; - YDB_RESOURCE_USAGE_LIMITED = 200803; + + YDB_API_VALIDATION_ERROR = 200800; + YDB_AUTH_UNAVAILABLE = 200801; + YDB_DB_NOT_READY = 200802; + YDB_RESOURCE_USAGE_LIMITED = 200803; } message TIssueId { diff --git a/ydb/core/protos/kesus.proto b/ydb/core/protos/kesus.proto index 95da801369..7c03a3ccd9 100644 --- a/ydb/core/protos/kesus.proto +++ b/ydb/core/protos/kesus.proto @@ -12,7 +12,7 @@ import "ydb/public/api/protos/ydb_issue_message.proto"; message TKesusError { Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; + repeated Ydb.Issue.IssueMessage Issues = 2; } message TEvDummyRequest { diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index c690935f5d..9b46b720e1 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -1,6 +1,6 @@ package NKikimrKqp; option java_package = "ru.yandex.kikimr.proto"; -option cc_enable_arenas = true; +option cc_enable_arenas = true; import "library/cpp/actors/protos/actors.proto"; import "ydb/core/protos/kqp_physical.proto"; @@ -25,7 +25,7 @@ enum ERequestType { enum EQueryType { QUERY_TYPE_UNDEFINED = 0; - reserved 1, 2, 3; + reserved 1, 2, 3; QUERY_TYPE_SQL_DML = 4; QUERY_TYPE_SQL_DDL = 5; @@ -84,7 +84,7 @@ message TQueryRequest { optional bool Profile = 10; // DEPRECATED, use StatsMode instead optional bytes PreparedQuery = 11; optional uint32 ReplyFlags = 12 [default = 1]; - optional Ydb.Table.TransactionControl TxControl = 13; + optional Ydb.Table.TransactionControl TxControl = 13; optional string Database = 14; optional Ydb.Table.QueryCachePolicy QueryCachePolicy = 15; optional uint64 CancelAfterMs = 16; @@ -139,18 +139,18 @@ message TKqpTableMetadataProto { repeated TKqpTableMetadataProto SecondaryGlobalIndexMetadata = 12; } -message TRlPath { - optional string CoordinationNode = 1; - optional string ResourcePath = 2; -} - +message TRlPath { + optional string CoordinationNode = 1; + optional string ResourcePath = 2; +} + message TEvQueryRequest { optional TQueryRequest Request = 1; optional string TraceId = 2; optional string UserToken = 3; optional NActorsProto.TActorId RequestActorId = 4; optional string RequestType = 5; - optional TRlPath RlPath = 6; + optional TRlPath RlPath = 6; } message TMkqlProfile { @@ -246,10 +246,10 @@ message TQueryResponse { optional bytes PreparedQuery = 5; optional string QueryAst = 6; reserved 7; - repeated Ydb.Issue.IssueMessage QueryIssues = 8; + repeated Ydb.Issue.IssueMessage QueryIssues = 8; optional bytes QueryPlan = 9; repeated TParameterDescription QueryParameters = 10; - optional Ydb.Table.TransactionMeta TxMeta = 11; + optional Ydb.Table.TransactionMeta TxMeta = 11; optional NKqpProto.TKqpStatsQuery QueryStats = 12; } @@ -259,7 +259,7 @@ message TEvQueryResponse { optional TQueryResponse Response = 3; optional Ydb.StatusIds.StatusCode YdbStatus = 4; reserved 5; // (deprecated) ForcedNewEngine - optional uint64 ConsumedRu = 6; + optional uint64 ConsumedRu = 6; optional bool WorkerIsClosing = 7 [default = false]; } @@ -297,7 +297,7 @@ message TKqpSetting { message TKqpDefaultSettings { repeated TKqpSetting DefaultSettings = 1; -} +} message TCreateSessionRequest { optional string Database = 1; @@ -306,7 +306,7 @@ message TCreateSessionRequest { message TEvCreateSessionRequest { optional TCreateSessionRequest Request = 1; optional string TraceId = 2; - optional uint64 DeadlineUs = 3; + optional uint64 DeadlineUs = 3; } message TCreateSessionResponse { @@ -370,7 +370,7 @@ message TExecuterTxResult { message TExecuterTxResponse { optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; + repeated Ydb.Issue.IssueMessage Issues = 2; optional TExecuterTxResult Result = 3; }; diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index e05003d347..df7cda5980 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -129,7 +129,7 @@ message TResponse { optional bytes ProxyErrors = 6; optional NKikimrIssues.TStatusIds.EStatusCode StatusCode = 7; - repeated Ydb.Issue.IssueMessage Issues = 8; + repeated Ydb.Issue.IssueMessage Issues = 8; optional uint32 ExecutionEngineStatus = 100 [ (CommonDescription) = "MiniKQL engine processing status, if OK - see ExecutionEngineResponseStatus", @@ -453,7 +453,7 @@ message TFlatDescribeResponse { optional int32 SchemeStatus = 4; optional NKikimrIssues.TStatusIds.EStatusCode StatusCode = 7; - repeated Ydb.Issue.IssueMessage Issues = 8; + repeated Ydb.Issue.IssueMessage Issues = 8; }; message TSchemeOperationStatus { @@ -713,7 +713,7 @@ message TConsoleRequest { NKikimrConsole.TGetNodeConfigItemsRequest GetNodeConfigItemsRequest = 9; NKikimrConsole.TGetNodeConfigRequest GetNodeConfigRequest = 10; NKikimrConsole.TRemoveTenantRequest RemoveTenantRequest = 11; - Ydb.Operations.GetOperationRequest GetOperationRequest = 12; + Ydb.Operations.GetOperationRequest GetOperationRequest = 12; NKikimrConsole.TCheckConfigUpdatesRequest CheckConfigUpdatesRequest = 13; NKikimrConsole.TListConfigValidatorsRequest ListConfigValidatorsRequest = 14; NKikimrConsole.TToggleConfigValidatorRequest ToggleConfigValidatorRequest = 15; @@ -735,7 +735,7 @@ message TConsoleResponse { NKikimrConsole.TGetNodeConfigItemsResponse GetNodeConfigItemsResponse = 9; NKikimrConsole.TGetNodeConfigResponse GetNodeConfigResponse = 10; NKikimrConsole.TRemoveTenantResponse RemoveTenantResponse = 11; - Ydb.Operations.GetOperationResponse GetOperationResponse = 12; + Ydb.Operations.GetOperationResponse GetOperationResponse = 12; NKikimrConsole.TCheckConfigUpdatesResponse CheckConfigUpdatesResponse = 13; NKikimrConsole.TListConfigValidatorsResponse ListConfigValidatorsResponse = 14; NKikimrConsole.TToggleConfigValidatorResponse ToggleConfigValidatorResponse = 15; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index cf5162c329..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -677,13 +677,13 @@ message TActivity { INTERCONNECT_MONACTOR = 362; BS_MONSTREAM_ACTOR = 363; NODE_WARDEN_STATAGGR_ACTOR = 364; - UPLOAD_ROWS_INTERNAL = 365; + UPLOAD_ROWS_INTERNAL = 365; PERSQUEUE_CLUSTER_TRACKER = 366; NET_CLASSIFIER_ACTOR = 367; KQP_COMPILE_SERVICE = 368; KQP_COMPILE_ACTOR = 369; PERSQUEUE_CLUSTER_DISCOVERY = 370; - TX_FILL_INDEX_SCAN = 371; + TX_FILL_INDEX_SCAN = 371; NET_CLASSIFIER_UPDATER = 372; BS_LOAD_PDISK_LOG_WRITE = 373; RTMR_CONFIGURATION_STORAGE = 374; diff --git a/ydb/core/protos/tx.proto b/ydb/core/protos/tx.proto index 040e8861c6..1f18e7eca6 100644 --- a/ydb/core/protos/tx.proto +++ b/ydb/core/protos/tx.proto @@ -279,5 +279,5 @@ message TEvInterruptTransaction { } message TEvStreamDataAck { -} - +} + diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index b1357264d3..89863b2540 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -28,7 +28,7 @@ enum EDatashardState { Readonly = 3; Offline = 4; PreOffline = 5; // Offline but waits for loaned snapshots to be returned and for SchemaChangedResult to be received - Frozen = 6; // Read only transactions are allowed. Scheme modification is forbiden + Frozen = 6; // Read only transactions are allowed. Scheme modification is forbiden // Split/Merge Src states SplitSrcWaitForNoTxInFlight = 101; // Temporary state: split src waits for all Tx to finish and then starts splitting @@ -52,7 +52,7 @@ message TEvGetShardStateResult { optional uint64 DropTxId = 3; } -message TShardOpResult { +message TShardOpResult { optional bool Success = 1; optional string Explain = 2; optional uint64 BytesProcessed = 3; @@ -67,7 +67,7 @@ message TEvSchemaChanged { optional uint64 Step = 5; optional uint32 Generation = 6; - optional TShardOpResult OpResult = 7; + optional TShardOpResult OpResult = 7; } message TEvSchemaChangedResult { @@ -107,7 +107,7 @@ message TReadTableTransaction { optional TTableId TableId = 1; repeated TColumn Columns = 2; optional NKikimrTx.TKeyRange Range = 3; - optional uint32 ApiVersion = 4; + optional uint32 ApiVersion = 4; optional uint64 SnapshotStep = 5; optional uint64 SnapshotTxId = 6; } @@ -575,7 +575,7 @@ message TEvProposeTransactionResult { // For read table tx result holds offsets in TxResult to cut response repeated uint32 RowOffsets = 20; repeated fixed64 DomainCoordinators = 21; - optional uint32 ApiVersion = 22; // Version of TxResult response data + optional uint32 ApiVersion = 22; // Version of TxResult response data optional NKikimrQueryStats.TTxStats TxStats = 23; optional uint64 DataSeqNo = 24; // Response data seqno (1, 2, ...) optional bytes DataLastKey = 25; // Response data last key (for retries) @@ -858,7 +858,7 @@ message TEvGetInfoResponse { optional NKikimrSchemeOp.TTableDescription Description = 5; optional TStats Stats = 6; optional NKikimrTabletBase.TMetrics Metrics = 7; - optional uint64 SchemaVersion = 8; + optional uint64 SchemaVersion = 8; } message TTabletInfo { @@ -1323,7 +1323,7 @@ message TEvBuildIndexCreateRequest { optional uint64 PathId = 4; optional string TargetName = 5; - repeated string IndexColumns = 6; // key columns that is needed to transer in terms of target table + repeated string IndexColumns = 6; // key columns that is needed to transer in terms of target table optional NKikimrTx.TKeyRange KeyRange = 7; // key range to transfer in term of source table @@ -1337,8 +1337,8 @@ message TEvBuildIndexCreateRequest { optional uint64 SeqNoRound = 13; // monotonically increasing sequence, second part optional uint64 MaxRetries = 14; - - repeated string DataColumns = 15; + + repeated string DataColumns = 15; } message TEvBuildIndexProgressResponse { diff --git a/ydb/core/protos/tx_proxy.proto b/ydb/core/protos/tx_proxy.proto index 547907ef6d..dedd2f9cbb 100644 --- a/ydb/core/protos/tx_proxy.proto +++ b/ydb/core/protos/tx_proxy.proto @@ -57,17 +57,17 @@ message TKeyRange { }; message TReadTableTransaction { - enum EVersion { - UNSPECIFIED = 0; - YDB_V1 = 1; - } + enum EVersion { + UNSPECIFIED = 0; + YDB_V1 = 1; + } optional string Path = 1; optional bool Ordered = 2 [default = false]; optional bool AllowDuplicates = 3 [default = false]; repeated string Columns = 4; // All by default. optional TKeyRange KeyRange = 5; optional uint64 RowLimit = 6 [default = 0]; // All rows by default. - optional uint32 ApiVersion = 7; + optional uint32 ApiVersion = 7; optional uint64 SnapshotStep = 8; optional uint64 SnapshotTxId = 9; }; @@ -162,9 +162,9 @@ message TMiniKQLTransaction { message TMiniKQLCompileResults { optional bytes CompiledProgram = 1; - repeated Ydb.Issue.IssueMessage ProgramCompileErrors = 2; + repeated Ydb.Issue.IssueMessage ProgramCompileErrors = 2; optional bytes CompiledParams = 20; - repeated Ydb.Issue.IssueMessage ParamsCompileErrors = 21; + repeated Ydb.Issue.IssueMessage ParamsCompileErrors = 21; }; message TTransaction { @@ -214,7 +214,7 @@ message TEvProposeTransactionStatus { optional fixed64 Step = 3; optional NKikimrIssues.TStatusIds.EStatusCode StatusCode = 7; - repeated Ydb.Issue.IssueMessage Issues = 8; + repeated Ydb.Issue.IssueMessage Issues = 8; optional uint32 ExecutionEngineStatus = 10; optional uint32 ExecutionEngineResponseStatus = 11; @@ -242,7 +242,7 @@ message TEvProposeTransactionStatus { optional fixed64 DataShardTabletId = 40; optional bytes SerializedReadTableResponse = 50; - optional uint32 ReadTableResponseVersion = 51; + optional uint32 ReadTableResponseVersion = 51; } message TEvNavigate { diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 49877e98be..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -125,7 +125,7 @@ SRCS( grpc_pq_old.proto grpc_status_proxy.proto ydb_result_set_old.proto - ydb_table_impl.proto + ydb_table_impl.proto scheme_board.proto scheme_board_mon.proto sys_view.proto diff --git a/ydb/core/protos/ydb_result_set_old.proto b/ydb/core/protos/ydb_result_set_old.proto index eb49c94455..9f7e4ebfc4 100644 --- a/ydb/core/protos/ydb_result_set_old.proto +++ b/ydb/core/protos/ydb_result_set_old.proto @@ -94,7 +94,7 @@ message Value { repeated Value items = 12; // Used for List, Tuple, Struct types repeated ValuePair pairs = 13; // Used for Dict type uint32 variant_index = 14; // Used for Variant type - fixed64 high_128 = 16; + fixed64 high_128 = 16; } message ColumnMeta { diff --git a/ydb/core/protos/ydb_table_impl.proto b/ydb/core/protos/ydb_table_impl.proto index 3e5b33b2a3..fa2fde23b2 100644 --- a/ydb/core/protos/ydb_table_impl.proto +++ b/ydb/core/protos/ydb_table_impl.proto @@ -1,16 +1,16 @@ -syntax = "proto3"; -//option cc_enable_arenas = true; - +syntax = "proto3"; +//option cc_enable_arenas = true; + import "ydb/public/api/protos/ydb_issue_message.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; - -package Ydb.Impl; -message ReadTableResponse { - StatusIds.StatusCode status = 1; - repeated Ydb.Issue.IssueMessage issues = 2; - ReadTobleResult result = 3; -}; - -message ReadTobleResult { - bytes result_set = 1; -}; + +package Ydb.Impl; +message ReadTableResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; + ReadTobleResult result = 3; +}; + +message ReadTobleResult { + bytes result_set = 1; +}; diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h index 035b6e168d..bccfdca97d 100644 --- a/ydb/core/scheme/scheme_tablecell.h +++ b/ydb/core/scheme/scheme_tablecell.h @@ -161,7 +161,7 @@ inline int CompareTypedCells(const TCell& a, const TCell& b, NScheme::TTypeIdOrd case NKikimr::NScheme::NTypeIds::String4k: case NKikimr::NScheme::NTypeIds::String2m: case NKikimr::NScheme::NTypeIds::Utf8: - case NKikimr::NScheme::NTypeIds::Json: + case NKikimr::NScheme::NTypeIds::Json: case NKikimr::NScheme::NTypeIds::Yson: // XXX: using memcmp is meaningless for both JsonDocument and Json case NKikimr::NScheme::NTypeIds::JsonDocument: diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h index 5a1843c47f..190233fdee 100644 --- a/ydb/core/scheme/scheme_tabledefs.h +++ b/ydb/core/scheme/scheme_tabledefs.h @@ -15,8 +15,8 @@ namespace NKikimr { -using TSchemaVersion = ui64; - +using TSchemaVersion = ui64; + // ident for table, must be unique in selected scope // for global transactions ownerid is tabletid of owning schemeshard and tableid is counter designated by schemeshard // SysViewInfo is not empty for system views attached to corresponding table @@ -24,7 +24,7 @@ using TSchemaVersion = ui64; struct TTableId { TPathId PathId; TString SysViewInfo; - TSchemaVersion SchemaVersion = 0; + TSchemaVersion SchemaVersion = 0; TTableId() = default; @@ -39,10 +39,10 @@ struct TTableId { : TTableId(ownerId, tableId, sysViewInfo, 0) {} - TTableId(ui64 ownerId, ui64 tableId, ui64 schemaVersion) + TTableId(ui64 ownerId, ui64 tableId, ui64 schemaVersion) : TTableId(ownerId, tableId, TString(), schemaVersion) - {} - + {} + TTableId(ui64 ownerId, ui64 tableId) : TTableId(ownerId, tableId, TString(), 0) {} @@ -68,7 +68,7 @@ struct TTableId { return bool(PathId); } - bool HasSamePath(const TTableId &x) const noexcept { + bool HasSamePath(const TTableId &x) const noexcept { return PathId == x.PathId && SysViewInfo == x.SysViewInfo; } @@ -96,7 +96,7 @@ struct TTableId { return hash; } - ui64 PathHash() const noexcept { + ui64 PathHash() const noexcept { auto hash = PathId.Hash(); if (SysViewInfo) { hash = CombineHashes(hash, THash<TString>()(SysViewInfo)); @@ -107,37 +107,37 @@ struct TTableId { }; struct TIndexId { - TPathId PathId; - TSchemaVersion SchemaVersion = 0; - - TIndexId(const TPathId& pathId, TSchemaVersion schemaVersion) - : PathId(pathId) - , SchemaVersion(schemaVersion) - {} - - TIndexId(const TOwnerId ownerId, const TLocalPathId localPathId, TSchemaVersion schemaVersion) - : PathId(ownerId, localPathId) - , SchemaVersion(schemaVersion) - {} -}; - -struct TTablePathHashFn { - ui64 operator()(const NKikimr::TTableId &x) const noexcept { - return x.PathHash(); - } -}; - -struct TTablePathEqualFn { - bool operator()(const NKikimr::TTableId &x, const NKikimr::TTableId &y) const noexcept { - return x.HasSamePath(y); - } -}; - -template <typename T> -using TTablePathHashMap = THashMap<TTableId, T, TTablePathHashFn, TTablePathEqualFn>; - -using TTablePathHashSet = THashSet<TTableId, TTablePathHashFn, TTablePathEqualFn>; - + TPathId PathId; + TSchemaVersion SchemaVersion = 0; + + TIndexId(const TPathId& pathId, TSchemaVersion schemaVersion) + : PathId(pathId) + , SchemaVersion(schemaVersion) + {} + + TIndexId(const TOwnerId ownerId, const TLocalPathId localPathId, TSchemaVersion schemaVersion) + : PathId(ownerId, localPathId) + , SchemaVersion(schemaVersion) + {} +}; + +struct TTablePathHashFn { + ui64 operator()(const NKikimr::TTableId &x) const noexcept { + return x.PathHash(); + } +}; + +struct TTablePathEqualFn { + bool operator()(const NKikimr::TTableId &x, const NKikimr::TTableId &y) const noexcept { + return x.HasSamePath(y); + } +}; + +template <typename T> +using TTablePathHashMap = THashMap<TTableId, T, TTablePathHashFn, TTablePathEqualFn>; + +using TTablePathHashSet = THashSet<TTableId, TTablePathHashFn, TTablePathEqualFn>; + // defines key range as low and high borders (possibly partial - w/o defined key values on tail) // missing values interpreted as infinity and mean "greater then any key with set prefix" // column order must match table key order @@ -738,8 +738,8 @@ inline void Out<NKikimr::TTableId>(IOutputStream& o, const NKikimr::TTableId& x) } o << ']'; } - -template<> -inline void Out<NKikimr::TIndexId>(IOutputStream& o, const NKikimr::TIndexId& x) { - o << '[' << x.PathId.OwnerId << ':' << x.PathId.LocalPathId << ':' << x.SchemaVersion << ']'; -} + +template<> +inline void Out<NKikimr::TIndexId>(IOutputStream& o, const NKikimr::TIndexId& x) { + o << '[' << x.PathId.OwnerId << ':' << x.PathId.LocalPathId << ':' << x.SchemaVersion << ']'; +} diff --git a/ydb/core/scheme_types/scheme_type_registry.cpp b/ydb/core/scheme_types/scheme_type_registry.cpp index 309a58010c..d737254361 100644 --- a/ydb/core/scheme_types/scheme_type_registry.cpp +++ b/ydb/core/scheme_types/scheme_type_registry.cpp @@ -28,7 +28,7 @@ TTypeRegistry::TTypeRegistry() RegisterType<TLargeBoundedString>(); RegisterType<TUtf8>(); RegisterType<TYson>(); - RegisterType<TJson>(); + RegisterType<TJson>(); RegisterType<TJsonDocument>(); RegisterType<TDecimal>(); RegisterType<TDate>(); diff --git a/ydb/core/scheme_types/scheme_type_traits.h b/ydb/core/scheme_types/scheme_type_traits.h index 4c4ea8169a..7e07f95976 100644 --- a/ydb/core/scheme_types/scheme_type_traits.h +++ b/ydb/core/scheme_types/scheme_type_traits.h @@ -47,7 +47,7 @@ constexpr bool IsStringType(TTypeId id) noexcept { || NTypeIds::String2m == id || NTypeIds::Utf8 == id || NTypeIds::Yson == id - || NTypeIds::Json == id + || NTypeIds::Json == id || NTypeIds::JsonDocument == id || NTypeIds::DyNumber ; diff --git a/ydb/core/scheme_types/scheme_types_defs.cpp b/ydb/core/scheme_types/scheme_types_defs.cpp index 659d81db66..eeaba2e9af 100644 --- a/ydb/core/scheme_types/scheme_types_defs.cpp +++ b/ydb/core/scheme_types/scheme_types_defs.cpp @@ -22,7 +22,7 @@ namespace NNames { DECLARE_TYPED_TYPE_NAME(Utf8); DECLARE_TYPED_TYPE_NAME(Yson); - DECLARE_TYPED_TYPE_NAME(Json); + DECLARE_TYPED_TYPE_NAME(Json); DECLARE_TYPED_TYPE_NAME(JsonDocument); DECLARE_TYPED_TYPE_NAME(Decimal); diff --git a/ydb/core/scheme_types/scheme_types_defs.h b/ydb/core/scheme_types/scheme_types_defs.h index 2f7e8eaf55..bb06b09702 100644 --- a/ydb/core/scheme_types/scheme_types_defs.h +++ b/ydb/core/scheme_types/scheme_types_defs.h @@ -177,9 +177,9 @@ public: }; class TJson : public TStringBase<TJson, NTypeIds::Json, NNames::Json> { -public: -}; - +public: +}; + class TJsonDocument : public TStringBase<TJsonDocument, NTypeIds::JsonDocument, NNames::JsonDocument> { public: }; @@ -247,7 +247,7 @@ class TInterval : public IIntegerTypeWithKeyString<i64, NTypeIds::Interval, NNam xx(String2m, TLargeBoundedString, __VA_ARGS__) \ xx(Utf8, TUtf8, __VA_ARGS__) \ xx(Yson, TYson, __VA_ARGS__) \ - xx(Json, TJson, __VA_ARGS__) \ + xx(Json, TJson, __VA_ARGS__) \ xx(JsonDocument, TJsonDocument, __VA_ARGS__) \ xx(Decimal, TDecimal, __VA_ARGS__) \ xx(Date, TDate, __VA_ARGS__) \ diff --git a/ydb/core/security/secure_request.h b/ydb/core/security/secure_request.h index 38df0ad610..04e276a897 100644 --- a/ydb/core/security/secure_request.h +++ b/ydb/core/security/secure_request.h @@ -36,14 +36,14 @@ private: const TEvTicketParser::TEvAuthorizeTicketResult& result(*ev->Get()); if (!result.Error.empty()) { if (IsTokenRequired()) { - return static_cast<TDerived*>(this)->OnAccessDenied(result.Error, ctx); + return static_cast<TDerived*>(this)->OnAccessDenied(result.Error, ctx); } } else { if (RequireAdminAccess) { if (!GetAdministrationAllowedSIDs().empty()) { const auto& allowedSIDs(GetAdministrationAllowedSIDs()); if (std::find_if(allowedSIDs.begin(), allowedSIDs.end(), [&result](const TString& sid) -> bool { return result.Token->IsExist(sid); }) == allowedSIDs.end()) { - return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Administrative access denied", false}, ctx); + return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Administrative access denied", false}, ctx); } } UserAdmin = true; @@ -55,7 +55,7 @@ private: void Handle(TEvents::TEvUndelivered::TPtr&, const TActorContext& ctx) { if (IsTokenRequired()) { - return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Access denied - error parsing token", false}, ctx); + return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Access denied - error parsing token", false}, ctx); } static_cast<TBootstrap*>(this)->Bootstrap(ctx); } @@ -88,8 +88,8 @@ public: void SetEntries(const TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry>& entries) { Entries = entries; - } - + } + const TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry>& GetEntries() const { return Entries; } @@ -135,7 +135,7 @@ public: void Bootstrap(const TActorContext& ctx) { if (IsTokenRequired() && !IsTokenExists()) { - return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Access denied without user token", false}, ctx); + return static_cast<TDerived*>(this)->OnAccessDenied(TEvTicketParser::TError{"Access denied without user token", false}, ctx); } if (SecurityToken.empty()) { if (!GetDefaultUserSIDs().empty()) { diff --git a/ydb/core/tablet/tablet_req_blockbs.cpp b/ydb/core/tablet/tablet_req_blockbs.cpp index 1f69bca182..7e2e94ec95 100644 --- a/ydb/core/tablet/tablet_req_blockbs.cpp +++ b/ydb/core/tablet/tablet_req_blockbs.cpp @@ -135,13 +135,13 @@ public: for (auto &x : Info->Channels) { if (auto *g = x.LatestEntry()) { if (blocked.insert(g->GroupID).second) - ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, g->GroupID, Generation))); + ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, g->GroupID, Generation))); } if (BlockPrevEntry) { if (auto *pg = x.PreviousEntry()) if (blocked.insert(pg->GroupID).second) - ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, pg->GroupID, Generation))); + ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, pg->GroupID, Generation))); } } diff --git a/ydb/core/tablet_flat/flat_executor_db_mon.cpp b/ydb/core/tablet_flat/flat_executor_db_mon.cpp index 06845c9148..d5353747be 100644 --- a/ydb/core/tablet_flat/flat_executor_db_mon.cpp +++ b/ydb/core/tablet_flat/flat_executor_db_mon.cpp @@ -208,7 +208,7 @@ public: str << *(TActorId*)data; break; case NScheme::NTypeIds::Utf8: - case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::Json: str << EncodeHtmlPcdata(TStringBuf((const char*)data, size)); break; case NScheme::NTypeIds::JsonDocument: { diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index cf61dfb039..c9cf528971 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -282,7 +282,7 @@ private: Scheme = std::move(scheme); auto ctx = TActivationContext::AsActorContext(); - ctx.RegisterWithSameMailbox(this); + ctx.RegisterWithSameMailbox(this); if (Postponed) { ctx.Send(Tablet, new NFake::TEvReturn); diff --git a/ydb/core/tablet_flat/flat_ops_compact.h b/ydb/core/tablet_flat/flat_ops_compact.h index 4545718509..2ac81c4113 100644 --- a/ydb/core/tablet_flat/flat_ops_compact.h +++ b/ydb/core/tablet_flat/flat_ops_compact.h @@ -96,7 +96,7 @@ namespace NTabletFlatExecutor { THello Prepare(IDriver *driver, TIntrusiveConstPtr<TScheme> scheme) noexcept override { - TActivationContext::AsActorContext().RegisterWithSameMailbox(this); + TActivationContext::AsActorContext().RegisterWithSameMailbox(this); Spent = new TSpent(TAppData::TimeProvider.Get()); Registry = AppData()->TypeRegistry; diff --git a/ydb/core/tablet_flat/tablet_flat_executed.cpp b/ydb/core/tablet_flat/tablet_flat_executed.cpp index 7238bd12e8..4b3c15490c 100644 --- a/ydb/core/tablet_flat/tablet_flat_executed.cpp +++ b/ydb/core/tablet_flat/tablet_flat_executed.cpp @@ -19,7 +19,7 @@ TTabletExecutedFlat::TTabletExecutedFlat(TTabletStorageInfo *info, const TActorI IExecutor* TTabletExecutedFlat::CreateExecutor(const TActorContext &ctx) { if (!Executor()) { IActor *executor = NFlatExecutorSetup::CreateExecutor(this, ctx.SelfID); - const TActorId executorID = ctx.RegisterWithSameMailbox(executor); + const TActorId executorID = ctx.RegisterWithSameMailbox(executor); Executor0 = dynamic_cast<TExecutor *>(executor); Y_VERIFY(Executor0); diff --git a/ydb/core/testlib/actors/test_runtime_ut.cpp b/ydb/core/testlib/actors/test_runtime_ut.cpp index 63591bbdf4..fce7f19293 100644 --- a/ydb/core/testlib/actors/test_runtime_ut.cpp +++ b/ydb/core/testlib/actors/test_runtime_ut.cpp @@ -195,7 +195,7 @@ Y_UNIT_TEST_SUITE(TActorTest) { STFUNC(StateFunc) { Y_UNUSED(ev); - ChildId = ctx.RegisterWithSameMailbox(new TChildActor()); + ChildId = ctx.RegisterWithSameMailbox(new TChildActor()); ctx.Send(ChildId, new TEvents::TEvPing()); } diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h index 5a4ecede4a..7a70063f7a 100644 --- a/ydb/core/testlib/basics/helpers.h +++ b/ydb/core/testlib/basics/helpers.h @@ -49,7 +49,7 @@ namespace NFake { void SetupMonitoringProxy(TTestActorRuntime& runtime, ui32 nodeIndex); void SetupGRpcProxyStatus(TTestActorRuntime& runtime, ui32 nodeIndex); void SetupNodeTabletMonitor(TTestActorRuntime& runtime, ui32 nodeIndex); - void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root); + void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root); void SetupQuoterService(TTestActorRuntime& runtime, ui32 nodeIndex); void SetupSysViewService(TTestActorRuntime& runtime, ui32 nodeIndex); diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp index b280a5f54b..b4585ad96c 100644 --- a/ydb/core/testlib/basics/services.cpp +++ b/ydb/core/testlib/basics/services.cpp @@ -45,16 +45,16 @@ namespace NPDisk { nodeIndex); } - void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root) - { + void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root) + { auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); cacheConfig->Roots.emplace_back(1, TTestTxConfig::SchemeShard, root); - cacheConfig->Counters = new NMonitoring::TDynamicCounters(); - + cacheConfig->Counters = new NMonitoring::TDynamicCounters(); + runtime.AddLocalService(MakeSchemeCacheID(), TActorSetupCmd(CreateSchemeBoardSchemeCache(cacheConfig.Get()), TMailboxType::Revolving, 0), nodeIndex); - } - + } + void SetupTabletResolver(TTestActorRuntime& runtime, ui32 nodeIndex) { TIntrusivePtr<TTabletResolverConfig> tabletResolverConfig(new TTabletResolverConfig()); diff --git a/ydb/core/testlib/minikql_compile.h b/ydb/core/testlib/minikql_compile.h index 73bfaecc47..4d76ca8d73 100644 --- a/ydb/core/testlib/minikql_compile.h +++ b/ydb/core/testlib/minikql_compile.h @@ -83,7 +83,7 @@ inline TExprContainer::TPtr ParseText(const TString& programText) { TExprContainer::TPtr expr(new TExprContainer()); bool isOk = CompileExpr(*astRes.Root, expr->Root, expr->Context, nullptr); - expr->Context.IssueManager.GetIssues().PrintTo(Cerr); + expr->Context.IssueManager.GetIssues().PrintTo(Cerr); UNIT_ASSERT(isOk); return expr; } diff --git a/ydb/core/testlib/tenant_helpers.h b/ydb/core/testlib/tenant_helpers.h index d541f4a602..209b73baa0 100644 --- a/ydb/core/testlib/tenant_helpers.h +++ b/ydb/core/testlib/tenant_helpers.h @@ -168,18 +168,18 @@ inline void CheckCreateTenant(TTenantTestRuntime &runtime, unit.set_availability_zone(req.Zone); unit.set_count(req.Count); } - + for (auto &pool : request.Pools) { auto &unit = *resources->add_storage_units(); unit.set_unit_kind(pool.PoolType); unit.set_count(pool.PoolSize); } - + if (token) event->Record.SetUserToken(token); - - for (const auto& [key, value] : request.Attrs) { - (*event->Record.MutableRequest()->mutable_attributes())[key] = value; + + for (const auto& [key, value] : request.Attrs) { + (*event->Record.MutableRequest()->mutable_attributes())[key] = value; } if (request.Type == EType::Serverless) { diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index bbf1b8c794..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -92,7 +92,7 @@ #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/grpc/server/actors/logger.h> - + #include <util/system/sanitizers.h> #include <util/system/valgrind.h> #include <util/system/env.h> @@ -246,12 +246,12 @@ namespace Tests { void TServer::EnableGRpc(const NGrpc::TServerOptions& options) { GRpcServer.reset(new NGrpc::TGRpcServer(options)); - auto grpcService = new NGRpcProxy::TGRpcService(); + auto grpcService = new NGRpcProxy::TGRpcService(); auto system(Runtime->GetAnyNodeActorSystem()); auto grpcRequestProxy = NGRpcService::CreateGRpcRequestProxy(Settings->AppConfig); auto grpcRequestProxyId = system->Register(grpcRequestProxy, TMailboxType::ReadAsFilled); - system->RegisterLocalService(NGRpcService::CreateGRpcRequestProxyId(), grpcRequestProxyId); + system->RegisterLocalService(NGRpcService::CreateGRpcRequestProxyId(), grpcRequestProxyId); auto grpcMon = system->Register(NGRpcService::CreateGrpcMonService(), TMailboxType::ReadAsFilled); system->RegisterLocalService(NGRpcService::GrpcMonServiceId(), grpcMon); @@ -279,27 +279,27 @@ namespace Tests { system->Register(NGRpcService::CreateGrpcEndpointPublishActor(desc.Get()), TMailboxType::ReadAsFilled, appData.UserPoolId); } - auto future = grpcService->Prepare( + auto future = grpcService->Prepare( system, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), counters ); - auto startCb = [grpcService] (NThreading::TFuture<void> result) { - if (result.HasException()) { - try { - result.GetValue(); - } catch (const std::exception& ex) { - Y_FAIL("Unable to prepare GRpc service: %s", ex.what()); - } - } else { - grpcService->Start(); - } - }; - - future.Subscribe(startCb); - - GRpcServer->AddService(grpcService); + auto startCb = [grpcService] (NThreading::TFuture<void> result) { + if (result.HasException()) { + try { + result.GetValue(); + } catch (const std::exception& ex) { + Y_FAIL("Unable to prepare GRpc service: %s", ex.what()); + } + } else { + grpcService->Start(); + } + }; + + future.Subscribe(startCb); + + GRpcServer->AddService(grpcService); GRpcServer->AddService(new NGRpcService::TGRpcYdbExportService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbImportService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbSchemeService(system, counters, grpcRequestProxyId)); @@ -310,7 +310,7 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcCmsService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcDiscoveryService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcDiscoveryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbExperimentalService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbClickhouseInternalService(system, counters, appData.InFlightLimiterRegistry, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbS3InternalService(system, counters, grpcRequestProxyId)); @@ -321,26 +321,26 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcYandexQueryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId)); } - if (const auto& factory = Settings->GrpcServiceFactory) { - // All services enabled by default for ut - static const std::unordered_set<TString> dummy; - for (const auto& service : factory->Create(dummy, dummy, system, counters, grpcRequestProxyId)) { - GRpcServer->AddService(service); - } - } + if (const auto& factory = Settings->GrpcServiceFactory) { + // All services enabled by default for ut + static const std::unordered_set<TString> dummy; + for (const auto& service : factory->Create(dummy, dummy, system, counters, grpcRequestProxyId)) { + GRpcServer->AddService(service); + } + } GRpcServer->AddService(new NGRpcService::TGRpcYdbLogStoreService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcAuthService(system, counters, grpcRequestProxyId)); - GRpcServer->Start(); + GRpcServer->Start(); } - void TServer::EnableGRpc(ui16 port) { - EnableGRpc(NGrpc::TServerOptions() - .SetHost("localhost") - .SetPort(port) + void TServer::EnableGRpc(ui16 port) { + EnableGRpc(NGrpc::TServerOptions() + .SetHost("localhost") + .SetPort(port) .SetLogger(NGrpc::CreateActorSystemLogger(*Runtime->GetAnyNodeActorSystem(), NKikimrServices::GRPC_SERVER)) - ); - } - + ); + } + void TServer::SetupDomains(TAppPrepare &app) { const ui32 domainId = Settings->Domain; ui64 planResolution = Settings->DomainPlanResolution; @@ -875,14 +875,14 @@ namespace Tests { } TServer::~TServer() { - if (Runtime->GetAppData().Mon) { - Runtime->GetAppData().Mon->Stop(); - } - - if (GRpcServer) { - GRpcServer->Stop(); - } - + if (Runtime->GetAppData().Mon) { + Runtime->GetAppData().Mon->Stop(); + } + + if (GRpcServer) { + GRpcServer->Stop(); + } + if (Runtime) { Runtime.Destroy(); } @@ -1333,35 +1333,35 @@ namespace Tests { return (NMsgBusProxy::EResponseStatus)response.GetStatus(); } - NMsgBusProxy::EResponseStatus TClient::CreateTableWithUniformShardedIndex(const TString& parent, - const NKikimrSchemeOp::TTableDescription &table, const TString& indexName, const TVector<TString> indexColumns, TDuration timeout) - { - TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); - auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); - op->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable); - op->SetWorkingDir(parent); - - NKikimrSchemeOp::TTableDescription* tableDesc = op->MutableCreateIndexedTable()->MutableTableDescription(); - tableDesc->CopyFrom(table); - - { - auto indexDesc = op->MutableCreateIndexedTable()->MutableIndexDescription()->Add(); - indexDesc->SetName(indexName); - for (const auto& c : indexColumns) { - indexDesc->AddKeyColumnNames(c); - } - - indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobal); - indexDesc->MutableIndexImplTableDescription()->SetUniformPartitionsCount(16); - } - - TAutoPtr<NBus::TBusMessage> reply; - NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply, timeout); - UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); - const NKikimrClient::TResponse &response = dynamic_cast<NMsgBusProxy::TBusResponse *>(reply.Get())->Record; - return (NMsgBusProxy::EResponseStatus)response.GetStatus(); - } - + NMsgBusProxy::EResponseStatus TClient::CreateTableWithUniformShardedIndex(const TString& parent, + const NKikimrSchemeOp::TTableDescription &table, const TString& indexName, const TVector<TString> indexColumns, TDuration timeout) + { + TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); + auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); + op->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable); + op->SetWorkingDir(parent); + + NKikimrSchemeOp::TTableDescription* tableDesc = op->MutableCreateIndexedTable()->MutableTableDescription(); + tableDesc->CopyFrom(table); + + { + auto indexDesc = op->MutableCreateIndexedTable()->MutableIndexDescription()->Add(); + indexDesc->SetName(indexName); + for (const auto& c : indexColumns) { + indexDesc->AddKeyColumnNames(c); + } + + indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobal); + indexDesc->MutableIndexImplTableDescription()->SetUniformPartitionsCount(16); + } + + TAutoPtr<NBus::TBusMessage> reply; + NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply, timeout); + UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); + const NKikimrClient::TResponse &response = dynamic_cast<NMsgBusProxy::TBusResponse *>(reply.Get())->Record; + return (NMsgBusProxy::EResponseStatus)response.GetStatus(); + } + NMsgBusProxy::EResponseStatus TClient::SplitTable(const TString& table, ui64 datashardId, ui64 border, TDuration timeout) { TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); auto op = request->Record.MutableTransaction()->MutableModifyScheme(); @@ -1875,11 +1875,11 @@ namespace Tests { const auto &compileRes = response.GetMiniKQLCompileResults(); if (compileRes.ProgramCompileErrorsSize()) { - NYql::TIssues issues; - NYql::IssuesFromMessage(compileRes.GetProgramCompileErrors(), issues); - TStringStream err; - issues.PrintTo(err); - Cerr << "error: " << err.Str() << Endl; + NYql::TIssues issues; + NYql::IssuesFromMessage(compileRes.GetProgramCompileErrors(), issues); + TStringStream err; + issues.PrintTo(err); + Cerr << "error: " << err.Str() << Endl; return false; } @@ -1952,18 +1952,18 @@ namespace Tests { if (response.HasMiniKQLCompileResults()) { const auto &compileRes = response.GetMiniKQLCompileResults(); if (compileRes.ProgramCompileErrorsSize()) { - NYql::TIssues issues; - NYql::IssuesFromMessage(compileRes.GetProgramCompileErrors(), issues); - TStringStream err; - issues.PrintTo(err); - Cerr << "error: " << err.Str() << Endl; + NYql::TIssues issues; + NYql::IssuesFromMessage(compileRes.GetProgramCompileErrors(), issues); + TStringStream err; + issues.PrintTo(err); + Cerr << "error: " << err.Str() << Endl; } if (compileRes.ParamsCompileErrorsSize()) { - NYql::TIssues issues; - NYql::IssuesFromMessage(compileRes.GetParamsCompileErrors(), issues); - TStringStream err; - issues.PrintTo(err); - Cerr << "error: " << err.Str() << Endl; + NYql::TIssues issues; + NYql::IssuesFromMessage(compileRes.GetParamsCompileErrors(), issues); + TStringStream err; + issues.PrintTo(err); + Cerr << "error: " << err.Str() << Endl; } } if (response.HasHadFollowerReads() && response.GetHadFollowerReads()) { diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 948df99175..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -126,7 +126,7 @@ namespace Tests { TString MeteringFilePath; std::function<IActor*(const NKikimrProto::TAuthConfig&)> CreateTicketParser = NKikimr::CreateTicketParser; - std::shared_ptr<TGrpcServiceFactory> GrpcServiceFactory; + std::shared_ptr<TGrpcServiceFactory> GrpcServiceFactory; TServerSettings& SetGrpcPort(ui16 value) { GrpcPort = value; return *this; } TServerSettings& SetSupportsRedirect(bool value) { SupportsRedirect = value; return *this; } @@ -166,19 +166,19 @@ namespace Tests { return *this; } - // Add additional grpc services - template <typename TService> + // Add additional grpc services + template <typename TService> TServerSettings& RegisterGrpcService( const TString& name, std::optional<NActors::TActorId> proxyId = std::nullopt ) { - if (!GrpcServiceFactory) { - GrpcServiceFactory = std::make_shared<TGrpcServiceFactory>(); - } + if (!GrpcServiceFactory) { + GrpcServiceFactory = std::make_shared<TGrpcServiceFactory>(); + } GrpcServiceFactory->Register<TService>(name, true, proxyId); - return *this; - } - + return *this; + } + explicit TServerSettings(ui16 port, const NKikimrProto::TAuthConfig authConfig = {}, const NKikimrPQ::TPQConfig pqConfig = {}) : Port(port) , AuthConfig(authConfig) @@ -218,7 +218,7 @@ namespace Tests { virtual ~TServer(); void EnableGRpc(const NGrpc::TServerOptions& options); - void EnableGRpc(ui16 port); + void EnableGRpc(ui16 port); void SetupDefaultProfiles(); @@ -332,7 +332,7 @@ namespace Tests { return msgbusStatus; } - static ui64 GetPatchedSchemeRoot(ui64 schemeRoot, ui32 domain, bool supportsRedirect); + static ui64 GetPatchedSchemeRoot(ui64 schemeRoot, ui32 domain, bool supportsRedirect); void WaitRootIsUp(const TString& root); TAutoPtr<NBus::TBusMessage> InitRootSchemeWithReply(const TString& root); void InitRootScheme(); @@ -361,9 +361,9 @@ namespace Tests { NMsgBusProxy::EResponseStatus CreateTable(const TString& parent, const TString& scheme, TDuration timeout = TDuration::Seconds(5000)); NMsgBusProxy::EResponseStatus CreateTable(const TString& parent, const NKikimrSchemeOp::TTableDescription &table, TDuration timeout = TDuration::Seconds(5000)); - NMsgBusProxy::EResponseStatus CreateTableWithUniformShardedIndex(const TString& parent, - const NKikimrSchemeOp::TTableDescription &table, const TString& indexName, - const TVector<TString> indexColumns, TDuration timeout = TDuration::Seconds(5000)); + NMsgBusProxy::EResponseStatus CreateTableWithUniformShardedIndex(const TString& parent, + const NKikimrSchemeOp::TTableDescription &table, const TString& indexName, + const TVector<TString> indexColumns, TDuration timeout = TDuration::Seconds(5000)); NMsgBusProxy::EResponseStatus SplitTable(const TString& table, ui64 datashardId, ui64 border, TDuration timeout = TDuration::Seconds(5000)); NMsgBusProxy::EResponseStatus CopyTable(const TString& parent, const TString& name, const TString& src); NMsgBusProxy::EResponseStatus CreateKesus(const TString& parent, const TString& name); diff --git a/ydb/core/tx/coordinator/mediator_queue.cpp b/ydb/core/tx/coordinator/mediator_queue.cpp index 8c61f3d9ea..9b3955fa1b 100644 --- a/ydb/core/tx/coordinator/mediator_queue.cpp +++ b/ydb/core/tx/coordinator/mediator_queue.cpp @@ -39,7 +39,7 @@ class TTxCoordinatorMediatorQueue : public TActorBootstrapped<TTxCoordinatorMedi PipeClient = TActorId(); } - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, Mediator)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, Mediator)); LOG_DEBUG_S(ctx, NKikimrServices::TX_COORDINATOR_MEDIATOR_QUEUE, "Actor# " << ctx.SelfID.ToString() << " tablet# " << Coordinator << " SEND EvCoordinatorSync to# " << Mediator << " Mediator"); diff --git a/ydb/core/tx/datashard/alter_table_unit.cpp b/ydb/core/tx/datashard/alter_table_unit.cpp index a99d5be97a..78786adae7 100644 --- a/ydb/core/tx/datashard/alter_table_unit.cpp +++ b/ydb/core/tx/datashard/alter_table_unit.cpp @@ -134,11 +134,11 @@ EExecutionStatus TAlterTableUnit::Execute(TOperation::TPtr op, if (!schemeTx.HasAlterTable()) return EExecutionStatus::Executed; - const auto& alterTableTx = schemeTx.GetAlterTable(); - + const auto& alterTableTx = schemeTx.GetAlterTable(); + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Trying to ALTER TABLE at " << DataShard.TabletID() - << " version " << alterTableTx.GetTableSchemaVersion()); + "Trying to ALTER TABLE at " << DataShard.TabletID() + << " version " << alterTableTx.GetTableSchemaVersion()); TPathId tableId(DataShard.GetPathOwnerId(), alterTableTx.GetId_Deprecated()); if (alterTableTx.HasPathId()) { diff --git a/ydb/core/tx/datashard/backup_unit.cpp b/ydb/core/tx/datashard/backup_unit.cpp index 739ad7cef5..002b3e74b2 100644 --- a/ydb/core/tx/datashard/backup_unit.cpp +++ b/ydb/core/tx/datashard/backup_unit.cpp @@ -111,7 +111,7 @@ protected: auto* result = CheckedCast<TExportScanProduct*>(op->ScanResult().Get()); auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); - schemeOp->Success = result->Success; + schemeOp->Success = result->Success; schemeOp->Error = std::move(result->Error); schemeOp->BytesProcessed = result->BytesRead; schemeOp->RowsProcessed = result->RowsRead; diff --git a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp index 60ddcdc112..bbba1c6d95 100644 --- a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp @@ -37,8 +37,8 @@ private: bool CheckCreateCdcStream(TActiveTransaction *activeTx); bool CheckAlterCdcStream(TActiveTransaction *activeTx); bool CheckDropCdcStream(TActiveTransaction *activeTx); - - bool CheckSchemaVersion(TActiveTransaction *activeTx, ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion); + + bool CheckSchemaVersion(TActiveTransaction *activeTx, ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion); using TPipelineHasSmthFunc = std::function<bool(TPipeline* const)>; bool HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker); @@ -76,17 +76,17 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, TActiveTransaction *activeTx = dynamic_cast<TActiveTransaction*>(op.Get()); Y_VERIFY_S(activeTx, "cannot cast operation of kind " << op->GetKind()); - const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - bool unfreezeTx = false; - if (tx.HasAlterTable() && tx.GetAlterTable().HasPartitionConfig() && - tx.GetAlterTable().GetPartitionConfig().HasFreezeState()) - { - auto cmd = tx.GetAlterTable().GetPartitionConfig().GetFreezeState(); + const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); + bool unfreezeTx = false; + if (tx.HasAlterTable() && tx.GetAlterTable().HasPartitionConfig() && + tx.GetAlterTable().GetPartitionConfig().HasFreezeState()) + { + auto cmd = tx.GetAlterTable().GetPartitionConfig().GetFreezeState(); unfreezeTx = cmd == NKikimrSchemeOp::EFreezeState::Unfreeze; - } + } // Check state is proper for scheme transactions. - if (!DataShard.IsStateActive() || (DataShard.IsStateFrozen() && !unfreezeTx)) { + if (!DataShard.IsStateActive() || (DataShard.IsStateFrozen() && !unfreezeTx)) { TString error = TStringBuilder() << "Wrong shard state for scheme transaction at tablet " << DataShard.TabletID() << " state: " << DataShard.GetState() << " txId: " @@ -207,30 +207,30 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, return EExecutionStatus::ExecutedNoMoreRestarts; } -bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, - ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion) -{ - LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Check scheme tx, proposed scheme version# " << proposedSchemaVersion << - " current version# " << currentSchemaVersion << - " expected version# " << expectedSchemaVersion << - " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId()); - - // Allow scheme tx if proposed or current schema version is zero. This simplify migration a lot. - if (proposedSchemaVersion && currentSchemaVersion && expectedSchemaVersion != proposedSchemaVersion) { - TString err = TStringBuilder() - << "Wrong schema version: proposed# " << proposedSchemaVersion << - " current version# " << currentSchemaVersion << - " expected version# " << expectedSchemaVersion << - " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId(); - - LOG_CRIT_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, err); - return false; - } - - return true; -} - +bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, + ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion) +{ + LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + "Check scheme tx, proposed scheme version# " << proposedSchemaVersion << + " current version# " << currentSchemaVersion << + " expected version# " << expectedSchemaVersion << + " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId()); + + // Allow scheme tx if proposed or current schema version is zero. This simplify migration a lot. + if (proposedSchemaVersion && currentSchemaVersion && expectedSchemaVersion != proposedSchemaVersion) { + TString err = TStringBuilder() + << "Wrong schema version: proposed# " << proposedSchemaVersion << + " current version# " << currentSchemaVersion << + " expected version# " << expectedSchemaVersion << + " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId(); + + LOG_CRIT_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, err); + return false; + } + + return true; +} + bool TCheckSchemeTxUnit::HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker) { if (!checker(&Pipeline)) { return false; @@ -316,26 +316,26 @@ bool TCheckSchemeTxUnit::CheckSchemeTx(TActiveTransaction *activeTx) { const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - bool res = false; + bool res = false; switch (activeTx->GetSchemeTxType()) { case TSchemaOperation::ETypeCreate: - res = CheckCreate(activeTx); - break; + res = CheckCreate(activeTx); + break; case TSchemaOperation::ETypeDrop: - res = CheckDrop(activeTx); - break; + res = CheckDrop(activeTx); + break; case TSchemaOperation::ETypeAlter: - res = CheckAlter(activeTx); - break; + res = CheckAlter(activeTx); + break; case TSchemaOperation::ETypeBackup: - res = CheckBackup(activeTx); - break; + res = CheckBackup(activeTx); + break; case TSchemaOperation::ETypeRestore: res = CheckRestore(activeTx); break; case TSchemaOperation::ETypeCopy: - res = CheckCopy(activeTx); - break; + res = CheckCopy(activeTx); + break; case TSchemaOperation::ETypeCreatePersistentSnapshot: res = CheckCreatePersistentSnapshot(activeTx); break; @@ -371,7 +371,7 @@ bool TCheckSchemeTxUnit::CheckSchemeTx(TActiveTransaction *activeTx) BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); } - return res; + return res; } bool TCheckSchemeTxUnit::CheckCreate(TActiveTransaction *activeTx) { @@ -430,65 +430,65 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) } const auto &alter = tx.GetAlterTable(); - const ui64 proposedSchemaVersion = alter.HasTableSchemaVersion() ? alter.GetTableSchemaVersion() : 0; - - if (alter.HasPartitionConfig() && alter.GetPartitionConfig().HasFreezeState()) { - if (alter.ColumnsSize() || alter.DropColumnsSize()) { - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring alter, combine freeze with other actions is forbiden, tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() << " currentTxId " - << Pipeline.CurrentSchemaTxId()); - BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); - return false; - } - + const ui64 proposedSchemaVersion = alter.HasTableSchemaVersion() ? alter.GetTableSchemaVersion() : 0; + + if (alter.HasPartitionConfig() && alter.GetPartitionConfig().HasFreezeState()) { + if (alter.ColumnsSize() || alter.DropColumnsSize()) { + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + "Ignoring alter, combine freeze with other actions is forbiden, tablet " << DataShard.TabletID() + << " txId " << activeTx->GetTxId() << " currentTxId " + << Pipeline.CurrentSchemaTxId()); + BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); + return false; + } + if (DataShard.IsFollower()) { - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, "Ignoring alter, attempt to freeze follower, tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() << " currentTxId " - << Pipeline.CurrentSchemaTxId()); - BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); - return false; - } - + << " txId " << activeTx->GetTxId() << " currentTxId " + << Pipeline.CurrentSchemaTxId()); + BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); + return false; + } + bool freeze = alter.GetPartitionConfig().GetFreezeState() == NKikimrSchemeOp::EFreezeState::Freeze; - const auto curState = DataShard.GetState(); - bool err = false; - - if (freeze) { + const auto curState = DataShard.GetState(); + bool err = false; + + if (freeze) { if (curState != NDataShard::TShardState::Ready) { - err = true; - } - } else { + err = true; + } + } else { if (curState != NDataShard::TShardState::Frozen) { - err = true; - } - } - if (err) { - const auto& cmdStr = freeze ? TString("freeze") : TString("unfreeze"); - TString errText = TStringBuilder() << "Ignoring alter, transaction wants to " - << cmdStr << " datashard but current state is " - << DatashardStateName(curState) << " tablet "<< DataShard.TabletID() - << " txId " << activeTx->GetTxId() << " currentTxId " - << Pipeline.CurrentSchemaTxId(); - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, errText); - BuildResult(activeTx)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, errText); - return false; - } - } - + err = true; + } + } + if (err) { + const auto& cmdStr = freeze ? TString("freeze") : TString("unfreeze"); + TString errText = TStringBuilder() << "Ignoring alter, transaction wants to " + << cmdStr << " datashard but current state is " + << DatashardStateName(curState) << " tablet "<< DataShard.TabletID() + << " txId " << activeTx->GetTxId() << " currentTxId " + << Pipeline.CurrentSchemaTxId(); + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, errText); + BuildResult(activeTx)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, errText); + return false; + } + } + ui64 tableId = alter.GetId_Deprecated(); if (alter.HasPathId()) { Y_VERIFY(DataShard.GetPathOwnerId() == alter.GetPathId().GetOwnerId()); tableId = alter.GetPathId().GetLocalId(); } - const auto tablePtr = DataShard.GetUserTables().FindPtr(tableId); + const auto tablePtr = DataShard.GetUserTables().FindPtr(tableId); Y_VERIFY_S(tablePtr, "tableId: " << tableId); - const TUserTable &table = **tablePtr; - - auto curSchemaVersion = table.GetTableSchemaVersion(); - + const TUserTable &table = **tablePtr; + + auto curSchemaVersion = table.GetTableSchemaVersion(); + for (const auto &col : alter.GetColumns()) { Y_VERIFY(col.HasId()); Y_VERIFY(col.HasTypeId()); @@ -510,8 +510,8 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) Y_VERIFY(userColumn->Name == col.GetName()); } - auto res = CheckSchemaVersion(activeTx, proposedSchemaVersion, curSchemaVersion, curSchemaVersion + 1); - Y_VERIFY_DEBUG(res, "Unexpected schema version mutation"); + auto res = CheckSchemaVersion(activeTx, proposedSchemaVersion, curSchemaVersion, curSchemaVersion + 1); + Y_VERIFY_DEBUG(res, "Unexpected schema version mutation"); return true; } diff --git a/ydb/core/tx/datashard/create_table_unit.cpp b/ydb/core/tx/datashard/create_table_unit.cpp index edc5215295..85f38d456a 100644 --- a/ydb/core/tx/datashard/create_table_unit.cpp +++ b/ydb/core/tx/datashard/create_table_unit.cpp @@ -44,24 +44,24 @@ EExecutionStatus TCreateTableUnit::Execute(TOperation::TPtr op, TActiveTransaction *tx = dynamic_cast<TActiveTransaction*>(op.Get()); Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - const auto &schemeTx = tx->GetSchemeTx(); + const auto &schemeTx = tx->GetSchemeTx(); if (!schemeTx.HasCreateTable()) return EExecutionStatus::Executed; - const auto &createTableTx = schemeTx.GetCreateTable(); - + const auto &createTableTx = schemeTx.GetCreateTable(); + TPathId tableId(DataShard.GetPathOwnerId(), createTableTx.GetId_Deprecated()); if (createTableTx.HasPathId()) { Y_VERIFY(DataShard.GetPathOwnerId() == createTableTx.GetPathId().GetOwnerId()); tableId.LocalPathId = createTableTx.GetPathId().GetLocalId(); } - - const ui64 schemaVersion = createTableTx.HasTableSchemaVersion() ? createTableTx.GetTableSchemaVersion() : 0u; - + + const ui64 schemaVersion = createTableTx.HasTableSchemaVersion() ? createTableTx.GetTableSchemaVersion() : 0u; + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Trying to CREATE TABLE at " << DataShard.TabletID() - << " tableId# " << tableId - << " schema version# " << schemaVersion); + "Trying to CREATE TABLE at " << DataShard.TabletID() + << " tableId# " << tableId + << " schema version# " << schemaVersion); TUserTable::TPtr info = DataShard.CreateUserTable(txc, schemeTx.GetCreateTable()); DataShard.AddUserTable(tableId, info); diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp index fbb7b0ba0a..d86568b4a1 100644 --- a/ydb/core/tx/datashard/datashard.cpp +++ b/ydb/core/tx/datashard/datashard.cpp @@ -653,10 +653,10 @@ void TDataShard::NotifySchemeshard(const TActorContext& ctx, ui64 txId) { result->SetExplain(op->Error); result->SetBytesProcessed(op->BytesProcessed); result->SetRowsProcessed(op->RowsProcessed); - break; - } - default: - break; + break; + } + default: + break; } SendViaSchemeshardPipe(ctx, op->TabletId, THolder(event.Release())); @@ -937,15 +937,15 @@ TUserTable::TPtr TDataShard::AlterUserTable(const TActorContext& ctx, TTransacti if (configDelta.HasFreezeState()) { auto cmd = configDelta.GetFreezeState(); State = cmd == NKikimrSchemeOp::EFreezeState::Freeze ? TShardState::Frozen : TShardState::Ready; - PersistSys(db, Schema::Sys_State, State); - } - + PersistSys(db, Schema::Sys_State, State); + } + if (configDelta.HasTxReadSizeLimit()) { config.SetTxReadSizeLimit(configDelta.GetTxReadSizeLimit()); TxReadSizeLimit = configDelta.GetTxReadSizeLimit(); PersistSys(db, Schema::Sys_TxReadSizeLimit, TxReadSizeLimit); } - + if (configDelta.HasDisableStatisticsCalculation()) { StatisticsDisabled = configDelta.GetDisableStatisticsCalculation() ? 1 : 0; PersistSys(db, Schema::Sys_StatisticsDisabled, StatisticsDisabled); @@ -1732,11 +1732,11 @@ void TDataShard::Handle(TEvPrivate::TEvScanStats::TPtr& ev, const TActorContext } void TDataShard::Handle(TEvPrivate::TEvPersistScanState::TPtr& ev, const TActorContext &ctx) { - TabletCounters->Cumulative()[COUNTER_SCANNED_ROWS].Increment(ev->Get()->Rows); - TabletCounters->Cumulative()[COUNTER_SCANNED_BYTES].Increment(ev->Get()->Bytes); - Execute(new TTxStoreScanState(this, ev), ctx); -} - + TabletCounters->Cumulative()[COUNTER_SCANNED_ROWS].Increment(ev->Get()->Rows); + TabletCounters->Cumulative()[COUNTER_SCANNED_BYTES].Increment(ev->Get()->Bytes); + Execute(new TTxStoreScanState(this, ev), ctx); +} + void TDataShard::Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { Y_VERIFY(ev->Get()->Leader, "Unexpectedly connected to follower of tablet %" PRIu64, ev->Get()->TabletId); diff --git a/ydb/core/tx/datashard/datashard__build_index.cpp b/ydb/core/tx/datashard/datashard__build_index.cpp index f6712b32e7..5aebf94dc5 100644 --- a/ydb/core/tx/datashard/datashard__build_index.cpp +++ b/ydb/core/tx/datashard/datashard__build_index.cpp @@ -47,38 +47,38 @@ static TColumnsTypes GetAllTypes(const TUserTable::TCPtr tableInfo) { return result; } -static TTags BuildTags(const TColumnsTags& allTags, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { +static TTags BuildTags(const TColumnsTags& allTags, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { TTags result; - result.reserve(indexColumns.size()); + result.reserve(indexColumns.size()); - for (const auto& colName: indexColumns) { + for (const auto& colName: indexColumns) { + result.push_back(allTags.at(colName)); + } + + for (const auto& colName: dataColumns) { result.push_back(allTags.at(colName)); } - for (const auto& colName: dataColumns) { - result.push_back(allTags.at(colName)); - } - return result; } -static std::shared_ptr<TTypes> BuildTypes(const TColumnsTypes& types, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - auto result = std::make_shared<TTypes>(); - result->reserve(indexColumns.size()); +static std::shared_ptr<TTypes> BuildTypes(const TColumnsTypes& types, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + auto result = std::make_shared<TTypes>(); + result->reserve(indexColumns.size()); + + for (const auto& colName: indexColumns) { + Ydb::Type type; + type.set_type_id(static_cast<Ydb::Type_PrimitiveTypeId>(types.at(colName))); + result->emplace_back(colName, type); + } - for (const auto& colName: indexColumns) { + for (const auto& colName: dataColumns) { Ydb::Type type; type.set_type_id(static_cast<Ydb::Type_PrimitiveTypeId>(types.at(colName))); - result->emplace_back(colName, type); + result->emplace_back(colName, type); } - for (const auto& colName: dataColumns) { - Ydb::Type type; - type.set_type_id(static_cast<Ydb::Type_PrimitiveTypeId>(types.at(colName))); - result->emplace_back(colName, type); - } - return result; } @@ -108,30 +108,30 @@ struct TStatus { }; struct TUploadLimits { - ui64 BatchRowsLimit = 500; - ui64 BatchBytesLimit = 1u << 23; // 8MB + ui64 BatchRowsLimit = 500; + ui64 BatchBytesLimit = 1u << 23; // 8MB ui32 MaxUploadRowsRetryCount = 50; - ui32 BackoffCeiling = 3; + ui32 BackoffCeiling = 3; TDuration GetTimeoutBackouff(ui32 retryNo) const { return TDuration::Seconds(1u << Max(retryNo, BackoffCeiling)); } }; -class TBufferData : public IStatHolder, public TNonCopyable { -public: - TBufferData() - : Rows(new TRows) - { } +class TBufferData : public IStatHolder, public TNonCopyable { +public: + TBufferData() + : Rows(new TRows) + { } ui64 GetRows() const override final { - return Rows->size(); + return Rows->size(); + } + + std::shared_ptr<TRows> GetRowsData() const { + return Rows; } - std::shared_ptr<TRows> GetRowsData() const { - return Rows; - } - ui64 GetBytes() const override final { return ByteSize; } @@ -141,10 +141,10 @@ public: return; } - Y_VERIFY(other.Rows); + Y_VERIFY(other.Rows); Y_VERIFY(other.IsEmpty()); - other.Rows.swap(Rows); + other.Rows.swap(Rows); other.ByteSize = ByteSize; other.LastKey = std::move(LastKey); @@ -152,37 +152,37 @@ public: } void Clear() { - Rows->clear(); + Rows->clear(); ByteSize = 0; LastKey = {}; } - void AddRow(TSerializedCellVec&& key, TSerializedCellVec&& targetPk, TString&& targetValue) { - Rows->emplace_back(std::move(targetPk), std::move(targetValue)); - ByteSize += Rows->back().first.GetBuffer().size() + Rows->back().second.size(); + void AddRow(TSerializedCellVec&& key, TSerializedCellVec&& targetPk, TString&& targetValue) { + Rows->emplace_back(std::move(targetPk), std::move(targetValue)); + ByteSize += Rows->back().first.GetBuffer().size() + Rows->back().second.size(); LastKey = std::move(key); } bool IsEmpty() const { - return Rows->empty(); + return Rows->empty(); } bool IsReachLimits(const TUploadLimits& Limits) { - return Rows->size() >= Limits.BatchRowsLimit || ByteSize > Limits.BatchBytesLimit; - } - - void ExtractLastKey(TSerializedCellVec& out) { - out = std::move(LastKey); - } - - const TSerializedCellVec& GetLastKey() const { - return LastKey; - } - -private: - std::shared_ptr<TRows> Rows; - ui64 ByteSize = 0; - TSerializedCellVec LastKey; + return Rows->size() >= Limits.BatchRowsLimit || ByteSize > Limits.BatchBytesLimit; + } + + void ExtractLastKey(TSerializedCellVec& out) { + out = std::move(LastKey); + } + + const TSerializedCellVec& GetLastKey() const { + return LastKey; + } + +private: + std::shared_ptr<TRows> Rows; + ui64 ByteSize = 0; + TSerializedCellVec LastKey; }; class TBuildIndexScan : public TActor<TBuildIndexScan>, public NTable::IScan { @@ -196,9 +196,9 @@ class TBuildIndexScan : public TActor<TBuildIndexScan>, public NTable::IScan { const TActorId DatashardActorId; const TActorId SchemeShardActorID; - const TTags ScanTags; // first: columns we scan, order as in IndexTable - const std::shared_ptr<TTypes> UploadColumnsTypes; // columns types we upload to indexTable - const ui32 TargetDataColumnPos; // positon of first data column in target table + const TTags ScanTags; // first: columns we scan, order as in IndexTable + const std::shared_ptr<TTypes> UploadColumnsTypes; // columns types we upload to indexTable + const ui32 TargetDataColumnPos; // positon of first data column in target table const TTags KeyColumnIds; const TVector<NScheme::TTypeId> KeyTypes; @@ -230,8 +230,8 @@ public: const TActorId& datashardActorId, const TActorId& schemeshardActorId, const TSerializedTableRange& range, - const TVector<TString> targetIndexColumns, - const TVector<TString> targetDataColumns, + const TVector<TString> targetIndexColumns, + const TVector<TString> targetDataColumns, TUserTable::TCPtr tableInfo, TUploadLimits limits) : TActor(&TThis::StateWork) @@ -242,9 +242,9 @@ public: , DataShardId(dataShardId) , DatashardActorId(datashardActorId) , SchemeShardActorID(schemeshardActorId) - , ScanTags(BuildTags(GetAllTags(tableInfo), targetIndexColumns, targetDataColumns)) - , UploadColumnsTypes(BuildTypes(GetAllTypes(tableInfo), targetIndexColumns, targetDataColumns)) - , TargetDataColumnPos(targetIndexColumns.size()) + , ScanTags(BuildTags(GetAllTags(tableInfo), targetIndexColumns, targetDataColumns)) + , UploadColumnsTypes(BuildTypes(GetAllTypes(tableInfo), targetIndexColumns, targetDataColumns)) + , TargetDataColumnPos(targetIndexColumns.size()) , KeyColumnIds(tableInfo->KeyColumnIds) , KeyTypes(tableInfo->KeyColumnTypes) , TableRange(tableInfo->Range) @@ -255,7 +255,7 @@ public: ~TBuildIndexScan() override = default; THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme>) noexcept override { - auto selfActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this); + auto selfActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this); auto ctx = TActivationContext::AsActorContext().MakeFor(selfActorId); LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, @@ -307,16 +307,16 @@ public: EScan Feed(TArrayRef<const TCell> key, const TRow& row) noexcept override { auto ctx = TActivationContext::AsActorContext().MakeFor(SelfId()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Feed key " << DebugPrintPoint(KeyTypes, key, *AppData()->TypeRegistry) << " " << Debug()); - const TConstArrayRef<TCell> rowCells = *row; + const TConstArrayRef<TCell> rowCells = *row; + + ReadBuf.AddRow(TSerializedCellVec(TSerializedCellVec::Serialize(key)), + TSerializedCellVec(TSerializedCellVec::Serialize(rowCells.Slice(0, TargetDataColumnPos))), + TSerializedCellVec::Serialize(rowCells.Slice(TargetDataColumnPos))); - ReadBuf.AddRow(TSerializedCellVec(TSerializedCellVec::Serialize(key)), - TSerializedCellVec(TSerializedCellVec::Serialize(rowCells.Slice(0, TargetDataColumnPos))), - TSerializedCellVec::Serialize(rowCells.Slice(TargetDataColumnPos))); - if (!ReadBuf.IsReachLimits(Limits)) { return EScan::Feed; } @@ -394,18 +394,18 @@ public: } EScan PageFault() noexcept override { - auto ctx = TActivationContext::AsActorContext().MakeFor(SelfId()); - - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Page fault" - << " ReadBuf empty: " << ReadBuf.IsEmpty() - << " WriteBuf empty: " << WriteBuf.IsEmpty() - << " " << Debug()); - - if (ReadBuf.IsEmpty()) { - return EScan::Feed; - } - + auto ctx = TActivationContext::AsActorContext().MakeFor(SelfId()); + + LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, + "Page fault" + << " ReadBuf empty: " << ReadBuf.IsEmpty() + << " WriteBuf empty: " << WriteBuf.IsEmpty() + << " " << Debug()); + + if (ReadBuf.IsEmpty()) { + return EScan::Feed; + } + if (WriteBuf.IsEmpty()) { ReadBuf.FlushTo(WriteBuf); Upload(); @@ -457,7 +457,7 @@ private: if (UploadStatus.IsSuccess()) { Stats.Aggr(&WriteBuf); - WriteBuf.ExtractLastKey(LastUploadedKey); + WriteBuf.ExtractLastKey(LastUploadedKey); //send progress TAutoPtr<TEvDataShard::TEvBuildIndexProgressResponse> progress @@ -513,13 +513,13 @@ private: auto ctx = TActivationContext::AsActorContext().MakeFor(SelfId()); LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Upload, last key " << DebugPrintPoint(KeyTypes, WriteBuf.GetLastKey().GetCells(), *AppData()->TypeRegistry) + "Upload, last key " << DebugPrintPoint(KeyTypes, WriteBuf.GetLastKey().GetCells(), *AppData()->TypeRegistry) << " " << Debug()); auto actor = NTxProxy::CreateUploadRowsInternal( SelfId(), TargetTable, - UploadColumnsTypes, - WriteBuf.GetRowsData(), + UploadColumnsTypes, + WriteBuf.GetRowsData(), NTxProxy::EUploadRowsMode::WriteToTableShadow, true /*writeToPrivateTable*/); @@ -535,13 +535,13 @@ TAutoPtr<NTable::IScan> CreateBuildIndexScan( const TActorId& datashardActorId, const TActorId& schemeshardActorId, const TSerializedTableRange& range, - const TVector<TString>& targetIndexColumns, - const TVector<TString>& targetDataColumns, + const TVector<TString>& targetIndexColumns, + const TVector<TString>& targetDataColumns, TUserTable::TCPtr tableInfo, TUploadLimits limits) { return new TBuildIndexScan( - buildIndexId, target, seqNo, dataShardId, datashardActorId, schemeshardActorId, range, targetIndexColumns, targetDataColumns, tableInfo, limits); + buildIndexId, target, seqNo, dataShardId, datashardActorId, schemeshardActorId, range, targetIndexColumns, targetDataColumns, tableInfo, limits); } @@ -609,8 +609,8 @@ void TDataShard::Handle(TEvDataShard::TEvBuildIndexCreateRequest::TPtr& ev, cons return; } - const TVector<TString> targetIndexColumns(record.GetIndexColumns().begin(), record.GetIndexColumns().end()); - const TVector<TString> targetDataColumns(record.GetDataColumns().begin(), record.GetDataColumns().end()); + const TVector<TString> targetIndexColumns(record.GetIndexColumns().begin(), record.GetIndexColumns().end()); + const TVector<TString> targetDataColumns(record.GetDataColumns().begin(), record.GetDataColumns().end()); if (!record.HasSnapshotStep() || !record.HasSnapshotTxId()) { badRequest(TStringBuilder() << " request doesn't have Shapshot Step or TxId"); @@ -654,7 +654,7 @@ void TDataShard::Handle(TEvDataShard::TEvBuildIndexCreateRequest::TPtr& ev, cons ev->Sender, requestedRange, targetIndexColumns, - targetDataColumns, + targetDataColumns, userTable, limits), ev->Cookie, diff --git a/ydb/core/tx/datashard/datashard__cleanup_borrowed.cpp b/ydb/core/tx/datashard/datashard__cleanup_borrowed.cpp index 3d057ef7a6..c53faa1c60 100644 --- a/ydb/core/tx/datashard/datashard__cleanup_borrowed.cpp +++ b/ydb/core/tx/datashard/datashard__cleanup_borrowed.cpp @@ -39,7 +39,7 @@ public: for (const ui64 tabletId : kv.second) { auto res = WaitingTablets.insert(tabletId); if (res.second) { - ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, config)); + ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, config)); ++TotalTablets; } ++TotalBorrows; @@ -235,7 +235,7 @@ public: return; } - auto actorId = ctx.RegisterWithSameMailbox( + auto actorId = ctx.RegisterWithSameMailbox( new TTxMonitoringCleanupBorrowedPartsActor( ctx.SelfID, Ev->Sender, diff --git a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp index 10f255a6b2..026b14e461 100644 --- a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp @@ -236,7 +236,7 @@ public: } IScan::THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept override { - TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); + TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); Driver = driver; Scheme = std::move(scheme); diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index f154fe4577..876c337093 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -283,7 +283,7 @@ public: TDataShardEngineHost(TDataShard* self, NTable::TDatabase& db, TEngineHostCounters& counters, ui64& lockTxId, TInstant now) : TEngineHost(db, counters, TEngineHostSettings(self->TabletID(), - (self->State == TShardState::Readonly || self->State == TShardState::Frozen), + (self->State == TShardState::Readonly || self->State == TShardState::Frozen), self->ByKeyFilterDisabled(), self->GetKeyAccessSampler())) , Self(self) @@ -375,7 +375,7 @@ public: NUdf::TUnboxedValue SelectRange(const TTableId& tableId, const TTableRange& range, TStructLiteral* columnIds, TListLiteral* skipNullKeys, TStructType* returnType, const TReadTarget& readTarget, ui64 itemsLimit, ui64 bytesLimit, bool reverse, - std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) override + std::pair<const TListLiteral*, const TListLiteral*> forbidNullArgs, const THolderFactory& holderFactory) override { Y_VERIFY(!TSysTables::IsSystemTable(tableId), "SelectRange no system table is not supported"); @@ -383,7 +383,7 @@ public: Self->SetTableAccessTime(tableId, Now); return TEngineHost::SelectRange(tableId, range, columnIds, skipNullKeys, returnType, readTarget, - itemsLimit, bytesLimit, reverse, forbidNullArgs, holderFactory); + itemsLimit, bytesLimit, reverse, forbidNullArgs, holderFactory); } void UpdateRow(const TTableId& tableId, const TArrayRef<const TCell>& row, const TArrayRef<const TUpdateCommand>& commands) override { @@ -476,21 +476,21 @@ public: return Self->GetLocalTableId(tableId); } - ui64 GetTableSchemaVersion(const TTableId& tableId) const override { - if (TSysTables::IsSystemTable(tableId)) - return 0; - const auto& userTables = Self->GetUserTables(); - auto it = userTables.find(tableId.PathId.LocalPathId); - if (it == userTables.end()) { - Y_FAIL_S("DatshardEngineHost (tablet id: " << Self->TabletID() - << " state: " << Self->GetState() - << ") unables to find given table with id: " << tableId); - return 0; - } else { - return it->second->GetTableSchemaVersion(); - } - } - + ui64 GetTableSchemaVersion(const TTableId& tableId) const override { + if (TSysTables::IsSystemTable(tableId)) + return 0; + const auto& userTables = Self->GetUserTables(); + auto it = userTables.find(tableId.PathId.LocalPathId); + if (it == userTables.end()) { + Y_FAIL_S("DatshardEngineHost (tablet id: " << Self->TabletID() + << " state: " << Self->GetState() + << ") unables to find given table with id: " << tableId); + return 0; + } else { + return it->second->GetTableSchemaVersion(); + } + } + private: const TDataShardSysTable& DataShardSysTable(const TTableId& tableId) const { return static_cast<const TDataShardSysTables *>(Self->GetDataShardSysTables())->Get(tableId); diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp index a85cd60441..b6b6b7e337 100644 --- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp @@ -155,7 +155,7 @@ private: return; } - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Undelivered event: " << ev->GetTypeRewrite() + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Undelivered event: " << ev->GetTypeRewrite() << ", at: " << ScanActorId << ", tablet: " << DatashardActorId << ", scanId: " << ScanId << ", table: " << TablePath); @@ -217,10 +217,10 @@ private: YQL_ENSURE(seq == CurrentRange); if (CurrentRange == TableRanges.size()) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "TableRanges is over" - << ", at: " << ScanActorId << ", scanId: " << ScanId - << ", table: " << TablePath); + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, + "TableRanges is over" + << ", at: " << ScanActorId << ", scanId: " << ScanId + << ", table: " << TablePath); return EScan::Final; } diff --git a/ydb/core/tx/datashard/datashard__monitoring.cpp b/ydb/core/tx/datashard/datashard__monitoring.cpp index c34e163207..644f9a7419 100644 --- a/ydb/core/tx/datashard/datashard__monitoring.cpp +++ b/ydb/core/tx/datashard/datashard__monitoring.cpp @@ -77,7 +77,7 @@ public: rec.SetPath(pr.second->Path); rec.SetLocalId(pr.second->LocalTid); rec.SetPathId(pr.first); - rec.SetSchemaVersion(pr.second->GetTableSchemaVersion()); + rec.SetSchemaVersion(pr.second->GetTableSchemaVersion()); pr.second->GetSchema(*rec.MutableDescription()); if (pr.second->Stats.StatsUpdateTime) { diff --git a/ydb/core/tx/datashard/datashard__s3.cpp b/ydb/core/tx/datashard/datashard__s3.cpp index 74628192b6..2f7a74bbab 100644 --- a/ydb/core/tx/datashard/datashard__s3.cpp +++ b/ydb/core/tx/datashard/datashard__s3.cpp @@ -36,8 +36,8 @@ public: if (Self->State != TShardState::Ready && Self->State != TShardState::Readonly && - Self->State != TShardState::SplitSrcWaitForNoTxInFlight && - Self->State != TShardState::Frozen) + Self->State != TShardState::SplitSrcWaitForNoTxInFlight && + Self->State != TShardState::Frozen) { SetError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, Sprintf("Wrong shard state: %" PRIu32 " tablet id: %" PRIu64, Self->State, Self->TabletID())); diff --git a/ydb/core/tx/datashard/datashard__store_scan_state.cpp b/ydb/core/tx/datashard/datashard__store_scan_state.cpp index dd9ed417b9..0ffd149986 100644 --- a/ydb/core/tx/datashard/datashard__store_scan_state.cpp +++ b/ydb/core/tx/datashard/datashard__store_scan_state.cpp @@ -1,65 +1,65 @@ -#include "datashard_txs.h" - -namespace NKikimr { +#include "datashard_txs.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + TDataShard::TTxStoreScanState::TTxStoreScanState( TDataShard *ds, - TEvPrivate::TEvPersistScanState::TPtr ev) - : TBase(ds) - , Ev(ev) -{ -} - + TEvPrivate::TEvPersistScanState::TPtr ev) + : TBase(ds) + , Ev(ev) +{ +} + bool TDataShard::TTxStoreScanState::Execute(TTransactionContext &txc, - const TActorContext &ctx) -{ - const TEvPrivate::TEvPersistScanState* event = Ev->Get(); - ui64 txId = event->TxId; - auto op = Self->Pipeline.FindOp(txId); - - if (!op) { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Cannot find op " << txId << " to persist scan state at " << - Self->TabletID()); - return false; - } - - auto schemaOp = Self->FindSchemaTx(txId); - if (!schemaOp) { - LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, - "Cannot find schema op " << txId << " to update scan state at " << - Self->TabletID()); - return false; - } - - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Persist scan progress for " << txId << - " key size " << event->LastKey.size() << - " status " << event->StatusCode << " at " << - Self->TabletID()); - - auto binaryIssues = SerializeIssues(event->Issues); - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::ScanProgress>().Key(txId).Update( - NIceDb::TUpdate<Schema::ScanProgress::LastKey>(event->LastKey), - NIceDb::TUpdate<Schema::ScanProgress::LastStatus>(event->StatusCode), - NIceDb::TUpdate<Schema::ScanProgress::LastIssues>(binaryIssues)); - - schemaOp->ScanState.StatusCode = event->StatusCode; - schemaOp->ScanState.Issues = event->Issues; - schemaOp->ScanState.LastKey = event->LastKey; - schemaOp->ScanState.Bytes = event->Bytes; - - return true; -} - + const TActorContext &ctx) +{ + const TEvPrivate::TEvPersistScanState* event = Ev->Get(); + ui64 txId = event->TxId; + auto op = Self->Pipeline.FindOp(txId); + + if (!op) { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, + "Cannot find op " << txId << " to persist scan state at " << + Self->TabletID()); + return false; + } + + auto schemaOp = Self->FindSchemaTx(txId); + if (!schemaOp) { + LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, + "Cannot find schema op " << txId << " to update scan state at " << + Self->TabletID()); + return false; + } + + LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, + "Persist scan progress for " << txId << + " key size " << event->LastKey.size() << + " status " << event->StatusCode << " at " << + Self->TabletID()); + + auto binaryIssues = SerializeIssues(event->Issues); + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::ScanProgress>().Key(txId).Update( + NIceDb::TUpdate<Schema::ScanProgress::LastKey>(event->LastKey), + NIceDb::TUpdate<Schema::ScanProgress::LastStatus>(event->StatusCode), + NIceDb::TUpdate<Schema::ScanProgress::LastIssues>(binaryIssues)); + + schemaOp->ScanState.StatusCode = event->StatusCode; + schemaOp->ScanState.Issues = event->Issues; + schemaOp->ScanState.LastKey = event->LastKey; + schemaOp->ScanState.Bytes = event->Bytes; + + return true; +} + void TDataShard::TTxStoreScanState::Complete(const TActorContext &ctx) -{ +{ ctx.Send(Ev->Sender, new TDataShard::TEvPrivate::TEvPersistScanStateAck()); -} - +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp index 8d5c583da9..0226bd42e2 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.cpp +++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp @@ -152,13 +152,13 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self, KqpSetTxLocksKeys(GetKqpTransaction().GetLocks(), self->SysLocksTable(), EngineBay); EngineBay.MarkTxLoaded(); - - auto& tasksRunner = GetKqpTasksRunner(); // prepare tasks runner, can throw TMemoryLimitExceededException - - auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - TxSize); - - auto execCtx = DefaultKqpExecutionContext(); - tasksRunner.Prepare(DefaultKqpDataReqMemoryLimits(), *execCtx); + + auto& tasksRunner = GetKqpTasksRunner(); // prepare tasks runner, can throw TMemoryLimitExceededException + + auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - TxSize); + + auto execCtx = DefaultKqpExecutionContext(); + tasksRunner.Prepare(DefaultKqpDataReqMemoryLimits(), *execCtx); } catch (const TMemoryLimitExceededException&) { LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Not enough memory to create tasks runner, datashard: " << TabletId() << ", txid: " << StepTxId_.TxId); diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index 6738c59b73..43abd525e1 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -27,13 +27,13 @@ class TSysLocks; struct TReadSetKey; class TActiveTransaction; -struct TScanState { - TString LastKey; - ui64 Bytes = 0; +struct TScanState { + TString LastKey; + ui64 Bytes = 0; Ydb::StatusIds::StatusCode StatusCode = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; - NYql::TIssues Issues; -}; - + NYql::TIssues Issues; +}; + struct TSchemaOperation { enum EType : ui32 { ETypeDrop = 0, @@ -66,13 +66,13 @@ struct TSchemaOperation { bool ReadOnly; bool Done; - bool Success; + bool Success; TString Error; ui64 BytesProcessed; ui64 RowsProcessed; - TScanState ScanState; - + TScanState ScanState; + TSchemaOperation(ui64 txId, EType type, TActorId source, ui64 tabletId, ui64 minStep, ui64 maxStep, ui64 planStep, bool readOnly, bool success, const TString& error, ui64 bytes, ui64 rows) diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 5522fdd4c0..11313474f8 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -42,7 +42,7 @@ #include <ydb/core/protos/counters_datashard.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - + #include <library/cpp/actors/interconnect/interconnect.h> #include <util/string/join.h> @@ -183,7 +183,7 @@ class TDataShard class TTxGetInfo; class TTxListOperations; class TTxGetOperation; - class TTxStoreScanState; + class TTxStoreScanState; class TTxRefreshVolatileSnapshot; class TTxDiscardVolatileSnapshot; class TTxCleanupRemovedSnapshots; @@ -289,8 +289,8 @@ class TDataShard EvRegisterScanActor, EvNodeDisconnected, EvScanStats, - EvPersistScanState, - EvPersistScanStateAck, + EvPersistScanState, + EvPersistScanStateAck, EvConditionalEraseRowsRegistered, EvAsyncJobComplete, EvSubDomainPathIdFound, @@ -358,32 +358,32 @@ class TDataShard ui64 Rows; ui64 Bytes; }; - - // Also updates scan statistic, i.e. like TEvScanStats but persist state for given tx - struct TEvPersistScanState : public TEventLocal<TEvPersistScanState, EvPersistScanState> { - TEvPersistScanState( - ui64 txId, - TString lastKey, - ui64 rows, - ui64 bytes, - Ydb::StatusIds::StatusCode statusCode, - const NYql::TIssues& issues) - : TxId(txId) - , LastKey(lastKey) - , Rows(rows) - , Bytes(bytes) - , StatusCode(statusCode) - , Issues(issues) - {} - ui64 TxId; - TString LastKey; - ui64 Rows; - ui64 Bytes; - Ydb::StatusIds::StatusCode StatusCode; - NYql::TIssues Issues; - }; - - struct TEvPersistScanStateAck : public TEventLocal<TEvPersistScanStateAck, EvPersistScanStateAck> {}; + + // Also updates scan statistic, i.e. like TEvScanStats but persist state for given tx + struct TEvPersistScanState : public TEventLocal<TEvPersistScanState, EvPersistScanState> { + TEvPersistScanState( + ui64 txId, + TString lastKey, + ui64 rows, + ui64 bytes, + Ydb::StatusIds::StatusCode statusCode, + const NYql::TIssues& issues) + : TxId(txId) + , LastKey(lastKey) + , Rows(rows) + , Bytes(bytes) + , StatusCode(statusCode) + , Issues(issues) + {} + ui64 TxId; + TString LastKey; + ui64 Rows; + ui64 Bytes; + Ydb::StatusIds::StatusCode StatusCode; + NYql::TIssues Issues; + }; + + struct TEvPersistScanStateAck : public TEventLocal<TEvPersistScanStateAck, EvPersistScanStateAck> {}; struct TEvConditionalEraseRowsRegistered : public TEventLocal<TEvConditionalEraseRowsRegistered, EvConditionalEraseRowsRegistered> { explicit TEvConditionalEraseRowsRegistered(ui64 txId, const TActorId& actorId) @@ -579,17 +579,17 @@ class TDataShard using TColumns = TableColumns<TxId, Flags, Locks>; }; - struct ScanProgress : Table<13> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct LastKey : Column<2, NScheme::NTypeIds::String> {}; - struct LastBytes : Column<3, NScheme::NTypeIds::Uint64> {}; - struct LastStatus : Column<4, NScheme::NTypeIds::Uint64> {}; - struct LastIssues : Column<5, NScheme::NTypeIds::String> { using Type = TString; }; - - using TKey = TableKey<TxId>; - using TColumns = TableColumns<TxId, LastKey, LastBytes, LastStatus, LastIssues>; - }; - + struct ScanProgress : Table<13> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct LastKey : Column<2, NScheme::NTypeIds::String> {}; + struct LastBytes : Column<3, NScheme::NTypeIds::Uint64> {}; + struct LastStatus : Column<4, NScheme::NTypeIds::Uint64> {}; + struct LastIssues : Column<5, NScheme::NTypeIds::String> { using Type = TString; }; + + using TKey = TableKey<TxId>; + using TColumns = TableColumns<TxId, LastKey, LastBytes, LastStatus, LastIssues>; + }; + struct Snapshots : Table<14> { struct Oid : Column<1, NScheme::NTypeIds::Uint64> {}; // PathOwnerId struct Tid : Column<2, NScheme::NTypeIds::Uint64> {}; // LocalPathId @@ -898,7 +898,7 @@ class TDataShard void Handle(TEvPrivate::TEvRemoveOldInReadSets::TPtr &ev, const TActorContext &ctx); void Handle(TEvPrivate::TEvRegisterScanActor::TPtr &ev, const TActorContext &ctx); void Handle(TEvPrivate::TEvScanStats::TPtr &ev, const TActorContext &ctx); - void Handle(TEvPrivate::TEvPersistScanState::TPtr &ev, const TActorContext &ctx); + void Handle(TEvPrivate::TEvPersistScanState::TPtr &ev, const TActorContext &ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx); void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx); void Handle(TEvTabletPipe::TEvServerConnected::TPtr &ev, const TActorContext &ctx); @@ -1115,14 +1115,14 @@ public: return State == TShardState::Ready || State == TShardState::Readonly || State == TShardState::WaitScheme || - State == TShardState::SplitSrcWaitForNoTxInFlight || - State == TShardState::Frozen; + State == TShardState::SplitSrcWaitForNoTxInFlight || + State == TShardState::Frozen; + } + + bool IsStateFrozen() const { + return State == TShardState::Frozen; } - bool IsStateFrozen() const { - return State == TShardState::Frozen; - } - ui32 Generation() const { return Executor()->Generation(); } bool IsFollower() const { return Executor()->GetStats().IsFollower; } @@ -2159,7 +2159,7 @@ protected: HFuncTraced(TEvPrivate::TEvRemoveOldInReadSets, Handle); HFuncTraced(TEvPrivate::TEvRegisterScanActor, Handle); HFuncTraced(TEvPrivate::TEvScanStats, Handle); - HFuncTraced(TEvPrivate::TEvPersistScanState, Handle); + HFuncTraced(TEvPrivate::TEvPersistScanState, Handle); HFuncTraced(TEvTabletPipe::TEvClientConnected, Handle); HFuncTraced(TEvTabletPipe::TEvClientDestroyed, Handle); HFuncTraced(TEvTabletPipe::TEvServerConnected, Handle); diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp index 8892e9942e..edd6ca8e19 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp @@ -101,24 +101,24 @@ ui64 TKqpDatashardComputeContext::GetLocalTableId(const TTableId &tableId) const TVector<std::pair<NScheme::TTypeId, TString>> TKqpDatashardComputeContext::GetKeyColumnsInfo( const TTableId &tableId) const { - MKQL_ENSURE_S(Shard); - const NDataShard::TUserTable::TCPtr* tablePtr = Shard->GetUserTables().FindPtr(tableId.PathId.LocalPathId); - MKQL_ENSURE_S(tablePtr); - const NDataShard::TUserTable::TCPtr table = *tablePtr; - MKQL_ENSURE_S(table); - - TVector<std::pair<NScheme::TTypeId, TString>> res; - res.reserve(table->KeyColumnTypes.size()); - - for (size_t i = 0 ; i < table->KeyColumnTypes.size(); i++) { - auto col = table->Columns.at(table->KeyColumnIds[i]); - MKQL_ENSURE_S(col.IsKey); - MKQL_ENSURE_S(table->KeyColumnTypes[i] == col.Type); - res.push_back({table->KeyColumnTypes[i], col.Name}); - } - return res; -} - + MKQL_ENSURE_S(Shard); + const NDataShard::TUserTable::TCPtr* tablePtr = Shard->GetUserTables().FindPtr(tableId.PathId.LocalPathId); + MKQL_ENSURE_S(tablePtr); + const NDataShard::TUserTable::TCPtr table = *tablePtr; + MKQL_ENSURE_S(table); + + TVector<std::pair<NScheme::TTypeId, TString>> res; + res.reserve(table->KeyColumnTypes.size()); + + for (size_t i = 0 ; i < table->KeyColumnTypes.size(); i++) { + auto col = table->Columns.at(table->KeyColumnIds[i]); + MKQL_ENSURE_S(col.IsKey); + MKQL_ENSURE_S(table->KeyColumnTypes[i] == col.Type); + res.push_back({table->KeyColumnTypes[i], col.Name}); + } + return res; +} + THashMap<TString, NScheme::TTypeId> TKqpDatashardComputeContext::GetKeyColumnsMap(const TTableId &tableId) const { THashMap<TString, NScheme::TTypeId> columnsMap; diff --git a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp index a7154ae490..6a6c3d8231 100644 --- a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp @@ -37,20 +37,20 @@ public: void Apply(NUdf::IApplyContext& applyContext) const override { auto& engineCtx = *CheckedCast<TKqpDatashardApplyContext*>(&applyContext); - TVector<TCell> keyTuple(Owner.KeyIndices.size()); - FillKeyTupleValue(Row, Owner.KeyIndices, Owner.RowTypes, keyTuple, Owner.Env); - + TVector<TCell> keyTuple(Owner.KeyIndices.size()); + FillKeyTupleValue(Row, Owner.KeyIndices, Owner.RowTypes, keyTuple, Owner.Env); + if (engineCtx.Host->IsPathErased(Owner.TableId)) { return; } - if (!engineCtx.Host->IsMyKey(Owner.TableId, keyTuple)) { + if (!engineCtx.Host->IsMyKey(Owner.TableId, keyTuple)) { return; } ui64 nEraseRow = Owner.ShardTableStats.NEraseRow; - engineCtx.Host->EraseRow(Owner.TableId, keyTuple); + engineCtx.Host->EraseRow(Owner.TableId, keyTuple); if (i64 delta = Owner.ShardTableStats.NEraseRow - nEraseRow; delta > 0) { Owner.TaskTableStats.NEraseRow += delta; @@ -94,12 +94,12 @@ public: public: TKqpDeleteRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx, - const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId> rowTypes, TVector<ui32> keyIndices, const TTypeEnvironment& env) + const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId> rowTypes, TVector<ui32> keyIndices, const TTypeEnvironment& env) : TBase(mutables) , TableId(tableId) , RowsNode(rowsNode) - , RowTypes(std::move(rowTypes)) - , KeyIndices(std::move(keyIndices)) + , RowTypes(std::move(rowTypes)) + , KeyIndices(std::move(keyIndices)) , Env(env) , ShardTableStats(computeCtx.GetDatashardCounters()) , TaskTableStats(computeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId())) {} @@ -112,8 +112,8 @@ private: private: TTableId TableId; IComputationNode* RowsNode; - const TVector<NUdf::TDataTypeId> RowTypes; - const TVector<ui32> KeyIndices; + const TVector<NUdf::TDataTypeId> RowTypes; + const TVector<ui32> KeyIndices; const TTypeEnvironment& Env; TKqpTableStats& ShardTableStats; TKqpTableStats& TaskTableStats; @@ -138,12 +138,12 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF MKQL_ENSURE_S(tableKeyTypes.size() == rowType->GetMembersCount(), "Table key column count mismatch" << ", expected: " << tableKeyTypes.size() << ", actual: " << rowType->GetMembersCount()); - - THashMap<TString, ui32> inputIndex; - TVector<NUdf::TDataTypeId> rowTypes(rowType->GetMembersCount()); - for (ui32 i = 0; i < rowType->GetMembersCount(); ++i) { - const auto& name = rowType->GetMemberName(i); - MKQL_ENSURE_S(inputIndex.emplace(TString(name), i).second); + + THashMap<TString, ui32> inputIndex; + TVector<NUdf::TDataTypeId> rowTypes(rowType->GetMembersCount()); + for (ui32 i = 0; i < rowType->GetMembersCount(); ++i) { + const auto& name = rowType->GetMemberName(i); + MKQL_ENSURE_S(inputIndex.emplace(TString(name), i).second); auto memberType = rowType->GetMemberType(i); auto typeId = memberType->IsOptional() @@ -151,22 +151,22 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF : AS_TYPE(TDataType, memberType)->GetSchemeType(); rowTypes[i] = typeId; - } + } - TVector<ui32> keyIndices(tableKeyTypes.size()); - for (ui32 i = 0; i < tableKeyTypes.size(); i++) { - auto it = inputIndex.find(tableKeyTypes[i].second); + TVector<ui32> keyIndices(tableKeyTypes.size()); + for (ui32 i = 0; i < tableKeyTypes.size(); i++) { + auto it = inputIndex.find(tableKeyTypes[i].second); MKQL_ENSURE_S(rowTypes[it->second] == tableKeyTypes[i].first, "Key type mismatch" << ", column: " << tableKeyTypes[i].second << ", expected: " << tableKeyTypes[i].first << ", actual: " << rowTypes[it->second]); - keyIndices[i] = it->second; - } - + keyIndices[i] = it->second; + } + return new TKqpDeleteRowsWrapper(ctx.Mutables, computeCtx, tableId, - LocateNode(ctx.NodeLocator, *rowsNode.GetNode()), std::move(rowTypes), std::move(keyIndices), ctx.Env); + LocateNode(ctx.NodeLocator, *rowsNode.GetNode()), std::move(rowTypes), std::move(keyIndices), ctx.Env); } } // namespace NMiniKQL diff --git a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp index c4f42cb8c6..a2dfa61bbf 100644 --- a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp @@ -43,7 +43,7 @@ public: void Apply(NUdf::IApplyContext& applyContext) const override { auto& dsApplyCtx = *CheckedCast<TKqpDatashardApplyContext*>(&applyContext); - TVector<TCell> keyTuple(Owner.KeyIndices.size()); + TVector<TCell> keyTuple(Owner.KeyIndices.size()); FillKeyTupleValue(Row, Owner.KeyIndices, Owner.RowTypes, keyTuple, Owner.Env); if (dsApplyCtx.Host->IsPathErased(Owner.TableId)) { @@ -160,11 +160,11 @@ private: IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpDatashardComputeContext& computeCtx) { - MKQL_ENSURE_S(callable.GetInputsCount() >= 3); + MKQL_ENSURE_S(callable.GetInputsCount() >= 3); auto tableNode = callable.GetInput(0); auto rowsNode = callable.GetInput(1); - auto upsertColumnsNode = callable.GetInput(2); + auto upsertColumnsNode = callable.GetInput(2); auto tableId = NKqp::ParseTableId(tableNode); auto localTableId = computeCtx.GetLocalTableId(tableId); @@ -176,23 +176,23 @@ IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeF auto rowType = AS_TYPE(TStructType, AS_TYPE(TStreamType, rowsNode.GetStaticType())->GetItemType()); - MKQL_ENSURE_S(tableKeyTypes.size() <= rowType->GetMembersCount(), "not enough columns in the runtime node"); - - THashMap<TString, ui32> inputIndex; + MKQL_ENSURE_S(tableKeyTypes.size() <= rowType->GetMembersCount(), "not enough columns in the runtime node"); + + THashMap<TString, ui32> inputIndex; TVector<NUdf::TDataTypeId> rowTypes(rowType->GetMembersCount()); for (ui32 i = 0; i < rowTypes.size(); ++i) { - const auto& name = rowType->GetMemberName(i); - MKQL_ENSURE_S(inputIndex.emplace(TString(name), i).second); + const auto& name = rowType->GetMemberName(i); + MKQL_ENSURE_S(inputIndex.emplace(TString(name), i).second); rowTypes[i] = NKqp::UnwrapDataTypeFromStruct(*rowType, i); } - TVector<ui32> keyIndices(tableKeyTypes.size()); - for (ui32 i = 0; i < tableKeyTypes.size(); i++) { - auto it = inputIndex.find(tableKeyTypes[i].second); - MKQL_ENSURE_S(it != inputIndex.end()); + TVector<ui32> keyIndices(tableKeyTypes.size()); + for (ui32 i = 0; i < tableKeyTypes.size(); i++) { + auto it = inputIndex.find(tableKeyTypes[i].second); + MKQL_ENSURE_S(it != inputIndex.end()); auto typeId = NKqp::UnwrapDataTypeFromStruct(*rowType, it->second); - MKQL_ENSURE_S(typeId == tableKeyTypes[i].first, "row key type missmatch with table key type"); - keyIndices[i] = it->second; + MKQL_ENSURE_S(typeId == tableKeyTypes[i].first, "row key type missmatch with table key type"); + keyIndices[i] = it->second; } for (const auto& [_, column] : tableInfo->Columns) { @@ -200,13 +200,13 @@ IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeF auto it = inputIndex.find(column.Name); MKQL_ENSURE(it != inputIndex.end(), "Not null column " << column.Name << " has to be specified in upsert"); - + auto columnType = rowType->GetMemberType(it->second); MKQL_ENSURE(columnType->GetKind() != NMiniKQL::TType::EKind::Optional, "Not null column " << column.Name << " can't be optional"); - } - } - + } + } + auto upsertColumnsDict = AS_VALUE(TDictLiteral, upsertColumnsNode); TVector<TUpsertColumn> upsertColumns(upsertColumnsDict->GetItemsCount()); for (ui32 i = 0; i < upsertColumns.size(); ++i) { diff --git a/ydb/core/tx/datashard/datashard_locks.cpp b/ydb/core/tx/datashard/datashard_locks.cpp index 941908cc5a..aa5fc3876b 100644 --- a/ydb/core/tx/datashard/datashard_locks.cpp +++ b/ydb/core/tx/datashard/datashard_locks.cpp @@ -544,7 +544,7 @@ void TSysLocks::SetLock(const TTableId& tableId, const TTableRange& range, ui64 } void TSysLocks::BreakLock(const TTableId& tableId, const TArrayRef<const TCell>& key) { - Y_VERIFY(!tableId.HasSamePath(TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks))); + Y_VERIFY(!tableId.HasSamePath(TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks))); if (!Self->IsUserTable(tableId)) return; diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp index 5e87526a24..3498162c17 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.cpp +++ b/ydb/core/tx/datashard/datashard_pipeline.cpp @@ -781,7 +781,7 @@ void TPipeline::CompleteSchemaTx(NIceDb::TNiceDb& db, ui64 txId) { Y_VERIFY(txId == op->TxId); Y_VERIFY(op->Done); - Self->TransQueue.RemoveScanProgress(db, txId); + Self->TransQueue.RemoveScanProgress(db, txId); Self->TransQueue.RemoveSchemaOperation(db, txId); } @@ -1352,9 +1352,9 @@ EExecutionStatus TPipeline::RunExecutionUnit(TOperation::TPtr op, TTransactionCo return EExecutionStatus::Continue; } - NCpuTime::TCpuTimer timer; + NCpuTime::TCpuTimer timer; auto status = unit.Execute(op, txc, ctx); - op->AddExecutionTime(timer.GetTime()); + op->AddExecutionTime(timer.GetTime()); LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Execution status for " << *op << " at " << Self->TabletID() @@ -1401,9 +1401,9 @@ EExecutionStatus TPipeline::RunExecutionPlan(TOperation::TPtr op, return EExecutionStatus::Reschedule; } - NCpuTime::TCpuTimer timer; + NCpuTime::TCpuTimer timer; auto status = unit.Execute(op, txc, ctx); - op->AddExecutionTime(timer.GetTime()); + op->AddExecutionTime(timer.GetTime()); LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Execution status for " << *op << " at " << Self->TabletID() diff --git a/ydb/core/tx/datashard/datashard_trans_queue.cpp b/ydb/core/tx/datashard/datashard_trans_queue.cpp index 1df1422e84..d7096c25b3 100644 --- a/ydb/core/tx/datashard/datashard_trans_queue.cpp +++ b/ydb/core/tx/datashard/datashard_trans_queue.cpp @@ -155,31 +155,31 @@ bool TTransQueue::Load(NIceDb::TNiceDb& db) { if (!rowset.Next()) return false; } - // Load scan state (only for scheme tx) - { - auto rowset = db.Table<Schema::ScanProgress>().Range().Select(); - if (!rowset.IsReady()) - return false; - while (!rowset.EndOfSet()) { - ui64 txId = rowset.GetValue<Schema::ScanProgress::TxId>(); + // Load scan state (only for scheme tx) + { + auto rowset = db.Table<Schema::ScanProgress>().Range().Select(); + if (!rowset.IsReady()) + return false; + while (!rowset.EndOfSet()) { + ui64 txId = rowset.GetValue<Schema::ScanProgress::TxId>(); TSchemaOperation* op = FindSchemaTx(txId); - if (!op) { - LOG_WARN_S(TlsActivationContext->AsActorContext(), NKikimrServices::TX_DATASHARD, - "Op was not found for persisted scan tx id " << txId - << " on tablet " - << Self->TabletID()); - continue; - } - op->ScanState.LastKey = rowset.GetValue<Schema::ScanProgress::LastKey>(); - op->ScanState.Bytes = rowset.GetValue<Schema::ScanProgress::LastBytes>(); - ui64 lastStatus = rowset.GetValue<Schema::ScanProgress::LastStatus>(); - op->ScanState.StatusCode = static_cast<Ydb::StatusIds::StatusCode>(lastStatus); - auto binaryIssues = rowset.GetValue<Schema::ScanProgress::LastIssues>(); - op->ScanState.Issues = DeserializeIssues(binaryIssues); - if (!rowset.Next()) - return false; - } - } + if (!op) { + LOG_WARN_S(TlsActivationContext->AsActorContext(), NKikimrServices::TX_DATASHARD, + "Op was not found for persisted scan tx id " << txId + << " on tablet " + << Self->TabletID()); + continue; + } + op->ScanState.LastKey = rowset.GetValue<Schema::ScanProgress::LastKey>(); + op->ScanState.Bytes = rowset.GetValue<Schema::ScanProgress::LastBytes>(); + ui64 lastStatus = rowset.GetValue<Schema::ScanProgress::LastStatus>(); + op->ScanState.StatusCode = static_cast<Ydb::StatusIds::StatusCode>(lastStatus); + auto binaryIssues = rowset.GetValue<Schema::ScanProgress::LastIssues>(); + op->ScanState.Issues = DeserializeIssues(binaryIssues); + if (!rowset.Next()) + return false; + } + } } return true; @@ -284,11 +284,11 @@ void TTransQueue::RemoveSchemaOperation(NIceDb::TNiceDb& db, ui64 txId) { db.Table<Schema::SchemaOperations>().Key(txId).Delete(); } -void TTransQueue::RemoveScanProgress(NIceDb::TNiceDb& db, ui64 txId) { +void TTransQueue::RemoveScanProgress(NIceDb::TNiceDb& db, ui64 txId) { using Schema = TDataShard::Schema; - db.Table<Schema::ScanProgress>().Key(txId).Delete(); -} - + db.Table<Schema::ScanProgress>().Key(txId).Delete(); +} + /// @arg inOutStep, inOutTxId - last (complete) tx /// @return inOutStep, inOutTxId - next planned Tx void TTransQueue::GetPlannedTxId(ui64& step, ui64& txId) const { diff --git a/ydb/core/tx/datashard/datashard_trans_queue.h b/ydb/core/tx/datashard/datashard_trans_queue.h index 5dff772862..1d392676da 100644 --- a/ydb/core/tx/datashard/datashard_trans_queue.h +++ b/ydb/core/tx/datashard/datashard_trans_queue.h @@ -112,7 +112,7 @@ private: // for pipeline only void RemoveTx(NIceDb::TNiceDb &db, const TOperation &op); void RemoveSchemaOperation(NIceDb::TNiceDb& db, ui64 txId); - void RemoveScanProgress(NIceDb::TNiceDb& db, ui64 txId); + void RemoveScanProgress(NIceDb::TNiceDb& db, ui64 txId); ui64 NextDeadlineStep() const { if (DeadlineQueue.empty()) diff --git a/ydb/core/tx/datashard/datashard_txs.h b/ydb/core/tx/datashard/datashard_txs.h index a12afb6849..4a9ab00878 100644 --- a/ydb/core/tx/datashard/datashard_txs.h +++ b/ydb/core/tx/datashard/datashard_txs.h @@ -167,16 +167,16 @@ private: }; class TDataShard::TTxStoreScanState : public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxStoreScanState(TDataShard* ds, TEvPrivate::TEvPersistScanState::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext &ctx) override; - TTxType GetTxType() const override { return TXTYPE_STORE_SCAN_STATE; } -private: - TEvPrivate::TEvPersistScanState::TPtr Ev; + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext &ctx) override; + TTxType GetTxType() const override { return TXTYPE_STORE_SCAN_STATE; } +private: + TEvPrivate::TEvPersistScanState::TPtr Ev; TActorId Sender; -}; - +}; + class TDataShard::TTxRefreshVolatileSnapshot : public NTabletFlatExecutor::TTransactionBase<TDataShard> { public: TTxRefreshVolatileSnapshot(TDataShard* ds, TEvDataShard::TEvRefreshVolatileSnapshotRequest::TPtr ev); diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index d7e3194c3f..70bc940073 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -261,9 +261,9 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) descr.GetPartitionRangeEndIsInclusive()); } - TableSchemaVersion = descr.GetTableSchemaVersion(); + TableSchemaVersion = descr.GetTableSchemaVersion(); IsBackup = descr.GetIsBackup(); - + CheckSpecialColumns(); for (const auto& indexDesc : descr.GetTableIndexes()) { @@ -570,8 +570,8 @@ void TUserTable::ApplyAlter( alter.SetEraseCache(LocalTid, config.GetEnableEraseCache(), config.GetEraseCacheMinRows(), config.GetEraseCacheMaxBytes()); } - schema.SetTableSchemaVersion(delta.GetTableSchemaVersion()); - + schema.SetTableSchemaVersion(delta.GetTableSchemaVersion()); + SetSchema(schema); } diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 6843ff7539..0dab30cb84 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -395,10 +395,10 @@ struct TUserTable : public TThrRefBase { void SetPath(const TString &path); - ui64 GetTableSchemaVersion() const { return TableSchemaVersion; } + ui64 GetTableSchemaVersion() const { return TableSchemaVersion; } void SetTableSchemaVersion(ui64 schemaVersion); bool ResetTableSchemaVersion(); - + void AddIndex(const NKikimrSchemeOp::TIndexDescription& indexDesc); void DropIndex(const TPathId& indexPathId); bool HasAsyncIndexes() const; @@ -414,7 +414,7 @@ private: private: TString Schema; - ui64 TableSchemaVersion = 0; + ui64 TableSchemaVersion = 0; void CheckSpecialColumns(); void AlterSchema(); diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index 47f70192d6..d754f7ad0e 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -457,10 +457,10 @@ void TFakeScanTx::AddPlanStepShardResult(ui32 /*shardId*/, YdbOld::ResultSet part; UNIT_ASSERT(part.ParseFromArray(res.data(), res.size())); - if (Result.column_metaSize()) - part.Clearcolumn_meta(); + if (Result.column_metaSize()) + part.Clearcolumn_meta(); Result.MergeFrom(part); - UNIT_ASSERT(Result.column_metaSize()); + UNIT_ASSERT(Result.column_metaSize()); } else if (event->Record.GetStatus() == NKikimrTxDataShard::TEvProposeTransactionResult::ERROR) { Status = IEngineFlat::EStatus::Error; } else { diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index 0c0b0508a6..32472dce59 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -1169,8 +1169,8 @@ Y_UNIT_TEST_WITH_MVCC(ScanFollowedByUpdate) { const TFakeScanTx &scanTx = dynamic_cast<const TFakeScanTx &>(tx); YdbOld::ResultSet res = scanTx.GetScanResult(); //Cerr << res.DebugString() << Endl; - for (auto &row : res.rows()) { - auto &val = row.items(0).text_value(); + for (auto &row : res.rows()) { + auto &val = row.items(0).text_value(); UNIT_ASSERT(ref.contains(val)); ref.erase(val); } diff --git a/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp b/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp index 4b1be62959..ab1c586d30 100644 --- a/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp @@ -1,26 +1,26 @@ -#include "defs.h" -#include "datashard_ut_common.h" +#include "defs.h" +#include "datashard_ut_common.h" #include "datashard_ut_common_kqp.h" - + #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/testlib/test_client.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/tx_proxy/upload_rows.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - + +namespace NKikimr { + using namespace NKikimr::NDataShard::NKqpHelpers; using namespace NSchemeShard; -using namespace Tests; - +using namespace Tests; + namespace { -using TRows = TVector<std::pair<TSerializedCellVec, TString>>; -using TRowTypes = TVector<std::pair<TString, Ydb::Type>>; - +using TRows = TVector<std::pair<TSerializedCellVec, TString>>; +using TRowTypes = TVector<std::pair<TString, Ydb::Type>>; + static void DoStartUploadTestRows( const Tests::TServer::TPtr& server, const TActorId& sender, @@ -28,7 +28,7 @@ static void DoStartUploadTestRows( Ydb::Type::PrimitiveTypeId typeId) { auto& runtime = *server->GetRuntime(); - + std::shared_ptr<TRows> rows(new TRows); auto types = std::make_shared<TRowTypes>(); Ydb::Type type; @@ -42,11 +42,11 @@ static void DoStartUploadTestRows( TString serializedValue = TSerializedCellVec::Serialize(value); rows->emplace_back(serializedKey, serializedValue); } - + auto actor = NTxProxy::CreateUploadRowsInternal(sender, tableName, types, rows); runtime.Register(actor); -} - +} + static void DoWaitUploadTestRows( const Tests::TServer::TPtr& server, const TActorId& sender, @@ -74,19 +74,19 @@ static TActorId DoStartUploadRows( { auto sender = runtime.AllocateEdgeActor(); - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (const auto& kv : data) { auto key = TVector<TCell>{TCell::Make(kv.first)}; auto value = TVector<TCell>{TCell::Make(kv.second)}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( @@ -122,31 +122,31 @@ static void DoUploadRows( } // namespace -Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { - +Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { + Y_UNIT_TEST_WITH_MVCC(TestUploadRows) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings.SetDomainName("Root") + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") .SetEnableMvcc(WithMvcc) - .SetUseRealThreads(false); - - Tests::TServer::TPtr server = new TServer(serverSettings); - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - - InitRoot(server, sender); - + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + + InitRoot(server, sender); + CreateShardedTable(server, sender, "/Root", "table-1", 4, false); - - DoUploadTestRows(server, sender, "/Root/table-1", Ydb::Type::UINT32, Ydb::StatusIds::SUCCESS); - - DoUploadTestRows(server, sender, "/Root/table-doesnt-exist", Ydb::Type::UINT32, Ydb::StatusIds::SCHEME_ERROR); - - DoUploadTestRows(server, sender, "/Root/table-1", Ydb::Type::INT32, Ydb::StatusIds::SCHEME_ERROR); - } + + DoUploadTestRows(server, sender, "/Root/table-1", Ydb::Type::UINT32, Ydb::StatusIds::SUCCESS); + + DoUploadTestRows(server, sender, "/Root/table-doesnt-exist", Ydb::Type::UINT32, Ydb::StatusIds::SCHEME_ERROR); + + DoUploadTestRows(server, sender, "/Root/table-1", Ydb::Type::INT32, Ydb::StatusIds::SCHEME_ERROR); + } Y_UNIT_TEST_WITH_MVCC(TestUploadRowsDropColumnRace) { TPortManager pm; @@ -287,25 +287,25 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had value=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto value = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, "/Root/table-1", std::move(types), - rows, + rows, NTxProxy::EUploadRowsMode::WriteToTableShadow); runtime.Register(actor); @@ -353,19 +353,19 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had value=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto value = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, @@ -434,19 +434,19 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had value=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto value = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, @@ -533,19 +533,19 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had value=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto value = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, @@ -635,19 +635,19 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had value=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("value", type); + types->emplace_back("key", type); + types->emplace_back("value", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto value = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedValue = TSerializedCellVec::Serialize(value); - rows->emplace_back(serializedKey, serializedValue); + rows->emplace_back(serializedKey, serializedValue); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, @@ -687,19 +687,19 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { // Write shadow data: keys from 1 to 9 historically had extra=key*10 { - auto types = std::make_shared<TRowTypes>(); + auto types = std::make_shared<TRowTypes>(); Ydb::Type type; type.set_type_id(Ydb::Type::UINT32); - types->emplace_back("key", type); - types->emplace_back("extra", type); + types->emplace_back("key", type); + types->emplace_back("extra", type); - auto rows = std::make_shared<TRows>(); + auto rows = std::make_shared<TRows>(); for (ui32 i = 1; i <= 9; i++) { auto key = TVector<TCell>{TCell::Make(ui32(i))}; auto extra = TVector<TCell>{TCell::Make(ui32(i * 10))}; TSerializedCellVec serializedKey(TSerializedCellVec::Serialize(key)); TString serializedExtra = TSerializedCellVec::Serialize(extra); - rows->emplace_back(serializedKey, serializedExtra); + rows->emplace_back(serializedKey, serializedExtra); } auto actor = NTxProxy::CreateUploadRowsInternal( sender, @@ -738,6 +738,6 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { "key = 10, value = (empty maybe), extra = (empty maybe)\n"); } -} - -} // namespace NKikimr +} + +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/export_scan.cpp b/ydb/core/tx/datashard/export_scan.cpp index 08b6e1bba8..2cda75fdd0 100644 --- a/ydb/core/tx/datashard/export_scan.cpp +++ b/ydb/core/tx/datashard/export_scan.cpp @@ -170,7 +170,7 @@ public: } IScan::THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept override { - TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); + TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); Driver = driver; Scheme = std::move(scheme); diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp index 436f319203..63e94fde96 100644 --- a/ydb/core/tx/datashard/read_table_scan.cpp +++ b/ydb/core/tx/datashard/read_table_scan.cpp @@ -432,7 +432,7 @@ private: Driver = driver; auto ctx = TActivationContext::AsActorContext(); - auto aid = ctx.RegisterWithSameMailbox(this); + auto aid = ctx.RegisterWithSameMailbox(this); for (const auto& columnRecord : Tx.GetColumns()) { if (!scheme->ColInfo(columnRecord.GetId())) { diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 2ef5a77ad1..9c98e8e79b 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -1,9 +1,9 @@ LIBRARY() -OWNER( - ddoarn - g:kikimr -) +OWNER( + ddoarn + g:kikimr +) SRCS( alter_cdc_stream_unit.cpp @@ -68,7 +68,7 @@ SRCS( datashard__snapshot_txs.cpp datashard__stats.cpp datashard__store_table_path.cpp - datashard__store_scan_state.cpp + datashard__store_scan_state.cpp datashard_change_receiving.cpp datashard_change_sender_activation.cpp datashard_change_sending.cpp diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 5786300ad4..8e778a4a1d 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -961,7 +961,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { , CreateStep(0) , IsPrivatePath(false) , IsVirtual(isVirtual) - , SchemaVersion(0) + , SchemaVersion(0) { } @@ -979,7 +979,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { , PathId(other.PathId) , IsPrivatePath(other.IsPrivatePath) , IsVirtual(other.IsVirtual) - , SchemaVersion(other.SchemaVersion) + , SchemaVersion(other.SchemaVersion) { if (other.Subscriber) { other.Subscriber = TSubscriber(); @@ -1430,9 +1430,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { ListNodeEntry->Children.reserve(pathDesc.ChildrenSize()); for (const auto& child : pathDesc.GetChildren()) { - const auto& name = child.GetName(); - const auto pathId = TPathId(child.GetSchemeshardId(), child.GetPathId()); - + const auto& name = child.GetName(); + const auto pathId = TPathId(child.GetSchemeshardId(), child.GetPathId()); + switch (child.GetPathType()) { case NKikimrSchemeOp::EPathTypeSubDomain: case NKikimrSchemeOp::EPathTypeDir: @@ -1541,7 +1541,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { listNodeEntry->Kind = TNavigate::KindPath; listNodeEntry->Children.reserve(names.size()); for (const auto& name : names) { - listNodeEntry->Children.emplace_back(name, TPathId(), TNavigate::KindTable); + listNodeEntry->Children.emplace_back(name, TPathId(), TNavigate::KindTable); } entry.Kind = TNavigate::KindPath; @@ -1653,13 +1653,13 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } else { entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId); } - } else { + } else { entry.Path = SplitPath(Path); - // update schema version - entry.TableId.SchemaVersion = SchemaVersion; - - } - + // update schema version + entry.TableId.SchemaVersion = SchemaVersion; + + } + if (entry.Operation == TNavigate::OpList) { entry.ListNodeEntry = ListNodeEntry; } @@ -1901,9 +1901,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { bool IsPrivatePath; bool IsVirtual; - // Used for Table and Index - ui64 SchemaVersion; - + // Used for Table and Index + ui64 SchemaVersion; + // domain & path specific TIntrusivePtr<TNavigate::TListNodeEntry> ListNodeEntry; TIntrusivePtr<TNavigate::TDomainDescription> DomainDescription; diff --git a/ydb/core/tx/scheme_board/cache_ut.cpp b/ydb/core/tx/scheme_board/cache_ut.cpp index cd621f1bb9..c27b3517b3 100644 --- a/ydb/core/tx/scheme_board/cache_ut.cpp +++ b/ydb/core/tx/scheme_board/cache_ut.cpp @@ -50,7 +50,7 @@ public: UNIT_TEST(CheckSystemViewAccess); UNIT_TEST(SystemView); UNIT_TEST(SysLocks); - UNIT_TEST(TableSchemaVersion); + UNIT_TEST(TableSchemaVersion); UNIT_TEST(MigrationCommon); UNIT_TEST(MigrationCommit); UNIT_TEST(MigrationLostMessage); @@ -69,7 +69,7 @@ public: void CheckSystemViewAccess(); void SystemView(); void SysLocks(); - void TableSchemaVersion(); + void TableSchemaVersion(); void MigrationCommon(); void MigrationCommit(); void MigrationLostMessage(); @@ -336,36 +336,36 @@ void TCacheTest::SysLocks() { } } -void TCacheTest::TableSchemaVersion() { - ui64 txId = 100; - TestCreateTable(*Context, ++txId, "/Root", R"( - Name: "Table1" - Columns { Name: "key" Type: "Uint32" } - KeyColumnNames: [ "key" ] - PartitionConfig { - CompactionPolicy { - } - } +void TCacheTest::TableSchemaVersion() { + ui64 txId = 100; + TestCreateTable(*Context, ++txId, "/Root", R"( + Name: "Table1" + Columns { Name: "key" Type: "Uint32" } + KeyColumnNames: [ "key" ] + PartitionConfig { + CompactionPolicy { + } + } )", {NKikimrScheme::StatusAccepted}); - + TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - { + { auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); - UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 1); - } - - TestAlterTable(*Context, ++txId, "/Root", R"( - Name: "Table1" - Columns { Name: "added" Type: "Uint64"} - )"); - + UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 1); + } + + TestAlterTable(*Context, ++txId, "/Root", R"( + Name: "Table1" + Columns { Name: "added" Type: "Uint64"} + )"); + TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - { + { auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); - UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 2); - } -} - + UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 2); + } +} + TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired) { @@ -387,7 +387,7 @@ TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavi const TNavigate::TEntry result = ev->Get()->Request->ResultSet[0]; UNIT_ASSERT_VALUES_EQUAL(result.Status, expectedStatus); - return result; + return result; } TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus, diff --git a/ydb/core/tx/scheme_board/subscriber.cpp b/ydb/core/tx/scheme_board/subscriber.cpp index 0b7b6e4e99..97173d81ad 100644 --- a/ydb/core/tx/scheme_board/subscriber.cpp +++ b/ydb/core/tx/scheme_board/subscriber.cpp @@ -532,7 +532,7 @@ public: void Bootstrap(const TActorContext&) { TMonitorableActor<TDerived>::Bootstrap(); - ReplicaSubscriber = this->RegisterWithSameMailbox(new TReplicaDerived(this->SelfId(), Replica, Path, DomainOwnerId)); + ReplicaSubscriber = this->RegisterWithSameMailbox(new TReplicaDerived(this->SelfId(), Replica, Path, DomainOwnerId)); this->Become(&TDerived::StateWork); } @@ -840,7 +840,7 @@ class TSubscriber: public TMonitorableActor<TDerived> { } for (const auto& replica : replicas) { - Proxies.emplace(this->RegisterWithSameMailbox(new TProxyDerived(this->SelfId(), replica, Path, DomainOwnerId))); + Proxies.emplace(this->RegisterWithSameMailbox(new TProxyDerived(this->SelfId(), replica, Path, DomainOwnerId))); } this->Become(&TDerived::StateWork); diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index e1981f5ad4..26a07897a1 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -46,13 +46,13 @@ struct TDomainInfo : public TAtomicRefCount<TDomainInfo> { explicit TDomainInfo(const TPathId& domainKey, const TPathId& resourcesDomainKey) : DomainKey(domainKey) , ResourcesDomainKey(resourcesDomainKey) - , Coordinators(TVector<ui64>{}) + , Coordinators(TVector<ui64>{}) {} explicit TDomainInfo(const NKikimrSubDomains::TDomainDescription& descr) : DomainKey(GetDomainKey(descr.GetDomainKey())) - , Params(descr.GetProcessingParams()) - , Coordinators(descr.GetProcessingParams()) + , Params(descr.GetProcessingParams()) + , Coordinators(descr.GetProcessingParams()) { if (descr.HasResourcesDomainKey()) { ResourcesDomainKey = GetDomainKey(descr.GetResourcesDomainKey()); @@ -118,7 +118,7 @@ struct TSchemeCacheNavigate { KindSolomon = 7, KindSubdomain = 8, KindExtSubdomain = 9, - KindIndex = 10, + KindIndex = 10, KindOlapStore = 11, KindOlapTable = 12, KindCdcStream = 13, @@ -127,22 +127,22 @@ struct TSchemeCacheNavigate { }; struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> { - struct TChild { - const TString Name; + struct TChild { + const TString Name; const TPathId PathId; - const ui64 SchemaVersion; - const EKind Kind; + const ui64 SchemaVersion; + const EKind Kind; explicit TChild(const TString& name, const TPathId& pathId, EKind kind, ui64 schemaVersion = 0) : Name(name) - , PathId(pathId) + , PathId(pathId) , SchemaVersion(schemaVersion) , Kind(kind) {} - }; + }; EKind Kind = KindUnknown; - TVector<TChild> Children; + TVector<TChild> Children; }; struct TDirEntryInfo : public TAtomicRefCount<TDirEntryInfo>{ @@ -233,7 +233,7 @@ struct TSchemeCacheNavigate { THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; - + // other TIntrusiveConstPtr<TDomainDescription> DomainDescription; TIntrusiveConstPtr<TPQGroupInfo> PQGroupInfo; diff --git a/ydb/core/tx/schemeshard/schemeshard.h b/ydb/core/tx/schemeshard/schemeshard.h index 8f1df03e17..d33173508c 100644 --- a/ydb/core/tx/schemeshard/schemeshard.h +++ b/ydb/core/tx/schemeshard/schemeshard.h @@ -238,12 +238,12 @@ struct TEvSchemeShard { Record.SetSchemeshardId(tableId.PathId.OwnerId); Record.SetPathId(tableId.PathId.LocalPathId); } - - TEvDescribeScheme(NKikimr::TPathId pathId) - { - Record.SetSchemeshardId(pathId.OwnerId); - Record.SetPathId(pathId.LocalPathId); - } + + TEvDescribeScheme(NKikimr::TPathId pathId) + { + Record.SetSchemeshardId(pathId.OwnerId); + Record.SetPathId(pathId.LocalPathId); + } }; struct TEvDescribeSchemeResult : public TEventPreSerializedPB<TEvDescribeSchemeResult, diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 06c2f25887..650506549e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1063,9 +1063,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui64, TString> TTableIndexColRec; - typedef TDeque<TTableIndexColRec> TTableIndexKeyRows; - typedef TDeque<TTableIndexColRec> TTableIndexDataRows; + typedef std::tuple<TPathId, ui64, TString> TTableIndexColRec; + typedef TDeque<TTableIndexColRec> TTableIndexKeyRows; + typedef TDeque<TTableIndexColRec> TTableIndexDataRows; bool LoadTableIndexeKeys(NIceDb::TNiceDb& db, TTableIndexKeyRows& tableIndexKeys) const { { @@ -1110,26 +1110,26 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - bool LoadTableIndexDataColumns(NIceDb::TNiceDb& db, TTableIndexDataRows& tableIndexData) const { - auto rowSet = db.Table<Schema::TableIndexDataColumns>().Range().Select(); - if (!rowSet.IsReady()) { - return false; - } - - while (!rowSet.EndOfSet()) { - auto pathId = TPathId(rowSet.GetValue<Schema::TableIndexDataColumns::PathOwnerId>(), - rowSet.GetValue<Schema::TableIndexDataColumns::PathLocalId>()); - auto id = rowSet.GetValue<Schema::TableIndexDataColumns::DataColumnId>(); - auto name = rowSet.GetValue<Schema::TableIndexDataColumns::DataColumnName>(); - - tableIndexData.emplace_back(pathId, id, name); - if (!rowSet.Next()) { - return false; - } - } - return true; - } - + bool LoadTableIndexDataColumns(NIceDb::TNiceDb& db, TTableIndexDataRows& tableIndexData) const { + auto rowSet = db.Table<Schema::TableIndexDataColumns>().Range().Select(); + if (!rowSet.IsReady()) { + return false; + } + + while (!rowSet.EndOfSet()) { + auto pathId = TPathId(rowSet.GetValue<Schema::TableIndexDataColumns::PathOwnerId>(), + rowSet.GetValue<Schema::TableIndexDataColumns::PathLocalId>()); + auto id = rowSet.GetValue<Schema::TableIndexDataColumns::DataColumnId>(); + auto name = rowSet.GetValue<Schema::TableIndexDataColumns::DataColumnName>(); + + tableIndexData.emplace_back(pathId, id, name); + if (!rowSet.Next()) { + return false; + } + } + return true; + } + typedef std::tuple<TShardIdx, ui32, TString, TString> TChannelBindingRec; typedef TDeque<TChannelBindingRec> TChannelBindingRows; @@ -2582,21 +2582,21 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { { TTableIndexKeyRows indexKeys; - TTableIndexDataRows indexData; + TTableIndexDataRows indexData; if (!LoadTableIndexeKeys(db, indexKeys)) { return false; } - if (!LoadTableIndexDataColumns(db, indexData)) { - return false; - } - + if (!LoadTableIndexDataColumns(db, indexData)) { + return false; + } + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for TableIndexKeys" << ", readed records: " << indexKeys.size() << ", at schemeshard: " << Self->TabletID()); - for (const auto& rec: indexKeys) { + for (const auto& rec: indexKeys) { TPathId pathId = std::get<0>(rec); ui32 keyId = std::get<1>(rec); TString keyName = std::get<2>(rec); @@ -2612,35 +2612,35 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableIndex->IndexKeys.emplace_back(keyName); } - // See KIKIMR-13300 and restore VERIFY after at least one restart - TVector<std::pair<TPathId, ui32>> leakedDataColumns; - for (const auto& rec: indexData) { - TPathId pathId = std::get<0>(rec); - ui32 dataId = std::get<1>(rec); - TString dataName = std::get<2>(rec); - - if (Self->PathsById.contains(pathId)) { - TPathElement::TPtr path = Self->PathsById.at(pathId); - Y_VERIFY_S(path->IsTableIndex(), "Path is not a table index, pathId: " << pathId); - - if (Self->Indexes.contains(pathId)) { - auto tableIndex = Self->Indexes.at(pathId); - - Y_VERIFY(dataId == tableIndex->IndexDataColumns.size()); - tableIndex->IndexDataColumns.emplace_back(dataName); - } else { - leakedDataColumns.emplace_back(pathId, dataId); - } - } else { - leakedDataColumns.emplace_back(pathId, dataId); - } - } - - for (const auto& pair : leakedDataColumns) { - db.Table<Schema::TableIndexDataColumns>().Key(pair.first.OwnerId, pair.first.LocalPathId, pair.second).Delete(); - } - leakedDataColumns.clear(); - + // See KIKIMR-13300 and restore VERIFY after at least one restart + TVector<std::pair<TPathId, ui32>> leakedDataColumns; + for (const auto& rec: indexData) { + TPathId pathId = std::get<0>(rec); + ui32 dataId = std::get<1>(rec); + TString dataName = std::get<2>(rec); + + if (Self->PathsById.contains(pathId)) { + TPathElement::TPtr path = Self->PathsById.at(pathId); + Y_VERIFY_S(path->IsTableIndex(), "Path is not a table index, pathId: " << pathId); + + if (Self->Indexes.contains(pathId)) { + auto tableIndex = Self->Indexes.at(pathId); + + Y_VERIFY(dataId == tableIndex->IndexDataColumns.size()); + tableIndex->IndexDataColumns.emplace_back(dataName); + } else { + leakedDataColumns.emplace_back(pathId, dataId); + } + } else { + leakedDataColumns.emplace_back(pathId, dataId); + } + } + + for (const auto& pair : leakedDataColumns) { + db.Table<Schema::TableIndexDataColumns>().Key(pair.first.OwnerId, pair.first.LocalPathId, pair.second).Delete(); + } + leakedDataColumns.clear(); + // Read TableIndexKeysAlterData { auto rowset = db.Table<Schema::TableIndexKeysAlterData>().Range().Select(); @@ -2672,49 +2672,49 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return false; } } - - // Read TableIndexDataColumnsAlterData - { - TVector<std::pair<TPathId, ui32>> leakedDataColumnsAlterData; - auto rowset = db.Table<Schema::TableIndexDataColumnsAlterData>().Range().Select(); - if (!rowset.IsReady()) - return false; - - while (!rowset.EndOfSet()) { - TOwnerId ownerId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::PathOwnerId>(); - TLocalPathId localPathId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::PathLocalId>(); - TPathId pathId(ownerId, localPathId); - - ui32 dataColId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::DataColumnId>(); - TString dataColName = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::DataColumnName>(); - - if (Self->PathsById.contains(pathId)) { - TPathElement::TPtr path = Self->PathsById.at(pathId); - Y_VERIFY_S(path->IsTableIndex(), "Path is not a table index, pathId: " << pathId); - - if (Self->Indexes.contains(pathId)) { - auto tableIndex = Self->Indexes.at(pathId); - - Y_VERIFY(tableIndex->AlterData != nullptr); - auto alterData = tableIndex->AlterData; - Y_VERIFY(tableIndex->AlterVersion < alterData->AlterVersion); - - Y_VERIFY(dataColId == alterData->IndexDataColumns.size()); - alterData->IndexDataColumns.emplace_back(dataColName); - } else { + + // Read TableIndexDataColumnsAlterData + { + TVector<std::pair<TPathId, ui32>> leakedDataColumnsAlterData; + auto rowset = db.Table<Schema::TableIndexDataColumnsAlterData>().Range().Select(); + if (!rowset.IsReady()) + return false; + + while (!rowset.EndOfSet()) { + TOwnerId ownerId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::PathOwnerId>(); + TLocalPathId localPathId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::PathLocalId>(); + TPathId pathId(ownerId, localPathId); + + ui32 dataColId = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::DataColumnId>(); + TString dataColName = rowset.GetValue<Schema::TableIndexDataColumnsAlterData::DataColumnName>(); + + if (Self->PathsById.contains(pathId)) { + TPathElement::TPtr path = Self->PathsById.at(pathId); + Y_VERIFY_S(path->IsTableIndex(), "Path is not a table index, pathId: " << pathId); + + if (Self->Indexes.contains(pathId)) { + auto tableIndex = Self->Indexes.at(pathId); + + Y_VERIFY(tableIndex->AlterData != nullptr); + auto alterData = tableIndex->AlterData; + Y_VERIFY(tableIndex->AlterVersion < alterData->AlterVersion); + + Y_VERIFY(dataColId == alterData->IndexDataColumns.size()); + alterData->IndexDataColumns.emplace_back(dataColName); + } else { leakedDataColumnsAlterData.emplace_back(pathId, dataColId); - } - } else { + } + } else { leakedDataColumnsAlterData.emplace_back(pathId, dataColId); - } - - if (!rowset.Next()) - return false; - } - for (const auto& pair : leakedDataColumnsAlterData) { - db.Table<Schema::TableIndexDataColumnsAlterData>().Key(pair.first.OwnerId, pair.first.LocalPathId, pair.second).Delete(); - } - } + } + + if (!rowset.Next()) + return false; + } + for (const auto& pair : leakedDataColumnsAlterData) { + db.Table<Schema::TableIndexDataColumnsAlterData>().Key(pair.first.OwnerId, pair.first.LocalPathId, pair.second).Delete(); + } + } } // Read CdcStream @@ -2887,7 +2887,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } volume->AlterVersion = rowset.GetValue<Schema::BlockStoreVolumes::AlterVersion>(); volume->MountToken = rowset.GetValue<Schema::BlockStoreVolumes::MountToken>(); - volume->TokenVersion = rowset.GetValue<Schema::BlockStoreVolumes::TokenVersion>(); + volume->TokenVersion = rowset.GetValue<Schema::BlockStoreVolumes::TokenVersion>(); Self->BlockStoreVolumes[pathId] = volume; Self->IncrementPathDbRefCount(pathId); @@ -4145,27 +4145,27 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TIndexBuildInfo::TPtr buildInfo = Self->IndexBuilds.at(id); - TString columnName = rowset.GetValue<Schema::IndexBuildColumns::ColumnName>(); - EIndexColumnKind columnKind = rowset.GetValueOrDefault<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::KeyColumn); + TString columnName = rowset.GetValue<Schema::IndexBuildColumns::ColumnName>(); + EIndexColumnKind columnKind = rowset.GetValueOrDefault<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::KeyColumn); ui32 columnNo = rowset.GetValue<Schema::IndexBuildColumns::ColumnNo>(); - Y_VERIFY_S(columnNo == (buildInfo->IndexColumns.size() + buildInfo->DataColumns.size()), - "Unexpected non contiguous column number# " << columnNo << - " indexColumns# " << buildInfo->IndexColumns.size() << - " dataColumns# " << buildInfo->DataColumns.size()); - - switch (columnKind) { - case EIndexColumnKind::KeyColumn: - buildInfo->IndexColumns.push_back(columnName); - break; - case EIndexColumnKind::DataColumn: - buildInfo->DataColumns.push_back(columnName); - break; - default: - Y_FAIL_S("Unknown column kind# " << (int)columnKind); - break; - } - + Y_VERIFY_S(columnNo == (buildInfo->IndexColumns.size() + buildInfo->DataColumns.size()), + "Unexpected non contiguous column number# " << columnNo << + " indexColumns# " << buildInfo->IndexColumns.size() << + " dataColumns# " << buildInfo->DataColumns.size()); + + switch (columnKind) { + case EIndexColumnKind::KeyColumn: + buildInfo->IndexColumns.push_back(columnName); + break; + case EIndexColumnKind::DataColumn: + buildInfo->DataColumns.push_back(columnName); + break; + default: + Y_FAIL_S("Unknown column kind# " << (int)columnKind); + break; + } + if (!rowset.Next()) { return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp index 40a6a542ae..b6c0b6c8b2 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp @@ -10,24 +10,24 @@ using namespace NKikimr; using namespace NSchemeShard; bool CheckFreezeStateAlredySet(const TTableInfo::TPtr table, const NKikimrSchemeOp::TTableDescription& alter) { - if (!alter.HasPartitionConfig()) - return false; - if (alter.GetPartitionConfig().HasFreezeState()) { - if (table->PartitionConfig().HasFreezeState()) { - auto tableFreezeState = table->PartitionConfig().GetFreezeState(); - if (tableFreezeState == alter.GetPartitionConfig().GetFreezeState()) { - return true; - } - } else { + if (!alter.HasPartitionConfig()) + return false; + if (alter.GetPartitionConfig().HasFreezeState()) { + if (table->PartitionConfig().HasFreezeState()) { + auto tableFreezeState = table->PartitionConfig().GetFreezeState(); + if (tableFreezeState == alter.GetPartitionConfig().GetFreezeState()) { + return true; + } + } else { if (alter.GetPartitionConfig().GetFreezeState() == NKikimrSchemeOp::EFreezeState::Unfreeze) { - return true; - } - } - } - - return false; -} - + return true; + } + } + } + + return false; +} + bool IsSuperUser(const NACLib::TUserToken* userToken) { if (!userToken) return false; @@ -79,14 +79,14 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, return nullptr; } - if (copyAlter.HasPartitionConfig() && copyAlter.GetPartitionConfig().HasFreezeState()) { + if (copyAlter.HasPartitionConfig() && copyAlter.GetPartitionConfig().HasFreezeState()) { if (hasSchemaChanges) { - errStr = Sprintf("Mix freeze cmd with other options is forbiden"); + errStr = Sprintf("Mix freeze cmd with other options is forbiden"); status = NKikimrScheme::StatusInvalidParameter; - return nullptr; - } - } - + return nullptr; + } + } + // Ignore column ids if they were passed by user! for (auto& col : *copyAlter.MutableColumns()) { col.ClearId(); @@ -102,17 +102,17 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, } if (CheckFreezeStateAlredySet(table, copyAlter)) { - errStr = Sprintf("Requested freeze state alredy set"); + errStr = Sprintf("Requested freeze state alredy set"); status = NKikimrScheme::StatusAlreadyExists; - return nullptr; - } - + return nullptr; + } + NKikimrSchemeOp::TPartitionConfig compilationPartitionConfig; if (!TPartitionConfigMerger::ApplyChanges(compilationPartitionConfig, table->PartitionConfig(), copyAlter.GetPartitionConfig(), appData, errStr) || !TPartitionConfigMerger::VerifyAlterParams(table->PartitionConfig(), compilationPartitionConfig, appData, shadowDataAllowed, errStr)) { status = NKikimrScheme::StatusInvalidParameter; return nullptr; - } + } copyAlter.MutablePartitionConfig()->CopyFrom(compilationPartitionConfig); const TSubDomainInfo& subDomain = *path.DomainInfo(); @@ -206,17 +206,17 @@ bool CheckDropingColumns(const TSchemeShard* ss, const NKikimrSchemeOp::TTableDe return false; } } - - for (const auto& col: indexInfo->IndexDataColumns) { - if (deletedColumns.contains(col)) { - errStr = TStringBuilder () - << "Imposible drop column because table index covers that column" + + for (const auto& col: indexInfo->IndexDataColumns) { + if (deletedColumns.contains(col)) { + errStr = TStringBuilder () + << "Imposible drop column because table index covers that column" << ", column name: " << col - << ", table name: " << tablePath.PathString() + << ", table name: " << tablePath.PathString() << ", index name: " << childName; - return false; - } - } + return false; + } + } } return true; @@ -538,7 +538,7 @@ public: } Y_VERIFY(context.SS->Tables.contains(path.Base()->PathId)); - TTableInfo::TPtr table = context.SS->Tables.at(path.Base()->PathId); + TTableInfo::TPtr table = context.SS->Tables.at(path.Base()->PathId); if (table->AlterVersion == 0) { result->SetError(NKikimrScheme::StatusMultipleModifications, "Table is not created yet"); @@ -573,10 +573,10 @@ public: NKikimrScheme::EStatus status; TTableInfo::TAlterDataPtr alterData = ParseParams(path, table, alter, IsShadowDataAllowed(), errStr, status, context); if (!alterData) { - result->SetError(status, errStr); + result->SetError(status, errStr); return result; } - + Y_VERIFY(alterData->AlterVersion == table->AlterVersion + 1); if (!CheckDropingColumns(context.SS, alter, path, errStr)) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h index 8f0b4ff08d..640f5f1356 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h @@ -154,7 +154,7 @@ public: static void CollectStats(TOperationId operationId, const TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) { const auto& evRecord = ev->Get()->Record; - if (!evRecord.HasOpResult() || !evRecord.GetOpResult().HasSuccess()) { + if (!evRecord.HasOpResult() || !evRecord.GetOpResult().HasSuccess()) { return; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp index 7bac5c9404..5a88636ffa 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp @@ -145,12 +145,12 @@ bool NTableState::CollectSchemaChanged( txState.SchemeChangeNotificationReceived[shardIdx] = std::make_pair(ackTo, generation); - - if (evRecord.HasOpResult()) { - // TODO: remove TxBackup handling + + if (evRecord.HasOpResult()) { + // TODO: remove TxBackup handling Y_VERIFY_DEBUG(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); - } - + } + if (!txState.ReadyForNotifications) { return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index 5b224ddff3..4cc467dedc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -34,14 +34,14 @@ NKikimrSchemeOp::TModifyScheme CreateIndexTask(NKikimr::NSchemeShard::TTableInde operation->SetName(dst.LeafName()); operation->SetType(indexInfo->Type); - for (const auto& keyName: indexInfo->IndexKeys) { + for (const auto& keyName: indexInfo->IndexKeys) { *operation->MutableKeyColumnNames()->Add() = keyName; } - for (const auto& dataColumn: indexInfo->IndexDataColumns) { - *operation->MutableDataColumnNames()->Add() = dataColumn; - } - + for (const auto& dataColumn: indexInfo->IndexDataColumns) { + *operation->MutableDataColumnNames()->Add() = dataColumn; + } + return scheme; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index c8ff3777e1..f504c34ef5 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -68,7 +68,7 @@ public: Y_VERIFY(srcTableInfo->GetPartitions().size() == dstTableInfo->GetPartitions().size(), "CopyTable paritition counts don't match"); - const ui64 dstSchemaVersion = NEW_TABLE_ALTER_VERSION; + const ui64 dstSchemaVersion = NEW_TABLE_ALTER_VERSION; const ui64 subDomainPathId = context.SS->ResolveDomainId(txState->TargetPathId).LocalPathId; @@ -93,7 +93,7 @@ public: // Send "CreateTable + ReceiveParts" transaction to destination datashard NKikimrTxDataShard::TFlatSchemeTransaction newShardTx; context.SS->FillSeqNo(newShardTx, seqNo); - context.SS->FillTableDescription(txState->TargetPathId, i, dstSchemaVersion, newShardTx.MutableCreateTable()); + context.SS->FillTableDescription(txState->TargetPathId, i, dstSchemaVersion, newShardTx.MutableCreateTable()); newShardTx.MutableReceiveSnapshot()->SetTableId_Deprecated(txState->TargetPathId.LocalPathId); newShardTx.MutableReceiveSnapshot()->MutableTableId()->SetOwnerId(txState->TargetPathId.OwnerId); newShardTx.MutableReceiveSnapshot()->MutableTableId()->SetTableId(txState->TargetPathId.LocalPathId); @@ -188,7 +188,7 @@ public: TTableInfo::TPtr table = context.SS->Tables[pathId]; Y_VERIFY(table); - table->AlterVersion = NEW_TABLE_ALTER_VERSION; + table->AlterVersion = NEW_TABLE_ALTER_VERSION; context.SS->PersistTableCreated(db, pathId); context.SS->TabletCounters->Simple()[COUNTER_TABLE_COUNT].Add(1); @@ -726,12 +726,12 @@ TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxT auto operation = schema.MutableCreateTableIndex(); operation->SetName(name); operation->SetType(indexInfo->Type); - for (const auto& keyName: indexInfo->IndexKeys) { + for (const auto& keyName: indexInfo->IndexKeys) { *operation->MutableKeyColumnNames()->Add() = keyName; } - for (const auto& dataColumn: indexInfo->IndexDataColumns) { - *operation->MutableDataColumnNames()->Add() = dataColumn; - } + for (const auto& dataColumn: indexInfo->IndexDataColumns) { + *operation->MutableDataColumnNames()->Add() = dataColumn; + } result.push_back(CreateNewTableIndex(NextPartId(nextId, result), schema)); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp index 52f12fbc24..2d957165c6 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp @@ -58,7 +58,7 @@ TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId nextId, const TTx { auto indexImplTableCreation = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable); NTableIndex::TTableColumns baseTableColumns = NTableIndex::ExtractInfo(tableInfo); - NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexConfig); + NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexConfig); TString explain; if (!NTableIndex::IsCompatibleIndex(baseTableColumns, indexKeys, explain)) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index e9999102ec..5beb0a8848 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -27,14 +27,14 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T const NKikimrSchemeOp::TTableDescription& baseTableDescription = indexedTable.GetTableDescription(); ui32 indexesCount = indexedTable.IndexDescriptionSize(); - ui32 indexedTableShards = 0; - for (const auto& desc : indexedTable.GetIndexDescription()) { - if (desc.HasIndexImplTableDescription()) { - indexedTableShards += TTableInfo::ShardsToCreate(desc.GetIndexImplTableDescription()); - } else { - indexedTableShards += 1; - } - } + ui32 indexedTableShards = 0; + for (const auto& desc : indexedTable.GetIndexDescription()) { + if (desc.HasIndexImplTableDescription()) { + indexedTableShards += TTableInfo::ShardsToCreate(desc.GetIndexImplTableDescription()); + } else { + indexedTableShards += 1; + } + } ui32 sequencesCount = indexedTable.SequenceDescriptionSize(); ui32 baseShards = TTableInfo::ShardsToCreate(baseTableDescription); ui32 shardsToCreate = baseShards + indexedTableShards; @@ -108,12 +108,12 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T TTableColumns baseTableColumns = ExtractInfo(baseTableDescription); for (auto& indexDescription: indexedTable.GetIndexDescription()) { const auto& indexName = indexDescription.GetName(); - bool uniformIndexTable = false; - if (indexDescription.HasIndexImplTableDescription()) { - if (indexDescription.GetIndexImplTableDescription().HasUniformPartitionsCount()) { - uniformIndexTable = true; - } - } + bool uniformIndexTable = false; + if (indexDescription.HasIndexImplTableDescription()) { + if (indexDescription.GetIndexImplTableDescription().HasUniformPartitionsCount()) { + uniformIndexTable = true; + } + } TPath indexPath = baseTablePath.Child(indexName); { @@ -143,17 +143,17 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T break; } - TIndexColumns indexKeys = ExtractInfo(indexDescription); - if (indexKeys.KeyColumns.empty()) { + TIndexColumns indexKeys = ExtractInfo(indexDescription); + if (indexKeys.KeyColumns.empty()) { TString msg = TStringBuilder() << "no key colums in index creation config"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; } if (!indexKeys.DataColumns.empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { - TString msg = TStringBuilder() << "It is not allowed to create index with data column"; + TString msg = TStringBuilder() << "It is not allowed to create index with data column"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; - } - + } + TString explainErr; if (!IsCompatibleIndex(baseTableColumns, indexKeys, explainErr)) { TString msg = TStringBuilder() << "IsCompatibleIndex fail with explain: " << explainErr; @@ -176,7 +176,7 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T if (impTableColumns.Keys.size() > domainInfo->GetSchemeLimits().MaxTableKeyColumns) { TString msg = TStringBuilder() << "Too many key indexed, index table reaches the limit of the maximum keys colums count" - << ": indexing colums: " << indexKeys.KeyColumns.size() + << ": indexing colums: " << indexKeys.KeyColumns.size() << ": requested keys colums for index table: " << impTableColumns.Keys.size() << ". Limit: " << domainInfo->GetSchemeLimits().MaxTableKeyColumns; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusSchemeError, msg)}; @@ -285,9 +285,9 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T TTableColumns impTableColumns = indexes.at(indexDescription.GetName()); auto& indexImplTableDescription = *scheme.MutableCreateTable(); - // This description provided by user to override partition policy - const auto& userIndexDesc = indexDescription.GetIndexImplTableDescription(); - indexImplTableDescription = CalcImplTableDesc(baseTableDescription, impTableColumns, userIndexDesc); + // This description provided by user to override partition policy + const auto& userIndexDesc = indexDescription.GetIndexImplTableDescription(); + indexImplTableDescription = CalcImplTableDesc(baseTableDescription, impTableColumns, userIndexDesc); result.push_back(CreateNewTable(NextPartId(nextId, result), scheme)); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index d0fa02db5d..16ff55eaa6 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -204,8 +204,8 @@ public: } { - NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexSchema); - if (indexKeys.KeyColumns.empty()) { + NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexSchema); + if (indexKeys.KeyColumns.empty()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "no key colums in index creation config"); return result; } @@ -230,7 +230,7 @@ public: if (impTableColumns.Keys.size() > tablePath.DomainInfo()->GetSchemeLimits().MaxTableKeyColumns) { TString msg = TStringBuilder() << "Too many key indexed, index table reaches the limit of the maximum keys colums count" - << ": indexing colums: " << indexKeys.KeyColumns.size() + << ": indexing colums: " << indexKeys.KeyColumns.size() << ": requested keys colums for index table: " << impTableColumns.Keys.size() << ". Limit: " << tablePath.DomainInfo()->GetSchemeLimits().MaxTableKeyColumns; result->SetError(NKikimrScheme::StatusSchemeError, msg); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index d179b9950e..5004522529 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -186,10 +186,10 @@ public: << " seqNo: " << seqNo); NKikimrTxDataShard::TFlatSchemeTransaction tx(txTemplate); - auto tableDesc = tx.MutableCreateTable(); + auto tableDesc = tx.MutableCreateTable(); context.SS->FillSeqNo(tx, seqNo); - context.SS->FillTableDescription(txState->TargetPathId, i, NEW_TABLE_ALTER_VERSION, tableDesc); - + context.SS->FillTableDescription(txState->TargetPathId, i, NEW_TABLE_ALTER_VERSION, tableDesc); + TString txBody; Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); @@ -271,7 +271,7 @@ public: TTableInfo::TPtr table = context.SS->Tables[pathId]; Y_VERIFY(table); - table->AlterVersion = NEW_TABLE_ALTER_VERSION; + table->AlterVersion = NEW_TABLE_ALTER_VERSION; if (table->IsTTLEnabled()) { context.SS->TTLEnabledTables[pathId] = table; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp index be4fc30e7b..645d7f37e1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp @@ -116,11 +116,11 @@ public: db.Table<Schema::TxInFlightV2>().Key(OperationId.GetTxId(), OperationId.GetSubTxId()).Update( NIceDb::TUpdate<Schema::TxInFlightV2::ExtraBytes>(extraData)); - const auto tableInfo = context.SS->Tables.FindPtr(txState->TargetPathId); - Y_VERIFY(tableInfo); - - const ui64 alterVersion = (*tableInfo)->AlterVersion; - + const auto tableInfo = context.SS->Tables.FindPtr(txState->TargetPathId); + Y_VERIFY(tableInfo); + + const ui64 alterVersion = (*tableInfo)->AlterVersion; + const ui64 subDomainPathId = context.SS->ResolveDomainId(txState->TargetPathId).LocalPathId; for (const auto& shard: txState->Shards) { @@ -160,15 +160,15 @@ public: // diverge during a migration period. That's ok though, since // schemas may only become incompatible after column family storage // configuration is altered, and it's protected with a feature flag. - auto tableDesc = event->Record.MutableCreateTable(); + auto tableDesc = event->Record.MutableCreateTable(); context.SS->FillTableDescriptionForShardIdx( txState->TargetPathId, shard.Idx, - tableDesc, + tableDesc, rangeDescr.GetKeyRangeBegin(), rangeDescr.GetKeyRangeEnd(), true, false); - context.SS->FillTableSchemaVersion(alterVersion, tableDesc); + context.SS->FillTableSchemaVersion(alterVersion, tableDesc); context.OnComplete.BindMsgToPipe(OperationId, datashardId, shard.Idx, event.Release()); } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp index a78b80ba6a..6d8cdc38dc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp @@ -47,20 +47,20 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil NIceDb::TUpdate<Schema::IndexBuild::MaxRetries>(indexInfo->Limits.MaxRetries) ); - ui32 columnNo = 0; - for (ui32 i = 0; i < indexInfo->IndexColumns.size(); ++i, ++columnNo) { + ui32 columnNo = 0; + for (ui32 i = 0; i < indexInfo->IndexColumns.size(); ++i, ++columnNo) { db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Update( - NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnName>(indexInfo->IndexColumns[i]), - NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::KeyColumn) - ); + NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnName>(indexInfo->IndexColumns[i]), + NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::KeyColumn) + ); } - for (ui32 i = 0; i < indexInfo->DataColumns.size(); ++i, ++columnNo) { - db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Update( - NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnName>(indexInfo->DataColumns[i]), - NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::DataColumn) - ); - } + for (ui32 i = 0; i < indexInfo->DataColumns.size(); ++i, ++columnNo) { + db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Update( + NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnName>(indexInfo->DataColumns[i]), + NIceDb::TUpdate<Schema::IndexBuildColumns::ColumnKind>(EIndexColumnKind::DataColumn) + ); + } } void TSchemeShard::PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo) { @@ -171,15 +171,15 @@ void TSchemeShard::PersistBuildIndexUploadInitiate(NIceDb::TNiceDb& db, const TI void TSchemeShard::PersistBuildIndexForget(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo) { db.Table<Schema::IndexBuild>().Key(indexInfo->Id).Delete(); - ui32 columnNo = 0; - for (ui32 i = 0; i < indexInfo->IndexColumns.size(); ++i, ++columnNo) { + ui32 columnNo = 0; + for (ui32 i = 0; i < indexInfo->IndexColumns.size(); ++i, ++columnNo) { + db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Delete(); + } + + for (ui32 i = 0; i < indexInfo->DataColumns.size(); ++i, ++columnNo) { db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Delete(); } - for (ui32 i = 0; i < indexInfo->DataColumns.size(); ++i, ++columnNo) { - db.Table<Schema::IndexBuildColumns>().Key(indexInfo->Id, columnNo).Delete(); - } - for (const auto& item: indexInfo->Shards) { auto shardIdx = item.first; db.Table<Schema::IndexBuildShardStatus>().Key(indexInfo->Id, shardIdx.GetOwnerId(), shardIdx.GetLocalId()).Delete(); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__cancel.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__cancel.cpp index 813953b3ff..f68e76634a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__cancel.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__cancel.cpp @@ -69,7 +69,7 @@ public: if (indexBuildInfo->IsFinished()) { return Reply( std::move(response), - Ydb::StatusIds::PRECONDITION_FAILED, + Ydb::StatusIds::PRECONDITION_FAILED, TStringBuilder() << "Index build prcess with id <" << indexBuildId << "> has been finished already" ); } @@ -77,7 +77,7 @@ public: if (indexBuildInfo->IsCancellationRequested()) { return Reply( std::move(response), - Ydb::StatusIds::PRECONDITION_FAILED, + Ydb::StatusIds::PRECONDITION_FAILED, TStringBuilder() << "Index build prcess with id <" << indexBuildId << "> canceling already" ); } @@ -85,7 +85,7 @@ public: if (indexBuildInfo->State > TIndexBuildInfo::EState::Filling) { return Reply( std::move(response), - Ydb::StatusIds::PRECONDITION_FAILED, + Ydb::StatusIds::PRECONDITION_FAILED, TStringBuilder() << "Index build prcess with id <" << indexBuildId << "> are almost done, cancellation has no sense" ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp index 60a44530dc..8a94bce298 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp @@ -44,20 +44,20 @@ public: break; case Ydb::Table::TableIndex::kGlobalAsyncIndex: if (!Self->EnableAsyncIndexes) { - return Reply( - std::move(response), - Ydb::StatusIds::UNSUPPORTED, + return Reply( + std::move(response), + Ydb::StatusIds::UNSUPPORTED, TStringBuilder() << "Async indexes are not supported yet" - ); - } + ); + } break; default: break; - } - - const auto& dataColumns = request.GetSettings().index().data_columns(); - - if (!dataColumns.empty() && !Self->AllowDataColumnForIndexTable) { + } + + const auto& dataColumns = request.GetSettings().index().data_columns(); + + if (!dataColumns.empty() && !Self->AllowDataColumnForIndexTable) { return Reply( std::move(response), Ydb::StatusIds::UNSUPPORTED, @@ -202,7 +202,7 @@ private: buildInfo->IndexName = settings.index().name(); buildInfo->IndexColumns.assign(settings.index().index_columns().begin(), settings.index().index_columns().end()); - buildInfo->DataColumns.assign(settings.index().data_columns().begin(), settings.index().data_columns().end()); + buildInfo->DataColumns.assign(settings.index().data_columns().begin(), settings.index().data_columns().end()); buildInfo->Limits.MaxBatchRows = settings.max_batch_rows(); buildInfo->Limits.MaxBatchBytes = settings.max_batch_bytes(); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__get.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__get.cpp index 62fe95996f..ad6f5d01a4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__get.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__get.cpp @@ -40,7 +40,7 @@ public: if (!database.IsResolved()) { return Reply( std::move(response), - Ydb::StatusIds::BAD_REQUEST, + Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Database <" << record.GetDatabaseName() << "> not found" ); } @@ -51,7 +51,7 @@ public: if (!Self->IndexBuilds.contains(indexBuildId)) { return Reply( std::move(response), - Ydb::StatusIds::PRECONDITION_FAILED, + Ydb::StatusIds::PRECONDITION_FAILED, TStringBuilder() << "Index build prcess with id <" << indexBuildId << "> not found" ); } @@ -60,7 +60,7 @@ public: if (indexBuildInfo->DomainPathId != domainId) { return Reply( std::move(response), - Ydb::StatusIds::BAD_REQUEST, + Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Index build prcess with id <" << indexBuildId << "> not found in database <" << record.GetDatabaseName() << ">" ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index e511d5db15..a5fbbae703 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -23,9 +23,9 @@ NKikimrSchemeOp::TIndexBuildConfig GetInitiateIndexBuildMessage(TSchemeShard* ss for (const auto& x: buildInfo->IndexColumns) { *index.AddKeyColumnNames() = x; } - for (const auto& x: buildInfo->DataColumns) { - *index.AddDataColumnNames() = x; - } + for (const auto& x: buildInfo->DataColumns) { + *index.AddDataColumnNames() = x; + } index.SetType(buildInfo->IndexType); return message; @@ -280,15 +280,15 @@ public: ev->Record.SetPathId(buildInfo->TablePathId.LocalPathId); ev->Record.SetTargetName(buildInfo->ImplTablePath); - - THashSet<TString> columns = buildInfo->ImplTableColumns.Columns; + + THashSet<TString> columns = buildInfo->ImplTableColumns.Columns; for (const auto& x: buildInfo->ImplTableColumns.Keys) { *ev->Record.AddIndexColumns() = x; - columns.erase(x); + columns.erase(x); + } + for (const auto& x: columns) { + *ev->Record.AddDataColumns() = x; } - for (const auto& x: columns) { - *ev->Record.AddDataColumns() = x; - } TIndexBuildInfo::TShardStatus& shardStatus = buildInfo->Shards.at(shardIdx); if (shardStatus.LastKeyAck) { diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h index eaff538811..f2e4d9e56b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h @@ -201,12 +201,12 @@ protected: const TShardIdx& shardIdx = item.first; const TIndexBuildInfo::TShardStatus& status = item.second; - if (status.Status != NKikimrTxDataShard::TEvBuildIndexProgressResponse::INPROGRESS) { - if (status.UploadStatus != Ydb::StatusIds::SUCCESS) { - if (status.DebugMessage) { - AddIssue(index.MutableIssues(), status.ToString(shardIdx)); - } - } + if (status.Status != NKikimrTxDataShard::TEvBuildIndexProgressResponse::INPROGRESS) { + if (status.UploadStatus != Ydb::StatusIds::SUCCESS) { + if (status.DebugMessage) { + AddIssue(index.MutableIssues(), status.ToString(shardIdx)); + } + } } } @@ -214,30 +214,30 @@ protected: case TIndexBuildInfo::EState::Locking: case TIndexBuildInfo::EState::GatheringStatistics: case TIndexBuildInfo::EState::Initiating: - index.SetState(Ydb::Table::IndexBuildState::STATE_PREPARING); - index.SetProgress(0.0); + index.SetState(Ydb::Table::IndexBuildState::STATE_PREPARING); + index.SetProgress(0.0); break; case TIndexBuildInfo::EState::Filling: - index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); - index.SetProgress(indexInfo->CalcProgressPercent()); + index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); + index.SetProgress(indexInfo->CalcProgressPercent()); break; case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: - index.SetState(Ydb::Table::IndexBuildState::STATE_APPLYING); - index.SetProgress(100.0); + index.SetState(Ydb::Table::IndexBuildState::STATE_APPLYING); + index.SetProgress(100.0); break; case TIndexBuildInfo::EState::Done: - index.SetState(Ydb::Table::IndexBuildState::STATE_DONE); - index.SetProgress(100.0); + index.SetState(Ydb::Table::IndexBuildState::STATE_DONE); + index.SetProgress(100.0); break; case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: - index.SetState(Ydb::Table::IndexBuildState::STATE_CANCELLATION); - index.SetProgress(0.0); + index.SetState(Ydb::Table::IndexBuildState::STATE_CANCELLATION); + index.SetProgress(0.0); break; case TIndexBuildInfo::EState::Cancelled: - index.SetState(Ydb::Table::IndexBuildState::STATE_CANCELLED); - index.SetProgress(0.0); + index.SetState(Ydb::Table::IndexBuildState::STATE_CANCELLED); + index.SetProgress(0.0); break; case TIndexBuildInfo::EState::Rejection_Applying: index.SetState(Ydb::Table::IndexBuildState::STATE_REJECTION); @@ -245,14 +245,14 @@ protected: break; case TIndexBuildInfo::EState::Rejection_Unlocking: index.SetState(Ydb::Table::IndexBuildState::STATE_REJECTION); - index.SetProgress(0.0); + index.SetProgress(0.0); break; case TIndexBuildInfo::EState::Rejected: index.SetState(Ydb::Table::IndexBuildState::STATE_REJECTED); - index.SetProgress(0.0); + index.SetProgress(0.0); break; case TIndexBuildInfo::EState::Invalid: - index.SetState(Ydb::Table::IndexBuildState::STATE_UNSPECIFIED); + index.SetState(Ydb::Table::IndexBuildState::STATE_UNSPECIFIED); break; } @@ -266,22 +266,22 @@ protected: Ydb::Table::TableIndex& index = *settings.mutable_index(); index.set_name(indexInfo->IndexName); - *index.mutable_index_columns() = { - indexInfo->IndexColumns.begin(), - indexInfo->IndexColumns.end() - }; - - *index.mutable_data_columns() = { - indexInfo->DataColumns.begin(), - indexInfo->DataColumns.end() - }; - + *index.mutable_index_columns() = { + indexInfo->IndexColumns.begin(), + indexInfo->IndexColumns.end() + }; + + *index.mutable_data_columns() = { + indexInfo->DataColumns.begin(), + indexInfo->DataColumns.end() + }; + switch (indexInfo->IndexType) { case NKikimrSchemeOp::EIndexType::EIndexTypeGlobal: - *index.mutable_global_index() = Ydb::Table::GlobalIndex(); + *index.mutable_global_index() = Ydb::Table::GlobalIndex(); break; case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: - *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); break; case NKikimrSchemeOp::EIndexType::EIndexTypeInvalid: Y_FAIL("Unreachable"); diff --git a/ydb/core/tx/schemeshard/schemeshard_identificators.h b/ydb/core/tx/schemeshard/schemeshard_identificators.h index e4f46d348c..8347207152 100644 --- a/ydb/core/tx/schemeshard/schemeshard_identificators.h +++ b/ydb/core/tx/schemeshard/schemeshard_identificators.h @@ -85,11 +85,11 @@ class TIndexBuildIdTag {}; using TIndexBuildId = TUi64Id<TIndexBuildIdTag, Max<ui64>()>; constexpr TIndexBuildId InvalidIndexBuildId = TIndexBuildId(); -enum class EIndexColumnKind : ui8 { - KeyColumn = 0, - DataColumn = 1 -}; - +enum class EIndexColumnKind : ui8 { + KeyColumn = 0, + DataColumn = 1 +}; + class TPipeMessageId: public std::pair<ui64, ui64> { using TBase = std::pair<ui64, ui64>; public: diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 86a0577c84..0715543a22 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -14,8 +14,8 @@ namespace NKikimr { namespace NSchemeShard { -const ui64 NEW_TABLE_ALTER_VERSION = 1; - +const ui64 NEW_TABLE_ALTER_VERSION = 1; + namespace { bool ResolvePoolNames( @@ -1353,13 +1353,13 @@ void TSchemeShard::PersistTableIndex(NIceDb::TNiceDb& db, const TPathId& pathId) db.Table<Schema::TableIndexKeysAlterData>().Key(elemnt->PathId.LocalPathId, keyIdx).Delete(); } - - for (ui32 dataColIdx = 0; dataColIdx < alterData->IndexDataColumns.size(); ++dataColIdx) { + + for (ui32 dataColIdx = 0; dataColIdx < alterData->IndexDataColumns.size(); ++dataColIdx) { db.Table<Schema::TableIndexDataColumns>().Key(elemnt->PathId.OwnerId, elemnt->PathId.LocalPathId, dataColIdx).Update( - NIceDb::TUpdate<Schema::TableIndexDataColumns::DataColumnName>(alterData->IndexDataColumns[dataColIdx])); - + NIceDb::TUpdate<Schema::TableIndexDataColumns::DataColumnName>(alterData->IndexDataColumns[dataColIdx])); + db.Table<Schema::TableIndexDataColumnsAlterData>().Key(elemnt->PathId.OwnerId, elemnt->PathId.LocalPathId, dataColIdx).Delete(); - } + } } void TSchemeShard::PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId& pathId) { @@ -1384,11 +1384,11 @@ void TSchemeShard::PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId db.Table<Schema::TableIndexKeysAlterData>().Key(elemnt->PathId.LocalPathId, keyIdx).Update( NIceDb::TUpdate<Schema::TableIndexKeysAlterData::KeyName>(alterData->IndexKeys[keyIdx])); } - - for (ui32 dataColIdx = 0; dataColIdx < alterData->IndexDataColumns.size(); ++dataColIdx) { + + for (ui32 dataColIdx = 0; dataColIdx < alterData->IndexDataColumns.size(); ++dataColIdx) { db.Table<Schema::TableIndexDataColumnsAlterData>().Key(elemnt->PathId.OwnerId, elemnt->PathId.LocalPathId, dataColIdx).Update( - NIceDb::TUpdate<Schema::TableIndexDataColumnsAlterData::DataColumnName>(alterData->IndexDataColumns[dataColIdx])); - } + NIceDb::TUpdate<Schema::TableIndexDataColumnsAlterData::DataColumnName>(alterData->IndexDataColumns[dataColIdx])); + } } void TSchemeShard::PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId) { @@ -2170,7 +2170,7 @@ void TSchemeShard::PersistTableAlterVersion(NIceDb::TNiceDb& db, const TPathId p void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId, const TTableInfo::TPtr tableInfo) { TString partitionConfig; Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->PartitionConfig().SerializeToString(&partitionConfig); - + TString ttlSettings; if (tableInfo->HasTTLSettings()) { Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->TTLSettings().SerializeToString(&ttlSettings); @@ -3272,20 +3272,20 @@ void TSchemeShard::PersistRemoveTableIndex(NIceDb::TNiceDb &db, TPathId pathId) db.Table<Schema::MigratedTableIndexKeys>().Key(pathId.OwnerId, pathId.LocalPathId, kNo).Delete(); } - for (ui32 dNo = 0; dNo < index->IndexDataColumns.size(); ++dNo) { - db.Table<Schema::TableIndexDataColumns>().Key(pathId.OwnerId, pathId.LocalPathId, dNo).Delete(); - } - + for (ui32 dNo = 0; dNo < index->IndexDataColumns.size(); ++dNo) { + db.Table<Schema::TableIndexDataColumns>().Key(pathId.OwnerId, pathId.LocalPathId, dNo).Delete(); + } + if (index->AlterData) { auto alterData = index->AlterData; for (ui32 kNo = 0; kNo < alterData->IndexKeys.size(); ++kNo) { db.Table<Schema::TableIndexKeysAlterData>().Key(pathId.LocalPathId, kNo).Delete(); } - for (ui32 dNo = 0; dNo < alterData->IndexDataColumns.size(); ++dNo) { - db.Table<Schema::TableIndexDataColumnsAlterData>().Key(pathId.OwnerId, pathId.LocalPathId, dNo).Delete(); - } - + for (ui32 dNo = 0; dNo < alterData->IndexDataColumns.size(); ++dNo) { + db.Table<Schema::TableIndexDataColumnsAlterData>().Key(pathId.OwnerId, pathId.LocalPathId, dNo).Delete(); + } + db.Table<Schema::TableIndexAlterData>().Key(pathId.LocalPathId).Delete(); } @@ -3640,7 +3640,7 @@ TSchemeShard::TSchemeShard(const TActorId &tablet, TTabletStorageInfo *info) , PipeTracker(*PipeClientCache) , CompactionStarter(this) , ShardDeleter(info->TabletID) - , AllowDataColumnForIndexTable(0, 0, 1) + , AllowDataColumnForIndexTable(0, 0, 1) , EnableAsyncIndexes(0, 0, 1) , EnableSchemeTransactionsAtSchemeShard(0, 0, 1) { @@ -3683,9 +3683,9 @@ NTabletPipe::TClientConfig TSchemeShard::GetPipeClientConfig() { } void TSchemeShard::FillTableSchemaVersion(ui64 tableSchemaVersion, NKikimrSchemeOp::TTableDescription* tableDescr) const { - tableDescr->SetTableSchemaVersion(tableSchemaVersion); -} - + tableDescr->SetTableSchemaVersion(tableSchemaVersion); +} + void TSchemeShard::BreakTabletAndRestart(const TActorContext &ctx) { Become(&TThis::BrokenState); ctx.Send(Tablet(), new TEvents::TEvPoisonPill); @@ -3756,8 +3756,8 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { appData->Icb->RegisterSharedControl(AllowConditionalEraseOperations, "SchemeShard_AllowConditionalEraseOperations"); AllowDataColumnForIndexTable = appData->FeatureFlags.GetEnableDataColumnForIndexTable(); - appData->Icb->RegisterSharedControl(AllowDataColumnForIndexTable, "SchemeShard_AllowDataColumnForIndexTable"); - + appData->Icb->RegisterSharedControl(AllowDataColumnForIndexTable, "SchemeShard_AllowDataColumnForIndexTable"); + EnableAsyncIndexes = appData->FeatureFlags.GetEnableAsyncIndexes(); appData->Icb->RegisterSharedControl(EnableAsyncIndexes, "SchemeShard_EnableAsyncIndexes"); @@ -3771,7 +3771,7 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { AllowServerlessStorageBilling = appData->FeatureFlags.GetAllowServerlessStorageBillingForSchemeShard(); appData->Icb->RegisterSharedControl(AllowServerlessStorageBilling, "SchemeShard_AllowServerlessStorageBilling"); - TxAllocatorClient = RegisterWithSameMailbox(CreateTxAllocatorClient(CollectTxAllocators(appData))); + TxAllocatorClient = RegisterWithSameMailbox(CreateTxAllocatorClient(CollectTxAllocators(appData))); SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release()); @@ -5536,7 +5536,7 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T NKikimrTxDataShard::TFlatSchemeTransaction tx; FillSeqNo(tx, seqNo); auto proto = tx.MutableAlterTable(); - FillTableSchemaVersion(alterData->AlterVersion, proto); + FillTableSchemaVersion(alterData->AlterVersion, proto); proto->SetName(path->Name); proto->SetId_Deprecated(pathId.LocalPathId); @@ -5729,7 +5729,7 @@ void TSchemeShard::FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 std::move(rangeBegin), std::move(rangeEnd), true /* rangeBeginInclusive */, false /* rangeEndInclusive */, true /* newTable */); - FillTableSchemaVersion(schemaVersion, tableDescr); + FillTableSchemaVersion(schemaVersion, tableDescr); } bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeId firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 09bc8c42a3..e1a1d08b9a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -55,8 +55,8 @@ namespace NKikimr { namespace NSchemeShard { -extern const ui64 NEW_TABLE_ALTER_VERSION; - +extern const ui64 NEW_TABLE_ALTER_VERSION; + class TSchemeShard : public TActor<TSchemeShard> , public NTabletFlatExecutor::TTabletExecutedFlat @@ -113,7 +113,7 @@ public: TControlWrapper AllowConditionalEraseOperations; TControlWrapper AllowServerlessStorageBilling; - + TSplitSettings SplitSettings; struct TTenantInitState { @@ -727,7 +727,7 @@ public: TString FillAlterTableTxBody(TPathId tableId, TShardIdx shardIdx, TMessageSeqNo seqNo) const; TString FillBackupTxBody(TPathId pathId, const NKikimrSchemeOp::TBackupTask& task, ui32 shardNum, TMessageSeqNo seqNo) const; - + static void FillSeqNo(NKikimrTxDataShard::TFlatSchemeTransaction &tx, TMessageSeqNo seqNo); static void FillSeqNo(NKikimrTxColumnShard::TSchemaTxBody &tx, TMessageSeqNo seqNo); @@ -946,7 +946,7 @@ public: void FillTableSchemaVersion(ui64 schemaVersion, NKikimrSchemeOp::TTableDescription *tableDescr) const; // namespace NIndexBuilder { - TControlWrapper AllowDataColumnForIndexTable; + TControlWrapper AllowDataColumnForIndexTable; TControlWrapper EnableAsyncIndexes; TControlWrapper EnableSchemeTransactionsAtSchemeShard; diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 055aca1129..947870d51b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -300,7 +300,7 @@ private: << ": info# " << importInfo->ToString() << ", item# " << item.ToString(itemIdx)); - ctx.RegisterWithSameMailbox(CreateSchemeGetter(Self->SelfId(), importInfo, itemIdx)); + ctx.RegisterWithSameMailbox(CreateSchemeGetter(Self->SelfId(), importInfo, itemIdx)); } void CreateTable(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 1664a554c0..e2fc78da5d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -164,7 +164,7 @@ public: Send(Client, new TEvents::TEvPoisonPill()); } - Client = RegisterWithSameMailbox(CreateS3Wrapper(Credentials, Config)); + Client = RegisterWithSameMailbox(CreateS3Wrapper(Credentials, Config)); HeadObject(SchemeKey); Become(&TThis::StateWork); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 48c9dca888..fcb564b606 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -385,21 +385,21 @@ bool TPartitionConfigMerger::ApplyChanges( return false; } - if (changes.HasFreezeState()) { + if (changes.HasFreezeState()) { if (changes.GetFreezeState() == NKikimrSchemeOp::EFreezeState::Unspecified) { - errDesr = TStringBuilder() << "Unexpected freeze state"; - return false; - } - TVector<const NProtoBuf::FieldDescriptor*> fields; - auto reflection = changes.GetReflection(); - reflection->ListFields(changes, &fields); - if (fields.size() > 1) { - errDesr = TStringBuilder() - << "Mix freeze cmd with other options is forbiden"; - return false; - } - } - + errDesr = TStringBuilder() << "Unexpected freeze state"; + return false; + } + TVector<const NProtoBuf::FieldDescriptor*> fields; + auto reflection = changes.GetReflection(); + reflection->ListFields(changes, &fields); + if (fields.size() > 1) { + errDesr = TStringBuilder() + << "Mix freeze cmd with other options is forbiden"; + return false; + } + } + if (changes.HasCompactionPolicy()) { result.MutableCompactionPolicy()->Clear(); result.MutableCompactionPolicy()->CopyFrom(changes.GetCompactionPolicy()); @@ -1060,11 +1060,11 @@ bool TPartitionConfigMerger::VerifyAlterParams( } if (!VerifyCommandOnFrozenTable(srcConfig, dstConfig)) { - errDescr = TStringBuilder() << + errDescr = TStringBuilder() << "Table is frozen. Only unfreeze alter is allowed"; - return false; - } - + return false; + } + return true; } @@ -1098,14 +1098,14 @@ bool TPartitionConfigMerger::VerifyCommandOnFrozenTable(const NKikimrSchemeOp::T srcConfig.GetFreezeState() == NKikimrSchemeOp::EFreezeState::Freeze) { if (dstConfig.HasFreezeState() && dstConfig.GetFreezeState() == NKikimrSchemeOp::EFreezeState::Unfreeze) { - // Only unfreeze cmd is allowd - return true; - } - return false; - } - return true; -} - + // Only unfreeze cmd is allowd + return true; + } + return false; + } + return true; +} + void TTableInfo::FinishAlter() { Y_VERIFY(AlterData, "No alter data at Alter complete"); AlterVersion = AlterData->AlterVersion; @@ -1229,7 +1229,7 @@ void TTableInfo::DeserializeAlterExtraData(const TString& str) { void TTableInfo::SetPartitioning(TVector<TTableShardInfo>&& newPartitioning) { THashMap<TShardIdx, TPartitionStats> newPartitionStats; TPartitionStats newAggregatedStats; - newAggregatedStats.PartCount = newPartitioning.size(); + newAggregatedStats.PartCount = newPartitioning.size(); ui64 cpuTotal = 0; for (const auto& np : newPartitioning) { auto idx = np.ShardIdx; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 754340fc81..2e5710c3d3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2083,8 +2083,8 @@ struct TTableIndexInfo : public TSimpleRefCount<TTableIndexInfo> { TPtr alterData = result->CreateNextVersion(); alterData->IndexKeys.assign(config.GetKeyColumnNames().begin(), config.GetKeyColumnNames().end()); Y_VERIFY(alterData->IndexKeys.size()); - alterData->IndexDataColumns.assign(config.GetDataColumnNames().begin(), config.GetDataColumnNames().end()); - alterData->State = config.HasState() ? config.GetState() : EState::EIndexStateReady; + alterData->IndexDataColumns.assign(config.GetDataColumnNames().begin(), config.GetDataColumnNames().end()); + alterData->State = config.HasState() ? config.GetState() : EState::EIndexStateReady; return result; } @@ -2094,7 +2094,7 @@ struct TTableIndexInfo : public TSimpleRefCount<TTableIndexInfo> { EState State; TVector<TString> IndexKeys; - TVector<TString> IndexDataColumns; + TVector<TString> IndexDataColumns; TTableIndexInfo::TPtr AlterData = nullptr; }; @@ -2524,7 +2524,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> { TString IndexName; TVector<TString> IndexColumns; - TVector<TString> DataColumns; + TVector<TString> DataColumns; TString ImplTablePath; NTableIndex::TTableColumns ImplTableColumns; @@ -2629,15 +2629,15 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> { Y_VERIFY(!IsFinished()); Subscribers.insert(actorID); } - - float CalcProgressPercent() const { - if (Shards) { - float totalShards = Shards.size(); - return 100.0 * DoneShards.size() / totalShards; - } - // No shards - no progress - return 0.0; - } + + float CalcProgressPercent() const { + if (Shards) { + float totalShards = Shards.size(); + return 100.0 * DoneShards.size() / totalShards; + } + // No shards - no progress + return 0.0; + } }; bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, @@ -2687,9 +2687,9 @@ inline void Out<NKikimr::NSchemeShard::TIndexBuildInfo> for (const auto& x: info.IndexColumns) { o << ", IndexColumn: " << x; } - for (const auto& x: info.DataColumns) { - o << ", DataColumns: " << x; - } + for (const auto& x: info.DataColumns) { + o << ", DataColumns: " << x; + } o << ", State: " << info.State; o << ", IsCancellationRequested: " << info.CancelRequested; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index a0423e98a0..3789d3af1b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -566,9 +566,9 @@ void TPathDescriber::DescribeUserAttributes(TPathElement::TPtr pathEl) { } void TPathDescriber::DescribePathVersion(const TPath& path) { - const auto& version = Self->GetPathVersion(path); - Result->Record.MutablePathDescription()->MutableSelf()->SetPathVersion(version.GetGeneralVersion()); - Result->Record.MutablePathDescription()->MutableSelf()->MutableVersion()->CopyFrom(version); + const auto& version = Self->GetPathVersion(path); + Result->Record.MutablePathDescription()->MutableSelf()->SetPathVersion(version.GetGeneralVersion()); + Result->Record.MutablePathDescription()->MutableSelf()->MutableVersion()->CopyFrom(version); } void TPathDescriber::DescribeDomain(TPathElement::TPtr pathEl) { @@ -899,7 +899,7 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme THashMap<ui32, TString> familyNames; bool familyNamesBuilt = false; - entry->SetTableSchemaVersion(tableInfo->AlterVersion); + entry->SetTableSchemaVersion(tableInfo->AlterVersion); entry->MutableColumns()->Reserve(tableInfo->Columns.size()); for (auto col : tableInfo->Columns) { const auto& cinfo = col.second; @@ -940,9 +940,9 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme } } Y_VERIFY(!tableInfo->KeyColumnIds.empty()); - - entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size()); - entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size()); + + entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size()); + entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size()); for (ui32 keyColId : tableInfo->KeyColumnIds) { entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name); entry->AddKeyColumnIds(keyColId); diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 9668d921cd..f293136fe8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -837,7 +837,7 @@ struct Schema : NIceDb::Schema { struct VolumeConfig: Column<2, NScheme::NTypeIds::String> {}; struct AlterVersion: Column<3, NScheme::NTypeIds::Uint64> {}; struct MountToken: Column<4, NScheme::NTypeIds::Utf8> {}; - struct TokenVersion: Column<5, NScheme::NTypeIds::Uint64> {}; + struct TokenVersion: Column<5, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<PathId>; using TColumns = TableColumns<PathId, VolumeConfig, AlterVersion, MountToken, TokenVersion>; @@ -1289,14 +1289,14 @@ struct Schema : NIceDb::Schema { struct Id: Column<1, NScheme::NTypeIds::Uint64> {using Type = TIndexBuildId;}; struct ColumnNo: Column<2, NScheme::NTypeIds::Uint32> {}; struct ColumnName: Column<3, NScheme::NTypeIds::Utf8> {}; - struct ColumnKind: Column<4, NScheme::NTypeIds::Uint8> {using Type = EIndexColumnKind;}; + struct ColumnKind: Column<4, NScheme::NTypeIds::Uint8> {using Type = EIndexColumnKind;}; using TKey = TableKey<Id, ColumnNo>; using TColumns = TableColumns< Id, ColumnNo, - ColumnName, - ColumnKind + ColumnName, + ColumnKind >; }; @@ -1369,26 +1369,26 @@ struct Schema : NIceDb::Schema { >; }; - struct TableIndexDataColumns : Table<80> { - struct PathOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; - struct PathLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; - struct DataColumnId: Column<3, NScheme::NTypeIds::Uint32> {}; - struct DataColumnName: Column<4, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<PathOwnerId, PathLocalId, DataColumnId>; - using TColumns = TableColumns<PathOwnerId, PathLocalId, DataColumnId, DataColumnName>; - }; - - struct TableIndexDataColumnsAlterData : Table<81> { - struct PathOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; - struct PathLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; - struct DataColumnId: Column<3, NScheme::NTypeIds::Uint32> {}; - struct DataColumnName: Column<4, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<PathOwnerId, PathLocalId, DataColumnId>; - using TColumns = TableColumns<PathOwnerId, PathLocalId, DataColumnId, DataColumnName>; - }; - + struct TableIndexDataColumns : Table<80> { + struct PathOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; + struct PathLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; + struct DataColumnId: Column<3, NScheme::NTypeIds::Uint32> {}; + struct DataColumnName: Column<4, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<PathOwnerId, PathLocalId, DataColumnId>; + using TColumns = TableColumns<PathOwnerId, PathLocalId, DataColumnId, DataColumnName>; + }; + + struct TableIndexDataColumnsAlterData : Table<81> { + struct PathOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; + struct PathLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; + struct DataColumnId: Column<3, NScheme::NTypeIds::Uint32> {}; + struct DataColumnName: Column<4, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<PathOwnerId, PathLocalId, DataColumnId>; + using TColumns = TableColumns<PathOwnerId, PathLocalId, DataColumnId, DataColumnName>; + }; + struct RestoreTasks : Table<82> { struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId;}; @@ -1633,7 +1633,7 @@ struct Schema : NIceDb::Schema { Exports, ExportItems, TableShardPartitionConfigs, - PublishingPaths, + PublishingPaths, FillIndexDesc, MigratedPaths, MigratedUserAttributes, @@ -1665,8 +1665,8 @@ struct Schema : NIceDb::Schema { TablePartitionStats, SubDomainSchemeQuotas, FileStoreInfos, - FileStoreAlters, - TableIndexDataColumns, + FileStoreAlters, + TableIndexDataColumns, TableIndexDataColumnsAlterData, RestoreTasks, MigratedKesusInfos, diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index 6c8df37d05..41a8045155 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -75,7 +75,7 @@ struct TTxState { item(TxCreateExtSubDomain, 30) \ item(TxMergeTablePartition, 31) \ item(TxAlterExtSubDomain, 32) \ - item(TxForceDropExtSubDomain, 33) \ + item(TxForceDropExtSubDomain, 33) \ item(TxFillIndex, 34) \ item(TxUpgradeSubDomain, 35) \ item(TxUpgradeSubDomainDecision, 36) \ @@ -304,7 +304,7 @@ struct TTxState { case TxCreateSolomonVolume: case TxCreateRtmrVolume: case TxCreateTableIndex: - case TxFillIndex: + case TxFillIndex: case TxCreateCdcStream: case TxCreateSequence: case TxCreateReplication: @@ -402,7 +402,7 @@ struct TTxState { case TxCreateSolomonVolume: case TxCreateRtmrVolume: case TxCreateTableIndex: - case TxFillIndex: + case TxFillIndex: case TxCreateCdcStream: case TxCreateCdcStreamAtTable: case TxCreateSequence: @@ -511,7 +511,7 @@ struct TTxState { case TxUpgradeSubDomain: case TxUpgradeSubDomainDecision: case TxAlterUserAttributes: - case TxFillIndex: + case TxFillIndex: case TxAlterTableIndex: case TxAlterSolomonVolume: case TxAlterCdcStream: diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index 5cb6025896..4ee28e3543 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -215,13 +215,13 @@ TTableColumns ExtractInfo(const NKikimrSchemeOp::TTableDescription &tableDesrc) } TIndexColumns ExtractInfo(const NKikimrSchemeOp::TIndexCreationConfig &indexDesc) { - NTableIndex::TIndexColumns result; + NTableIndex::TIndexColumns result; for (auto& keyName: indexDesc.GetKeyColumnNames()) { - result.KeyColumns.push_back(keyName); + result.KeyColumns.push_back(keyName); + } + for (auto& keyName: indexDesc.GetDataColumnNames()) { + result.DataColumns.push_back(keyName); } - for (auto& keyName: indexDesc.GetDataColumnNames()) { - result.DataColumns.push_back(keyName); - } return result; } diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index 980aa1f923..a76dcf9d5b 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -681,8 +681,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { "Columns { Name: \"key2\" Type: \"Utf8\"}" "Columns { Name: \"RowId\" Type: \"Uint64\"}" "Columns { Name: \"Value\" Type: \"Utf8\"}" - "Columns { Name: \"YaValue\" Type: \"Yson\"}" - "Columns { Name: \"MoreValue\" Type: \"Json\"}" + "Columns { Name: \"YaValue\" Type: \"Yson\"}" + "Columns { Name: \"MoreValue\" Type: \"Json\"}" "KeyColumnNames: [\"RowId\", \"key1\", \"key2\"]" ); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", @@ -691,19 +691,19 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { "KeyColumnNames: [\"key\"]" ); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", - "Name: \"Table3\"" - "Columns { Name: \"RowId\" Type: \"Yson\"}" + "Name: \"Table3\"" + "Columns { Name: \"RowId\" Type: \"Yson\"}" "Columns { Name: \"Value\" Type: \"Utf8\"}" "KeyColumnNames: [\"RowId\"]"); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", - "Name: \"Table3\"" - "Columns { Name: \"RowId\" Type: \"Json\"}" + "Name: \"Table3\"" + "Columns { Name: \"RowId\" Type: \"Json\"}" "Columns { Name: \"Value\" Type: \"Utf8\"}" "KeyColumnNames: [\"RowId\"]"); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", - "Name: \"Table3\"" - "Columns { Name: \"RowId\" Type: \"Json\"}" - "Columns { Name: \"key1\" Type: \"Uint32\"}" + "Name: \"Table3\"" + "Columns { Name: \"RowId\" Type: \"Json\"}" + "Columns { Name: \"key1\" Type: \"Uint32\"}" "Columns { Name: \"Value\" Type: \"Utf8\"}" "KeyColumnNames: [\"RowId\", \"key1\"]"); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", @@ -1052,11 +1052,11 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { Name: "UserDefinedIndexByValues" KeyColumnNames: ["value0", "value1"] } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } )"); TestModificationResult(runtime, txId-2, NKikimrScheme::StatusAccepted); @@ -1069,8 +1069,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::PathVersionEqual(7), NLs::Finished, NLs::PathExist, - NLs::PathsInsideDomain(9), - NLs::ShardsInsideDomain(5), + NLs::PathsInsideDomain(9), + NLs::ShardsInsideDomain(5), NLs::ChildrenCount(2) }); @@ -1100,8 +1100,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::PathExist, NLs::Finished, NLs::PathExist, - NLs::PathsInsideDomain(17), - NLs::ShardsInsideDomain(10)}); + NLs::PathsInsideDomain(17), + NLs::ShardsInsideDomain(10)}); TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { @@ -1119,8 +1119,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::PathVersionEqual(15), NLs::PathExist, NLs::Finished, - NLs::PathsInsideDomain(25), - NLs::ShardsInsideDomain(15), + NLs::PathsInsideDomain(25), + NLs::ShardsInsideDomain(15), NLs::ChildrenCount(6) }); @@ -1153,15 +1153,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { }); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1"), - {NLs::Finished, + {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), - NLs::IndexKeys({"value0"}), - NLs::IndexDataColumns({"value1"})}); - + NLs::IndexKeys({"value0"}), + NLs::IndexDataColumns({"value1"})}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), - {NLs::Finished}); - + {NLs::Finished}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/seconddst1"), {NLs::PathVersionEqual(3), NLs::PathExist, @@ -1174,7 +1174,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::PathVersionEqual(3), NLs::PathExist, NLs::Finished, - NLs::IndexesCount(3), + NLs::IndexesCount(3), NLs::CreatedAt(txId) }); @@ -1195,8 +1195,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::PathVersionEqual(19), NLs::PathExist, NLs::Finished, - NLs::PathsInsideDomain(27), - NLs::ShardsInsideDomain(17), + NLs::PathsInsideDomain(27), + NLs::ShardsInsideDomain(17), NLs::ChildrenCount(8) }); @@ -1745,11 +1745,11 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { Name: "UserDefinedIndexByValues" KeyColumnNames: ["value0", "value1"] } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } )"); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/Table1"), {NLs::NotFinished, @@ -1768,7 +1768,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/Table1"), {NLs::Finished, NLs::PathVersionEqual(3), - NLs::IndexesCount(4)}); + NLs::IndexesCount(4)}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), @@ -1791,17 +1791,17 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValues/indexImplTable"), {NLs::Finished}); - + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1"), - {NLs::Finished, + {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), - NLs::IndexKeys({"value0"}), - NLs::IndexDataColumns({"value1"})}); - + NLs::IndexKeys({"value0"}), + NLs::IndexDataColumns({"value1"})}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), - {NLs::Finished}); - + {NLs::Finished}); + TestDropTable(runtime, ++txId, "/MyRoot/DirA", "Table1"); env.TestWaitNotification(runtime, 103); @@ -1836,11 +1836,11 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { Name: "UserDefinedIndexByValue1" KeyColumnNames: ["value1"] } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } )"); env.TestWaitNotification(runtime, {txId, txId-1}); @@ -1851,7 +1851,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/copy"), {NLs::Finished, NLs::PathVersionEqual(3), - NLs::IndexesCount(3)}); + NLs::IndexesCount(3)}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), @@ -1866,16 +1866,16 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::IndexKeys({"value1"})}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue1/indexImplTable"), {NLs::Finished}); - + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1"), - {NLs::Finished, + {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), - NLs::IndexKeys({"value0"}), - NLs::IndexDataColumns({"value1"})}); - + NLs::IndexKeys({"value0"}), + NLs::IndexDataColumns({"value1"})}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), - {NLs::Finished}); + {NLs::Finished}); } Y_UNIT_TEST(CreateIndexedTableRejects) { //+ @@ -2033,81 +2033,81 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( - TableDescription { - Name: "Table2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Uint64" } - Columns { Name: "value1" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value0"] - } + TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Uint64" } + Columns { Name: "value1" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value0"] + } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( - TableDescription { - Name: "Table2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Uint64" } - Columns { Name: "value1" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value0", "value1"] - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Uint64" } + Columns { Name: "value1" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value0", "value1"] + } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( - TableDescription { - Name: "Table2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Uint64" } - Columns { Name: "value1" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["key"] - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Uint64" } + Columns { Name: "value1" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["key"] + } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( - TableDescription { - Name: "Table2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Uint64" } - Columns { Name: "value1" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["key", "value1"] - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Uint64" } + Columns { Name: "value1" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["key", "value1"] + } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( - TableDescription { - Name: "Table2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Uint64" } - Columns { Name: "value1" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["blabla"] - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot/DirA", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Uint64" } + Columns { Name: "value1" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["blabla"] + } )", {TEvSchemeShard::EStatus::StatusInvalidParameter}); - + TestCreateTable(runtime, ++txId, "/MyRoot/DirA/Table1", R"( Name: "inside_table" Columns { Name: "key" Type: "Uint64" } diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp index 86056c67bd..5e1e42d17c 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp @@ -9,17 +9,17 @@ using namespace NSchemeShardUT_Private::NExportReboots; using namespace NKikimr::NWrappers::NTestHelpers; Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) { - using TUnderlying = std::function<void(const TVector<TString>&, const TString&, TTestWithReboots&)>; + using TUnderlying = std::function<void(const TVector<TString>&, const TString&, TTestWithReboots&)>; void Decorate(const TVector<TString>& tables, const TString& request, TUnderlying func) { TPortManager portManager; const ui16 port = portManager.GetPort(); - TTestWithReboots t; + TTestWithReboots t; TS3Mock s3Mock({}, TS3Mock::TSettings(port)); UNIT_ASSERT(s3Mock.Start()); - func(tables, Sprintf(request.c_str(), port), t); + func(tables, Sprintf(request.c_str(), port), t); } void RunS3(const TVector<TString>& tables, const TString& request) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp index 303e0568bc..a2d876fd05 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp @@ -5,7 +5,7 @@ namespace NSchemeShardUT_Private { namespace NExportReboots { -void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { +void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { { TInactiveZone inactive(activeZone); @@ -43,7 +43,7 @@ void Run(const TVector<TString>& tables, const TString& request, TTestWithReboot }); } -void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { +void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { { TInactiveZone inactive(activeZone); @@ -86,7 +86,7 @@ void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReb }); } -void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { +void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { { TInactiveZone inactive(activeZone); diff --git a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h index 69400481c3..a7f869c585 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h +++ b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h @@ -4,14 +4,14 @@ #include <util/generic/vector.h> namespace NSchemeShardUT_Private { - -class TTestWithReboots; - + +class TTestWithReboots; + namespace NExportReboots { -void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); -void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); -void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); +void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); +void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); +void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); } // NExportReboots } // NSchemeShardUT_Private diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 7eff635d64..6f999b268b 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -1493,14 +1493,14 @@ namespace NSchemeShardUT_Private { AsyncUpgradeSubDomainDecision(runtime, txId, parentPath, name, decision); TestModificationResults(runtime, txId, {TEvSchemeShard::EStatus::StatusAccepted}); } - - TRowVersion CreateVolatileSnapshot( - TTestActorRuntime& runtime, - const TVector<TString>& tables, + + TRowVersion CreateVolatileSnapshot( + TTestActorRuntime& runtime, + const TVector<TString>& tables, TDuration timeout) { TActorId sender = runtime.AllocateEdgeActor(); - + { auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); auto* tx = request->Record.MutableTransaction()->MutableCreateVolatileSnapshot(); @@ -1509,7 +1509,7 @@ namespace NSchemeShardUT_Private { } tx->SetTimeoutMs(timeout.MilliSeconds()); runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); - } + } auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); const auto& record = ev->Get()->Record; @@ -1523,8 +1523,8 @@ namespace NSchemeShardUT_Private { "Unexpected step " << step << " and txId " << txId); return { step, txId }; - } - + } + TEvIndexBuilder::TEvCreateRequest* CreateBuildIndexRequest(ui64 id, const TString& dbName, const TString& src, const TBuildIndexConfig& cfg) { NKikimrIndexBuilder::TIndexBuildSettings settings; settings.set_source_path(src); @@ -1535,7 +1535,7 @@ namespace NSchemeShardUT_Private { index.set_name(cfg.IndexName); *index.mutable_index_columns() = {cfg.IndexColumns.begin(), cfg.IndexColumns.end()}; *index.mutable_data_columns() = {cfg.DataColumns.begin(), cfg.DataColumns.end()}; - + switch (cfg.IndexType) { case NKikimrSchemeOp::EIndexTypeGlobal: *index.mutable_global_index() = Ydb::Table::GlobalIndex(); @@ -1545,7 +1545,7 @@ namespace NSchemeShardUT_Private { break; default: UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(cfg.IndexType)); - } + } return new TEvIndexBuilder::TEvCreateRequest(id, dbName, std::move(settings)); } @@ -2055,4 +2055,4 @@ namespace NSchemeShardUT_Private { runtime.SetObserverFunc(prevObserver); } -} +} diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index b90b176701..6b12c25534 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -386,11 +386,11 @@ namespace NSchemeShardUT_Private { }); } - TRowVersion CreateVolatileSnapshot( - TTestActorRuntime& runtime, - const TVector<TString>& tables, - TDuration timeout); - + TRowVersion CreateVolatileSnapshot( + TTestActorRuntime& runtime, + const TVector<TString>& tables, + TDuration timeout); + TPathId TestFindTabletSubDomainPathId( TTestActorRuntime& runtime, ui64 tabletId, NKikimrScheme::TEvFindTabletSubDomainPathIdResult::EStatus expected = NKikimrScheme::TEvFindTabletSubDomainPathIdResult::SUCCESS); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 7d3bbdaab7..c5d6c5a973 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -675,12 +675,12 @@ TCheckFunc IndexKeys(const TVector<TString>& keyNames) { TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().DataColumnNamesSize(), dataColumnNames.size()); - for (ui32 colId = 0; colId < dataColumnNames.size(); ++colId) { + for (ui32 colId = 0; colId < dataColumnNames.size(); ++colId) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetDataColumnNames(colId), dataColumnNames.at(colId)); - } - }; -} - + } + }; +} + void NoChildren(const NKikimrScheme::TEvDescribeSchemeResult& record) { ChildrenCount(0)(record); } diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index b0ca5889ad..33022e4257 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -115,7 +115,7 @@ namespace NLs { TCheckFunc IndexType(NKikimrSchemeOp::EIndexType type); TCheckFunc IndexState(NKikimrSchemeOp::EIndexState state); TCheckFunc IndexKeys(const TVector<TString>& keyNames); - TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames); + TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames); TCheckFunc HasBackupInFly(ui64 txId); void NoBackupInFly(const NKikimrScheme::TEvDescribeSchemeResult& record); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 5d388c47c9..9cfd8037b7 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -494,7 +494,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe TAppPrepare app(dsExportFactory ? dsExportFactory : static_cast<std::shared_ptr<NKikimr::NDataShard::IExportFactory>>(std::make_shared<TDataShardExportFactory>())); - app.SetEnableDataColumnForIndexTable(true); + app.SetEnableDataColumnForIndexTable(true); app.SetEnableSystemViews(opts.EnableSystemViews_); app.SetEnablePersistentPartitionStats(opts.EnablePersistentPartitionStats_); app.SetEnableTtlOnAsyncIndexedTables(opts.EnableTtlOnAsyncIndexedTables_); @@ -515,11 +515,11 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe SetupLogging(runtime); SetupChannelProfiles(app, TTestTxConfig::DomainUid, ChannelsCount); - - for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { + + for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { SetupSchemeCache(runtime, node, app.Domains->GetDomain(TTestTxConfig::DomainUid).Name); - } - + } + SetupTabletServices(runtime, &app); if (opts.EnablePipeRetries_) { EnableSchemeshardPipeRetriesGuard = EnableSchemeshardPipeRetries(runtime); @@ -534,12 +534,12 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe InitRootStoragePools(runtime, schemeRoot, sender, TTestTxConfig::DomainUid); - for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { - IActor* txProxy = CreateTxProxy(runtime.GetTxAllocatorTabletIds()); + for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { + IActor* txProxy = CreateTxProxy(runtime.GetTxAllocatorTabletIds()); TActorId txProxyId = runtime.Register(txProxy, node); - runtime.RegisterService(MakeTxProxyID(), txProxyId, node); - } - + runtime.RegisterService(MakeTxProxyID(), txProxyId, node); + } + //SetupBoxAndStoragePool(runtime, sender, TTestTxConfig::DomainUid); TxReliablePropose = runtime.Register(new TTxReliablePropose(schemeRoot)); @@ -653,8 +653,8 @@ TAutoPtr<ITabletScheduledEventsGuard> NSchemeShardUT_Private::TTestEnv::EnableSc } NActors::TActorId NSchemeShardUT_Private::CreateNotificationSubscriber(NActors::TTestActorRuntime &runtime, ui64 schemeshardId) { - return runtime.Register(new TTxNotificationSubscriber(schemeshardId)); -} + return runtime.Register(new TTxNotificationSubscriber(schemeshardId)); +} NActors::TActorId NSchemeShardUT_Private::CreateFakeMetering(NActors::TTestActorRuntime &runtime) { NActors::TActorId actorId = runtime.Register(new TFakeMetering()); @@ -663,13 +663,13 @@ NActors::TActorId NSchemeShardUT_Private::CreateFakeMetering(NActors::TTestActor } void NSchemeShardUT_Private::TestWaitNotification(NActors::TTestActorRuntime &runtime, TSet<ui64> txIds, TActorId subscriberActorId) { - + TActorId sender = runtime.AllocateEdgeActor(); for (ui64 txId : txIds) { Cerr << Endl << "TestWaitNotification wait txId: " << txId << Endl; auto ev = new TEvSchemeShard::TEvNotifyTxCompletion(txId); - runtime.Send(new IEventHandle(subscriberActorId, sender, ev)); + runtime.Send(new IEventHandle(subscriberActorId, sender, ev)); } TAutoPtr<IEventHandle> handle; @@ -683,14 +683,14 @@ void NSchemeShardUT_Private::TestWaitNotification(NActors::TTestActorRuntime &ru } } -void NSchemeShardUT_Private::TTestEnv::TestWaitNotification(NActors::TTestActorRuntime &runtime, TSet<ui64> txIds, ui64 schemeshardId) { - if (!TxNotificationSubcribers.contains(schemeshardId)) { - TxNotificationSubcribers[schemeshardId] = CreateNotificationSubscriber(runtime, schemeshardId); - } - - NSchemeShardUT_Private::TestWaitNotification(runtime, txIds, TxNotificationSubcribers.at(schemeshardId)); -} - +void NSchemeShardUT_Private::TTestEnv::TestWaitNotification(NActors::TTestActorRuntime &runtime, TSet<ui64> txIds, ui64 schemeshardId) { + if (!TxNotificationSubcribers.contains(schemeshardId)) { + TxNotificationSubcribers[schemeshardId] = CreateNotificationSubscriber(runtime, schemeshardId); + } + + NSchemeShardUT_Private::TestWaitNotification(runtime, txIds, TxNotificationSubcribers.at(schemeshardId)); +} + void NSchemeShardUT_Private::TTestEnv::TestWaitNotification(TTestActorRuntime &runtime, int txId, ui64 schemeshardId) { TestWaitNotification(runtime, (ui64)txId, schemeshardId); } @@ -936,10 +936,10 @@ void NSchemeShardUT_Private::TTestWithReboots::RestoreLogging() { TestEnv->SetupLogging(*Runtime); } -NSchemeShardUT_Private::TTestEnv* NSchemeShardUT_Private::TTestWithReboots::CreateTestEnv() { +NSchemeShardUT_Private::TTestEnv* NSchemeShardUT_Private::TTestWithReboots::CreateTestEnv() { return new TTestEnv(*Runtime, GetTestEnvOptions()); -} - +} + void NSchemeShardUT_Private::TTestWithReboots::Prepare(const TString &dispatchName, std::function<void (TTestActorRuntime &)> setup, bool &outActiveZone) { @@ -952,7 +952,7 @@ void NSchemeShardUT_Private::TTestWithReboots::Prepare(const TString &dispatchNa //TestEnv.Reset(new TTestEnv(*Runtime, 4, false, SchemeShardFactory)); - TestEnv.Reset(CreateTestEnv()); + TestEnv.Reset(CreateTestEnv()); RestoreLogging(); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index b6ef0241c7..72b934c570 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -19,7 +19,7 @@ namespace NSchemeShardUT_Private { void TestWaitNotification(NActors::TTestActorRuntime &runtime, TSet<ui64> txIds, TActorId subscriberActorId); NActors::TActorId CreateNotificationSubscriber(NActors::TTestActorRuntime &runtime, ui64 schemeshardId); NActors::TActorId CreateFakeMetering(NActors::TTestActorRuntime &runtime); - + struct TTestEnvOptions { using TSelf = TTestEnvOptions; @@ -112,8 +112,8 @@ namespace NSchemeShardUT_Private { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // A wrapper to run test scenarios with reboots of schemeshard, hive and coordinator - class TTestWithReboots { - public: + class TTestWithReboots { + public: TVector<ui64> TabletIds; THolder<TTestActorRuntime> Runtime; TTestEnvOptions EnvOpts; @@ -127,7 +127,7 @@ namespace NSchemeShardUT_Private { const bool KillOnCommit; explicit TTestWithReboots(bool killOnCommit = false, TTestEnv::TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard); - virtual ~TTestWithReboots() = default; + virtual ~TTestWithReboots() = default; void Run(std::function<void(TTestActorRuntime& runtime, bool& activeZone)> testScenario); void Run(std::function<void(TTestActorRuntime& runtime, bool& activeZone)> testScenario, bool allowLogBatching); @@ -145,8 +145,8 @@ namespace NSchemeShardUT_Private { void Prepare(const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone); void EnableTabletResolverScheduling(ui32 nodeIdx = 0); void Finalize(); - private: - virtual TTestEnv* CreateTestEnv(); + private: + virtual TTestEnv* CreateTestEnv(); // Make sure that user requests are not dropped static bool PassUserRequests(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); diff --git a/ydb/core/tx/schemeshard/ut_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build.cpp index a1cd7a7fe6..88860481cc 100644 --- a/ydb/core/tx/schemeshard/ut_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build.cpp @@ -1,15 +1,15 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard_billing_helpers.h> #include <ydb/core/testlib/tablet_helpers.h> - + #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/metering/metering.h> -using namespace NKikimr; +using namespace NKikimr; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(IndexBuildTest) { +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(IndexBuildTest) { Y_UNIT_TEST(ShadowDataNotAllowedByDefault) { TTestBasicRuntime runtime; TTestEnv env(runtime); @@ -289,7 +289,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { env.TestWaitNotification(runtime, txId, tenantSchemeShard); auto descr = TestGetBuilIndex(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB", txId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); const TString meteringData = R"({"usage":{"start":0,"quantity":179,"finish":0,"unit":"request_unit","type":"delta"},"tags":{},"id":"106-9437199-2-101-1818-101-1818","cloud_id":"CLOUD_ID_VAL","source_wt":0,"source_id":"sless-docapi-ydb-ss","resource_id":"DATABASE_ID_VAL","schema":"ydb.serverless.requests.v1","folder_id":"FOLDER_ID_VAL","version":"1.0.0"})"; @@ -703,7 +703,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { NLs::IndexState(NKikimrSchemeOp::EIndexState::EIndexStateReady)}); NKikimrIndexBuilder::TEvGetResponse descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", builIndexId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); // KIKIMR-9945 TestAlterTable(runtime, ++txId, "/MyRoot", R"( @@ -916,7 +916,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { env.TestWaitNotification(runtime, buildIndexId); auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_CANCELLED); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_CANCELLED); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathExist, @@ -973,7 +973,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { { auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_PREPARING); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_PREPARING); } // @@ -986,7 +986,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { { auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); } TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), @@ -997,4 +997,4 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table/index1", true, true, true), {NLs::PathExist}); } -} +} diff --git a/ydb/core/tx/schemeshard/ut_index_build/ya.make b/ydb/core/tx/schemeshard/ut_index_build/ya.make index e49923a63c..12a292282c 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ya.make +++ b/ydb/core/tx/schemeshard/ut_index_build/ya.make @@ -1,17 +1,17 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - + OWNER( vvvv g:kikimr ) - -FORK_SUBTESTS() -TIMEOUT(600) +FORK_SUBTESTS() + +TIMEOUT(600) -SIZE(MEDIUM) - -PEERDIR( +SIZE(MEDIUM) + +PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion @@ -19,12 +19,12 @@ PEERDIR( ydb/core/testlib ydb/core/tx ydb/core/tx/schemeshard/ut_helpers -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - ut_index_build.cpp -) - -END() +SRCS( + ut_index_build.cpp +) + +END() diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp index 790c3e7c96..d270fd87e6 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp @@ -6,63 +6,63 @@ using namespace NKikimr; using namespace NSchemeShard; using namespace NSchemeShardUT_Private; -static void WriteRows(TTestActorRuntime& runtime, ui64 tabletId, ui32 key, ui32 index) { - TString writeQuery = Sprintf(R"( - ( - (let key0 '( '('key (Uint32 '%u ) ) ) ) - (let row0 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key1 '( '('key (Uint32 '%u ) ) ) ) - (let row1 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key2 '( '('key (Uint32 '%u ) ) ) ) - (let row2 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key3 '( '('key (Uint32 '%u ) ) ) ) - (let row3 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key4 '( '('key (Uint32 '%u ) ) ) ) - (let row4 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key5 '( '('key (Uint32 '%u ) ) ) ) - (let row5 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key6 '( '('key (Uint32 '%u ) ) ) ) - (let row6 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key7 '( '('key (Uint32 '%u ) ) ) ) - (let row7 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key8 '( '('key (Uint32 '%u ) ) ) ) - (let row8 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - (let key9 '( '('key (Uint32 '%u ) ) ) ) - (let row9 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) - - (return (AsList - (UpdateRow '__user__Table key0 row0) - (UpdateRow '__user__Table key1 row1) - (UpdateRow '__user__Table key2 row2) - (UpdateRow '__user__Table key3 row3) - (UpdateRow '__user__Table key4 row4) - (UpdateRow '__user__Table key5 row5) - (UpdateRow '__user__Table key6 row6) - (UpdateRow '__user__Table key7 row7) - (UpdateRow '__user__Table key8 row8) - (UpdateRow '__user__Table key9 row9) - ) - ) - ) - )", - 1000*key + 0, 1000*index + 0, - 1000*key + 1, 1000*index + 1, - 1000*key + 2, 1000*index + 2, - 1000*key + 3, 1000*index + 3, - 1000*key + 4, 1000*index + 4, - 1000*key + 5, 1000*index + 5, - 1000*key + 6, 1000*index + 6, - 1000*key + 7, 1000*index + 7, - 1000*key + 8, 1000*index + 8, - 1000*key + 9, 1000*index + 9); - - NKikimrMiniKQL::TResult result; - TString err; +static void WriteRows(TTestActorRuntime& runtime, ui64 tabletId, ui32 key, ui32 index) { + TString writeQuery = Sprintf(R"( + ( + (let key0 '( '('key (Uint32 '%u ) ) ) ) + (let row0 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key1 '( '('key (Uint32 '%u ) ) ) ) + (let row1 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key2 '( '('key (Uint32 '%u ) ) ) ) + (let row2 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key3 '( '('key (Uint32 '%u ) ) ) ) + (let row3 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key4 '( '('key (Uint32 '%u ) ) ) ) + (let row4 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key5 '( '('key (Uint32 '%u ) ) ) ) + (let row5 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key6 '( '('key (Uint32 '%u ) ) ) ) + (let row6 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key7 '( '('key (Uint32 '%u ) ) ) ) + (let row7 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key8 '( '('key (Uint32 '%u ) ) ) ) + (let row8 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + (let key9 '( '('key (Uint32 '%u ) ) ) ) + (let row9 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) + + (return (AsList + (UpdateRow '__user__Table key0 row0) + (UpdateRow '__user__Table key1 row1) + (UpdateRow '__user__Table key2 row2) + (UpdateRow '__user__Table key3 row3) + (UpdateRow '__user__Table key4 row4) + (UpdateRow '__user__Table key5 row5) + (UpdateRow '__user__Table key6 row6) + (UpdateRow '__user__Table key7 row7) + (UpdateRow '__user__Table key8 row8) + (UpdateRow '__user__Table key9 row9) + ) + ) + ) + )", + 1000*key + 0, 1000*index + 0, + 1000*key + 1, 1000*index + 1, + 1000*key + 2, 1000*index + 2, + 1000*key + 3, 1000*index + 3, + 1000*key + 4, 1000*index + 4, + 1000*key + 5, 1000*index + 5, + 1000*key + 6, 1000*index + 6, + 1000*key + 7, 1000*index + 7, + 1000*key + 8, 1000*index + 8, + 1000*key + 9, 1000*index + 9); + + NKikimrMiniKQL::TResult result; + TString err; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, writeQuery, result, err); - UNIT_ASSERT_VALUES_EQUAL(err, ""); + UNIT_ASSERT_VALUES_EQUAL(err, ""); UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::EReplyStatus::OK);; -} - +} + Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { Y_UNIT_TEST(BaseCase) { @@ -135,76 +135,76 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { }); } - Y_UNIT_TEST(BaseCaseWithDataColumns) { - TTestWithReboots t(false); - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "dir/Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "index" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2 - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - for (ui32 delta = 0; delta < 2; ++delta) { + Y_UNIT_TEST(BaseCaseWithDataColumns) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "dir/Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "index" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2 + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + for (ui32 delta = 0; delta < 2; ++delta) { WriteRows(runtime, TTestTxConfig::FakeHiveTablets, 1 + delta, 100 + delta); - } - } - + } + } + AsyncBuilIndex(runtime, ++t.TxId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/dir/Table", "index1", {"index"}, {"value"}); - ui64 buildIndexId = t.TxId; - - { + ui64 buildIndexId = t.TxId; + + { auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); - UNIT_ASSERT_VALUES_EQUAL((ui64)descr.GetIndexBuild().GetState(), (ui64)Ydb::Table::IndexBuildState::STATE_PREPARING); - } - - t.TestEnv->TestWaitNotification(runtime, buildIndexId); - - { + UNIT_ASSERT_VALUES_EQUAL((ui64)descr.GetIndexBuild().GetState(), (ui64)Ydb::Table::IndexBuildState::STATE_PREPARING); + } + + t.TestEnv->TestWaitNotification(runtime, buildIndexId); + + { auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); - UNIT_ASSERT_VALUES_EQUAL((ui64)descr.GetIndexBuild().GetState(), (ui64)Ydb::Table::IndexBuildState::STATE_DONE); - } - - TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table"), - {NLs::PathExist, - NLs::IndexesCount(1)}); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table/index1", true, true, true), - {NLs::PathExist, + UNIT_ASSERT_VALUES_EQUAL((ui64)descr.GetIndexBuild().GetState(), (ui64)Ydb::Table::IndexBuildState::STATE_DONE); + } + + TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table"), + {NLs::PathExist, + NLs::IndexesCount(1)}); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table/index1", true, true, true), + {NLs::PathExist, NLs::IndexState(NKikimrSchemeOp::EIndexState::EIndexStateReady)}); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table/index1/indexImplTable", true, true, true), - {NLs::PathExist}); - - // Check result - { - TInactiveZone inactive(activeZone); - - NKikimrMiniKQL::TResult result; - TString err; + + TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table/index1/indexImplTable", true, true, true), + {NLs::PathExist}); + + // Check result + { + TInactiveZone inactive(activeZone); + + NKikimrMiniKQL::TResult result; + TString err; ui32 status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets+2, R"( - ( - (let range '( '('index (Uint32 '0) (Void) ) '('key (Uint32 '0) (Void) ))) - (let columns '('key 'index 'value) ) - (let result (SelectRange '__user__indexImplTable range columns '())) - (return (AsList (SetResult 'Result result) )) - ) - )", result, err); - - UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::OK, err); - UNIT_ASSERT_VALUES_EQUAL(err, ""); - + ( + (let range '( '('index (Uint32 '0) (Void) ) '('key (Uint32 '0) (Void) ))) + (let columns '('key 'index 'value) ) + (let result (SelectRange '__user__indexImplTable range columns '())) + (return (AsList (SetResult 'Result result) )) + ) + )", result, err); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::OK, err); + UNIT_ASSERT_VALUES_EQUAL(err, ""); + NKqp::CompareYson(R"([[[[[["100000"];["1000"];["aaaa"]];[["100001"];["1001"];["aaaa"]];[["100002"];["1002"];["aaaa"]];[["100003"];["1003"];["aaaa"]];[["100004"];["1004"];["aaaa"]];[["100005"];["1005"];["aaaa"]];[["100006"];["1006"];["aaaa"]];[["100007"];["1007"];["aaaa"]];[["100008"];["1008"];["aaaa"]];[["100009"];["1009"];["aaaa"]];[["101000"];["2000"];["aaaa"]];[["101001"];["2001"];["aaaa"]];[["101002"];["2002"];["aaaa"]];[["101003"];["2003"];["aaaa"]];[["101004"];["2004"];["aaaa"]];[["101005"];["2005"];["aaaa"]];[["101006"];["2006"];["aaaa"]];[["101007"];["2007"];["aaaa"]];[["101008"];["2008"];["aaaa"]];[["101009"];["2009"];["aaaa"]]];%false]]])", result); - } - }); - } - + } + }); + } + Y_UNIT_TEST(DropIndex) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { @@ -257,59 +257,59 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { }); } - Y_UNIT_TEST(DropIndexWithDataColumns) { - TTestWithReboots t(false); - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - TestCreateIndexedTable(runtime, ++t.TxId, "/MyRoot", R"( - TableDescription { - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value0" Type: "Utf8" } - Columns { Name: "value1" Type: "Utf8" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), - {NLs::Finished, - NLs::PathVersionEqual(3), - NLs::IndexesCount(1)}); + Y_UNIT_TEST(DropIndexWithDataColumns) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestCreateIndexedTable(runtime, ++t.TxId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value0" Type: "Utf8" } + Columns { Name: "value1" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), + {NLs::Finished, + NLs::PathVersionEqual(3), + NLs::IndexesCount(1)}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), - {NLs::Finished, + {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), - NLs::IndexKeys({"value0"})}); + NLs::IndexKeys({"value0"})}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), - {NLs::Finished, - NLs::PathVersionEqual(3)}); - } - + {NLs::Finished, + NLs::PathVersionEqual(3)}); + } + TestDropTableIndex(runtime, ++t.TxId, "/MyRoot", R"( TableName: "Table" IndexName: "UserDefinedIndexByValue0" )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), - {NLs::Finished, + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), + {NLs::Finished, NLs::PathVersionEqual(5), - NLs::IndexesCount(0)}); + NLs::IndexesCount(0)}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), - {NLs::PathNotExist}); + {NLs::PathNotExist}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), - {NLs::PathNotExist}); - }); - } - + {NLs::PathNotExist}); + }); + } + Y_UNIT_TEST(CancelBuild) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { @@ -400,7 +400,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { auto descr = TestGetBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildId); if (response.GetStatus() == Ydb::StatusIds::SUCCESS) { - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_CANCELLED); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_CANCELLED); TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table"), {NLs::PathExist, @@ -410,7 +410,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table/index1", true, true, true), {NLs::PathNotExist}); } else { - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); TestDescribeResult(DescribePath(runtime, "/MyRoot/dir/Table"), {NLs::PathExist, diff --git a/ydb/core/tx/schemeshard/ut_reboots.cpp b/ydb/core/tx/schemeshard/ut_reboots.cpp index a10cc9e04b..dce2c12712 100644 --- a/ydb/core/tx/schemeshard/ut_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_reboots.cpp @@ -1,7 +1,7 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/datashard/datashard.h> - + #include <ydb/core/protos/flat_scheme_op.pb.h> #include <google/protobuf/text_format.h> @@ -169,8 +169,8 @@ Y_UNIT_TEST_SUITE(IntermediateDirsReboots) { Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { Y_UNIT_TEST(Fake) { - } - + } + Y_UNIT_TEST(CopyWithData) { TTestWithReboots t; t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { @@ -353,13 +353,13 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { IndexDescription { Name: "UserDefinedIndexByValues" KeyColumnNames: ["value0", "value1"] - State: EIndexStateNotReady + State: EIndexStateNotReady + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } )"); t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); @@ -372,7 +372,7 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/Table1"), { NLs::Finished, NLs::PathVersionEqual(3), - NLs::IndexesCount(3)}); + NLs::IndexesCount(3)}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), @@ -388,13 +388,13 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValues/indexImplTable"), {NLs::Finished}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1"), - {NLs::Finished, + {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), - NLs::IndexKeys({"value0"}), - NLs::IndexDataColumns({"value1"})}); + NLs::IndexKeys({"value0"}), + NLs::IndexDataColumns({"value1"})}); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), - {NLs::Finished}); + {NLs::Finished}); } @@ -424,11 +424,11 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { Name: "UserDefinedIndexByValues" KeyColumnNames: ["value0", "value1"] } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } )"); t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); } @@ -566,13 +566,13 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { IndexDescription { Name: "UserDefinedIndexByValue1" KeyColumnNames: ["value1"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0CoveringValue1" - KeyColumnNames: ["value0"] - DataColumnNames: ["value1"] - } - )"); + } + IndexDescription { + Name: "UserDefinedIndexByValue0CoveringValue1" + KeyColumnNames: ["value0"] + DataColumnNames: ["value1"] + } + )"); t.TestEnv->TestWaitNotification(runtime, {t.TxId - 2, t.TxId - 1, t.TxId}); diff --git a/ydb/core/tx/time_cast/time_cast.cpp b/ydb/core/tx/time_cast/time_cast.cpp index ffc5469c00..08042194fc 100644 --- a/ydb/core/tx/time_cast/time_cast.cpp +++ b/ydb/core/tx/time_cast/time_cast.cpp @@ -99,7 +99,7 @@ class TMediatorTimecastProxy : public TActor<TMediatorTimecastProxy> { } if (ev->Record.BucketSize()) { - mediator.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, mediatorTabletId)); + mediator.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, mediatorTabletId)); LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() << " SEND to# " << mediatorTabletId << " Mediator " << ev->ToString()); NTabletPipe::SendData(ctx, mediator.PipeClient, ev.Release()); diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 53d9609510..af48b42d60 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -157,8 +157,8 @@ struct TReadTableRequest : public TThrRefBase { ui64 RowsRemain; THashMap<ui64, TQuotaRequest> QuotaRequests; ui64 QuotaRequestId; - ui32 RequestVersion; - ui32 ResponseVersion = NKikimrTxUserProxy::TReadTableTransaction::UNSPECIFIED; + ui32 RequestVersion; + ui32 ResponseVersion = NKikimrTxUserProxy::TReadTableTransaction::UNSPECIFIED; TRowVersion Snapshot = TRowVersion::Max(); TReadTableRequest(const NKikimrTxUserProxy::TReadTableTransaction &tx) @@ -169,7 +169,7 @@ struct TReadTableRequest : public TThrRefBase { , RowsLimited(tx.GetRowLimit() > 0) , RowsRemain(tx.GetRowLimit()) , QuotaRequestId(1) - , RequestVersion(tx.HasApiVersion() ? tx.GetApiVersion() : (ui32)NKikimrTxUserProxy::TReadTableTransaction::UNSPECIFIED) + , RequestVersion(tx.HasApiVersion() ? tx.GetApiVersion() : (ui32)NKikimrTxUserProxy::TReadTableTransaction::UNSPECIFIED) { for (auto &col : tx.GetColumns()) { Columns.emplace_back(col, 0, 0); @@ -184,10 +184,10 @@ struct TReadTableRequest : public TThrRefBase { class TDataReq : public TActor<TDataReq> { public: - enum class EParseRangeKeyExp { - NONE, - TO_NULL - }; + enum class EParseRangeKeyExp { + NONE, + TO_NULL + }; enum class ECoordinatorStatus { Unknown, @@ -317,7 +317,7 @@ private: THashMap<ui64, TPerTablet> PerTablet; NYql::TIssueManager IssueManager; - TTablePathHashSet InvalidatedTables; + TTablePathHashSet InvalidatedTables; ui64 TabletsLeft; // todo: add scale modifiers ui64 TabletErrors; @@ -354,8 +354,8 @@ private: TInstant WallClockFirstExecReply; TInstant WallClockLastExecReply; - TDuration CpuTime; - + TDuration CpuTime; + TAutoPtr<TEvTxProxySchemeCache::TEvResolveKeySet> PrepareFlatMKQLRequest(TStringBuf miniKQLProgram, TStringBuf miniKQLParams, const TActorContext &ctx); void ProcessFlatMKQLResolve(NSchemeCache::TSchemeCacheRequest *cacheRequest, const TActorContext &ctx); void ProcessReadTableResolve(NSchemeCache::TSchemeCacheRequest *cacheRequest, const TActorContext &ctx); @@ -388,7 +388,7 @@ private: void RegisterPlan(const TActorContext &ctx); void MergeResult(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, const TActorContext &ctx); - void MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TCpuTimer& timer); + void MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TCpuTimer& timer); void ProcessStreamResponseData(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, const TActorContext &ctx); @@ -441,7 +441,7 @@ private: bool ParseRangeKey(const NKikimrMiniKQL::TParams &proto, TConstArrayRef<NScheme::TTypeId> keyType, TSerializedCellVec &buf, - EParseRangeKeyExp exp); + EParseRangeKeyExp exp); bool CheckDomainLocality(NSchemeCache::TSchemeCacheRequest &cacheRequest); void BuildTxStats(NKikimrQueryStats::TTxStats& stats); @@ -743,7 +743,7 @@ void TDataReq::ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus if (ReadTableRequest) { x->Record.SetSerializedReadTableResponse(ReadTableRequest->ResponseData); x->Record.SetDataShardTabletId(ReadTableRequest->ResponseDataFrom); - x->Record.SetReadTableResponseVersion(ReadTableRequest->ResponseVersion); + x->Record.SetReadTableResponseVersion(ReadTableRequest->ResponseVersion); } if (!DatashardErrors.empty()) @@ -896,7 +896,7 @@ void Aggregate(NKikimrQueryStats::TTableAccessStats& aggr, const NKikimrQuerySta } void TDataReq::BuildTxStats(NKikimrQueryStats::TTxStats& stats) { - TTablePathHashMap<NKikimrQueryStats::TTableAccessStats> byTable; + TTablePathHashMap<NKikimrQueryStats::TTableAccessStats> byTable; for (const auto& shard : PerTablet) { if (!shard.second.Stats) @@ -925,8 +925,8 @@ void TDataReq::BuildTxStats(NKikimrQueryStats::TTxStats& stats) { for (auto& tableStats : byTable) { stats.AddTableAccessStats()->Swap(&tableStats.second); } - - stats.SetComputeCpuTimeUsec(CpuTime.MicroSeconds()); + + stats.SetComputeCpuTimeUsec(CpuTime.MicroSeconds()); } void TDataReq::ProcessFlatMKQLResolve(NSchemeCache::TSchemeCacheRequest *cacheRequest, const TActorContext &ctx) { @@ -1091,7 +1091,7 @@ void TDataReq::ProcessReadTableResolve(NSchemeCache::TSchemeCacheRequest *cacheR auto &tx = *dataTransaction.MutableReadTableTransaction(); tx.MutableTableId()->SetOwnerId(ReadTableRequest->KeyDesc->TableId.PathId.OwnerId); tx.MutableTableId()->SetTableId(ReadTableRequest->KeyDesc->TableId.PathId.LocalPathId); - tx.SetApiVersion(ReadTableRequest->RequestVersion); + tx.SetApiVersion(ReadTableRequest->RequestVersion); for (auto &col : ReadTableRequest->Columns) { auto &c = *tx.AddColumns(); c.SetId(col.Id); @@ -1243,7 +1243,7 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte // Subscribe for TEvStreamIsDead event. if (StreamResponse) - ctx.Send(RequestSource, new TEvents::TEvSubscribe, IEventHandle::FlagTrackDelivery); + ctx.Send(RequestSource, new TEvents::TEvSubscribe, IEventHandle::FlagTrackDelivery); LOG_DEBUG_S_SAMPLED_BY(ctx, NKikimrServices::TX_PROXY, TxId, "Actor# " << ctx.SelfID.ToString() << " Cookie# " << (ui64)ev->Cookie @@ -1272,7 +1272,7 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte TAutoPtr<TEvTxProxySchemeCache::TEvResolveKeySet> resolveReq; - NCpuTime::TCpuTimer timer(CpuTime); + NCpuTime::TCpuTimer timer(CpuTime); if (txbody.HasMiniKQLTransaction()) { const auto& mkqlTxBody = txbody.GetMiniKQLTransaction(); @@ -1350,7 +1350,7 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte return Die(ctx); } FlatMKQLRequest->Engine->AfterShardProgramsExtracted(); - return MakeFlatMKQLResponse(ctx, timer); + return MakeFlatMKQLResponse(ctx, timer); } else { IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::ENGINE_ERROR, "empty affected set")); ReportStatus( @@ -1468,28 +1468,28 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, // Parse range. TConstArrayRef<NScheme::TTypeId> keyTypes(keyColumnTypes.data(), keySize); - // Fix KeyRanges - bool fromInclusive = ReadTableRequest->Range.GetFromInclusive(); - EParseRangeKeyExp fromExpand = EParseRangeKeyExp::TO_NULL; - bool toInclusive = ReadTableRequest->Range.GetToInclusive(); - EParseRangeKeyExp toExpand = EParseRangeKeyExp::NONE; - if (ReadTableRequest->RequestVersion == NKikimrTxUserProxy::TReadTableTransaction::YDB_V1) { - if (!ReadTableRequest->Range.HasFrom()) { - fromExpand = EParseRangeKeyExp::TO_NULL; - } else { - fromExpand = fromInclusive ? EParseRangeKeyExp::TO_NULL : EParseRangeKeyExp::NONE; - } - - if (!ReadTableRequest->Range.HasTo()) { - toExpand = EParseRangeKeyExp::NONE; - } else { - toExpand = toInclusive ? EParseRangeKeyExp::NONE : EParseRangeKeyExp::TO_NULL; - } - } + // Fix KeyRanges + bool fromInclusive = ReadTableRequest->Range.GetFromInclusive(); + EParseRangeKeyExp fromExpand = EParseRangeKeyExp::TO_NULL; + bool toInclusive = ReadTableRequest->Range.GetToInclusive(); + EParseRangeKeyExp toExpand = EParseRangeKeyExp::NONE; + if (ReadTableRequest->RequestVersion == NKikimrTxUserProxy::TReadTableTransaction::YDB_V1) { + if (!ReadTableRequest->Range.HasFrom()) { + fromExpand = EParseRangeKeyExp::TO_NULL; + } else { + fromExpand = fromInclusive ? EParseRangeKeyExp::TO_NULL : EParseRangeKeyExp::NONE; + } + + if (!ReadTableRequest->Range.HasTo()) { + toExpand = EParseRangeKeyExp::NONE; + } else { + toExpand = toInclusive ? EParseRangeKeyExp::NONE : EParseRangeKeyExp::TO_NULL; + } + } if (!ParseRangeKey(ReadTableRequest->Range.GetFrom(), keyTypes, - ReadTableRequest->FromValues, fromExpand) + ReadTableRequest->FromValues, fromExpand) || !ParseRangeKey(ReadTableRequest->Range.GetTo(), keyTypes, - ReadTableRequest->ToValues, toExpand)) { + ReadTableRequest->ToValues, toExpand)) { IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::KEY_PARSE_ERROR, "could not parse key string")); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::QUERY_ERROR, true, ctx); TxProxyMon->ResolveKeySetWrongRequest->Inc(); @@ -1497,9 +1497,9 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, } TTableRange range(ReadTableRequest->FromValues.GetCells(), - fromInclusive, + fromInclusive, ReadTableRequest->ToValues.GetCells(), - toInclusive); + toInclusive); if (range.IsEmptyRange({keyTypes.begin(), keyTypes.end()})) { const TString errorExplanation = "empty range requested"; @@ -1575,7 +1575,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c TxProxyMon->TxPrepareResolveHgram->Collect((WallClockResolved - WallClockResolveStarted).MicroSeconds()); - NCpuTime::TCpuTimer timer(CpuTime); + NCpuTime::TCpuTimer timer(CpuTime); for (const NSchemeCache::TSchemeCacheRequest::TEntry& entry : request->ResultSet) { ui32 access = 0; switch (entry.KeyDescription->RowOperation) { @@ -1879,24 +1879,24 @@ void TDataReq::HandlePrepare(TEvDataShard::TEvProposeTransactionResult::TPtr &ev case NKikimrTxDataShard::TEvProposeTransactionResult::RESPONSE_DATA: ProcessStreamResponseData(ev, ctx); return; - case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: { + case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: { ExtractDatashardErrors(record); CancelProposal(tabletId); - bool schemeChanged = false; - for (const auto& e : record.GetError()) { - if (e.GetKind() == NKikimrTxDataShard::TError::SCHEME_CHANGED) { - schemeChanged = true; - } - } - if (schemeChanged) { - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::REJECTED, true, ctx); - } else { - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardNotAvailable, NKikimrIssues::TStatusIds::REJECTED, true, ctx); - } + bool schemeChanged = false; + for (const auto& e : record.GetError()) { + if (e.GetKind() == NKikimrTxDataShard::TError::SCHEME_CHANGED) { + schemeChanged = true; + } + } + if (schemeChanged) { + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::REJECTED, true, ctx); + } else { + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ProxyShardNotAvailable, NKikimrIssues::TStatusIds::REJECTED, true, ctx); + } Become(&TThis::StatePrepareErrors, ctx, TDuration::MilliSeconds(500), new TEvents::TEvWakeup); TxProxyMon->TxResultError->Inc(); return HandlePrepareErrors(ev, ctx); - } + } case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAborted, NKikimrIssues::TStatusIds::SUCCESS, true, ctx); TxProxyMon->TxResultAborted->Inc(); @@ -2522,7 +2522,7 @@ void TDataReq::MergeResult(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, } Y_VERIFY(FlatMKQLRequest); - NCpuTime::TCpuTimer timer; + NCpuTime::TCpuTimer timer; NMiniKQL::IEngineFlat &engine = *FlatMKQLRequest->Engine; for (const auto& lock : record.GetTxLocks()) { @@ -2567,14 +2567,14 @@ void TDataReq::MergeResult(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, } } - if (FlatMKQLRequest->BalanceCoverageBuilders.empty()) { - return MakeFlatMKQLResponse(ctx, timer); - } else { - CpuTime += timer.GetTime(); - } + if (FlatMKQLRequest->BalanceCoverageBuilders.empty()) { + return MakeFlatMKQLResponse(ctx, timer); + } else { + CpuTime += timer.GetTime(); + } } -void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TCpuTimer& timer) { +void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TCpuTimer& timer) { NMiniKQL::IEngineFlat &engine = *FlatMKQLRequest->Engine; engine.SetStepTxId({ PlanStep, TxId }); @@ -2592,7 +2592,7 @@ void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TC LOG_ERROR_S_SAMPLED_BY(ctx, NKikimrServices::TX_PROXY, TxId, "Actor# " << ctx.SelfID.ToString() << " txid# " << TxId << " MergeResult ExecError TDataReq marker# P16"); - CpuTime += timer.GetTime(); + CpuTime += timer.GetTime(); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecError, NKikimrIssues::TStatusIds::ERROR, true, ctx); TxProxyMon->MergeResultMiniKQLExecError->Inc(); return Die(ctx); @@ -2605,7 +2605,7 @@ void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TC auto fillResult = engine.FillResultValue(FlatMKQLRequest->EngineEvaluatedResponse); switch (fillResult) { case NMiniKQL::IEngineFlat::EResult::Ok: - CpuTime += timer.GetTime(); + CpuTime += timer.GetTime(); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete, NKikimrIssues::TStatusIds::SUCCESS, true, ctx); TxProxyMon->MergeResultMiniKQLExecComplete->Inc(); break; @@ -2615,7 +2615,7 @@ void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TC << " MergeResult Result too large TDataReq marker# P18"); FlatMKQLRequest->EngineResultStatusCode = fillResult; - CpuTime += timer.GetTime(); + CpuTime += timer.GetTime(); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecResultUnavailable, NKikimrIssues::TStatusIds::ERROR, true, ctx); TxProxyMon->MergeResultMiniKQLExecError->Inc(); break; @@ -2624,7 +2624,7 @@ void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TC "Actor# " << ctx.SelfID.ToString() << " txid# " << TxId << " MergeResult Execution was cancelled TDataReq marker# P20"); - CpuTime += timer.GetTime(); + CpuTime += timer.GetTime(); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecTimeout, NKikimrIssues::TStatusIds::TIMEOUT, true, ctx); TxProxyMon->ExecTimeout->Inc(); break; @@ -2633,7 +2633,7 @@ void TDataReq::MakeFlatMKQLResponse(const TActorContext &ctx, const NCpuTime::TC "Actor# " << ctx.SelfID.ToString() << " txid# " << TxId << " MergeResult Error: " << (ui32)fillResult << " TDataReq marker# P19"); FlatMKQLRequest->EngineResultStatusCode = fillResult; - CpuTime += timer.GetTime(); + CpuTime += timer.GetTime(); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecError, NKikimrIssues::TStatusIds::ERROR, true, ctx); TxProxyMon->MergeResultMiniKQLExecError->Inc(); break; @@ -2656,9 +2656,9 @@ void TDataReq::ProcessStreamResponseData(TEvDataShard::TEvProposeTransactionResu auto &rec = ev->Get()->Record; ReadTableRequest->ResponseData = rec.GetTxResult(); ReadTableRequest->ResponseDataFrom = rec.GetOrigin(); - if (rec.HasApiVersion()) { - ReadTableRequest->ResponseVersion = rec.GetApiVersion(); - } + if (rec.HasApiVersion()) { + ReadTableRequest->ResponseVersion = rec.GetApiVersion(); + } if (ReadTableRequest->RowsLimited) { auto rows = rec.RowOffsetsSize(); @@ -2935,7 +2935,7 @@ void TDataReq::ProcessStreamClearance(bool cleared, const TActorContext &ctx) bool TDataReq::ParseRangeKey(const NKikimrMiniKQL::TParams &proto, TConstArrayRef<NScheme::TTypeId> keyType, TSerializedCellVec &buf, - EParseRangeKeyExp exp) + EParseRangeKeyExp exp) { TVector<TCell> key; if (proto.HasValue()) { @@ -2954,12 +2954,12 @@ bool TDataReq::ParseRangeKey(const NKikimrMiniKQL::TParams &proto, } } - switch (exp) { - case EParseRangeKeyExp::TO_NULL: - key.resize(keyType.size()); - break; - case EParseRangeKeyExp::NONE: - break; + switch (exp) { + case EParseRangeKeyExp::TO_NULL: + key.resize(keyType.size()); + break; + case EParseRangeKeyExp::NONE: + break; } buf.Parse(TSerializedCellVec::Serialize(key)); diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h index 5f42d07b19..86d4f1ac79 100644 --- a/ydb/core/tx/tx_proxy/proxy.h +++ b/ydb/core/tx/tx_proxy/proxy.h @@ -54,11 +54,11 @@ struct TEvTxUserProxy { EvExportRequest = EvProposeTransaction + 2 * 512, EvExportResponse, - EvUploadRowsResponse, - - EvGetProxyServicesRequest, - EvGetProxyServicesResponse, - + EvUploadRowsResponse, + + EvGetProxyServicesRequest, + EvGetProxyServicesResponse, + EvResolveTablesResponse, EvEnd @@ -197,28 +197,28 @@ struct TEvTxUserProxy { { } }; - struct TEvGetProxyServicesRequest : public TEventLocal<TEvGetProxyServicesRequest, EvGetProxyServicesRequest> { - // empty - }; - - struct TEvGetProxyServicesResponse : public TEventLocal<TEvGetProxyServicesResponse, EvGetProxyServicesResponse> { - const NTxProxy::TTxProxyServices Services; - - TEvGetProxyServicesResponse( - const NTxProxy::TTxProxyServices& services) - : Services(services) - { } - }; - - struct TEvUploadRowsResponse: public TEventLocal<TEvUploadRowsResponse, EvUploadRowsResponse> { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - - TEvUploadRowsResponse(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) - : Status(status) - , Issues(issues) - {} - }; + struct TEvGetProxyServicesRequest : public TEventLocal<TEvGetProxyServicesRequest, EvGetProxyServicesRequest> { + // empty + }; + + struct TEvGetProxyServicesResponse : public TEventLocal<TEvGetProxyServicesResponse, EvGetProxyServicesResponse> { + const NTxProxy::TTxProxyServices Services; + + TEvGetProxyServicesResponse( + const NTxProxy::TTxProxyServices& services) + : Services(services) + { } + }; + + struct TEvUploadRowsResponse: public TEventLocal<TEvUploadRowsResponse, EvUploadRowsResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + + TEvUploadRowsResponse(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) + : Status(status) + , Issues(issues) + {} + }; }; struct TEvTxProxyReq { diff --git a/ydb/core/tx/tx_proxy/proxy_impl.cpp b/ydb/core/tx/tx_proxy/proxy_impl.cpp index 9866eda145..20dc8cce17 100644 --- a/ydb/core/tx/tx_proxy/proxy_impl.cpp +++ b/ydb/core/tx/tx_proxy/proxy_impl.cpp @@ -387,14 +387,14 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { ProcessRequest(ev, ctx, txIds.front()); } - void Handle(TEvTxUserProxy::TEvGetProxyServicesRequest::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, - "actor# " << SelfId() << " Handle TEvGetProxyServicesRequest"); - - auto reply = MakeHolder<TEvTxUserProxy::TEvGetProxyServicesResponse>(Services); - ctx.Send(ev->Sender, reply.Release(), 0, ev->Cookie); - } - + void Handle(TEvTxUserProxy::TEvGetProxyServicesRequest::TPtr &ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, + "actor# " << SelfId() << " Handle TEvGetProxyServicesRequest"); + + auto reply = MakeHolder<TEvTxUserProxy::TEvGetProxyServicesResponse>(Services); + ctx.Send(ev->Sender, reply.Release(), 0, ev->Cookie); + } + void Handle(TEvTxUserProxy::TEvNavigate::TPtr &ev, const TActorContext &ctx) { TString path = ev->Get()->Record.GetDescribePath().GetPath(); LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, @@ -472,8 +472,8 @@ public: HFunc(TEvTxUserProxy::TEvProposeKqpTransaction, Handle); HFunc(TEvTxUserProxy::TEvAllocateTxId, Handle); - HFunc(TEvTxUserProxy::TEvGetProxyServicesRequest, Handle); - + HFunc(TEvTxUserProxy::TEvGetProxyServicesRequest, Handle); + HFunc(TEvents::TEvPoisonPill, Handle); default: LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index abe7ebf55f..923fe381ae 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -114,7 +114,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { TVector<TString> result = l; std::move(r.begin(), r.end(), std::back_inserter(result)); return result; - } + } static TString& GetPathNameForScheme(NKikimrSchemeOp::TModifyScheme& modifyScheme) { switch (modifyScheme.GetOperationType()) { @@ -150,7 +150,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpAlterPersQueueGroup: return *modifyScheme.MutableAlterPersQueueGroup()->MutableName(); - + case NKikimrSchemeOp::ESchemeOpModifyACL: return *modifyScheme.MutableModifyACL()->MutableName(); @@ -1169,15 +1169,15 @@ void TFlatSchemeReq::HandleWorkingDir(TEvTxProxySchemeCache::TEvNavigateKeySetRe auto parts = Merge(SplitPath(GetModifyScheme().GetWorkingDir()), SplitPath(GetPathNameForScheme(GetModifyScheme()))); if (!workingDir || workingDir->size() >= parts.size()) { - const TString errText = TStringBuilder() - << "Cannot resolve working dir" - << " workingDir# " << (workingDir ? JoinPath(*workingDir) : "null") - << " path# " << JoinPath(parts); - LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, errText); + const TString errText = TStringBuilder() + << "Cannot resolve working dir" + << " workingDir# " << (workingDir ? JoinPath(*workingDir) : "null") + << " path# " << JoinPath(parts); + LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, errText); TxProxyMon->ResolveKeySetWrongRequest->Inc(); - const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, errText); - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, nullptr, &issue, ctx); + const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, errText); + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, nullptr, &issue, ctx); return Die(ctx); } diff --git a/ydb/core/tx/tx_proxy/snapshotreq.cpp b/ydb/core/tx/tx_proxy/snapshotreq.cpp index e4e57ad63f..4b2d7fe275 100644 --- a/ydb/core/tx/tx_proxy/snapshotreq.cpp +++ b/ydb/core/tx/tx_proxy/snapshotreq.cpp @@ -76,7 +76,7 @@ public: , Cookie(ev->Cookie) , Request(ev->Release()) , TxProxyMon(mon) - , DefaultTimeoutMs(60000, 0, 360000) + , DefaultTimeoutMs(60000, 0, 360000) , SnapshotTxId(txid) { } @@ -361,7 +361,7 @@ public: } SelectedCoordinator = SelectCoordinator(msg->FindDomainInfo(), ctx); - + const auto& params = Request->Record.GetTransaction().GetCreateVolatileSnapshot(); for (auto& kv : PerShardStates) { diff --git a/ydb/core/tx/tx_proxy/upload_rows.cpp b/ydb/core/tx/tx_proxy/upload_rows.cpp index 2b30ef730e..3c4580a8a8 100644 --- a/ydb/core/tx/tx_proxy/upload_rows.cpp +++ b/ydb/core/tx/tx_proxy/upload_rows.cpp @@ -1,27 +1,27 @@ -#include "upload_rows.h" -#include "upload_rows_common_impl.h" - +#include "upload_rows.h" +#include "upload_rows_common_impl.h" + #include <ydb/core/tx/tx_proxy/proxy.h> - -namespace NKikimr { -namespace NTxProxy { - + +namespace NKikimr { +namespace NTxProxy { + class TUploadRowsInternal : public TUploadRowsBase<NKikimrServices::TActivity::UPLOAD_ROWS_INTERNAL> { -public: - TUploadRowsInternal( +public: + TUploadRowsInternal( TActorId sender, - const TString& table, - std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> types, - std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> rows, + const TString& table, + std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> types, + std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> rows, EUploadRowsMode mode, bool writeToPrivateTable) - : Sender(sender) - , Table(table) - , ColumnTypes(types) - , Rows(rows) - { + : Sender(sender) + , Table(table) + , ColumnTypes(types) + , Rows(rows) + { AllowWriteToPrivateTable = writeToPrivateTable; - + switch (mode) { case EUploadRowsMode::Normal: // nothing @@ -30,38 +30,38 @@ public: WriteToTableShadow = true; break; } - } - -private: + } + +private: TString GetDatabase()override { return TString(); } - const TString& GetTable() override { - return Table; - } - - const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const override { - return *Rows; - } - - bool CheckAccess(TString&) override { - return true; - } - - void SendResult(const NActors::TActorContext& ctx, const Ydb::StatusIds::StatusCode& status) override { - auto ev = new TEvTxUserProxy::TEvUploadRowsResponse(status, Issues); - ctx.Send(Sender, ev); - } - + const TString& GetTable() override { + return Table; + } + + const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const override { + return *Rows; + } + + bool CheckAccess(TString&) override { + return true; + } + + void SendResult(const NActors::TActorContext& ctx, const Ydb::StatusIds::StatusCode& status) override { + auto ev = new TEvTxUserProxy::TEvUploadRowsResponse(status, Issues); + ctx.Send(Sender, ev); + } + void RaiseIssue(const NYql::TIssue& issue) override { - Issues.AddIssue(issue); - } - + Issues.AddIssue(issue); + } + bool ExtractRows(TString&) override { - return true; - } - + return true; + } + bool ExtractBatch(TString& errorMessage) override { errorMessage = "Not supported"; return false; @@ -69,32 +69,32 @@ private: TVector<std::pair<TString, Ydb::Type>> GetRequestColumns(TString& errorMessage) const override { Y_UNUSED(errorMessage); - return *ColumnTypes; - } - -private: + return *ColumnTypes; + } + +private: const TActorId Sender; - const TString Table; - const std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> ColumnTypes; - const std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> Rows; - - NYql::TIssues Issues; -}; - + const TString Table; + const std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> ColumnTypes; + const std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> Rows; + + NYql::TIssues Issues; +}; + IActor* CreateUploadRowsInternal(const TActorId& sender, - const TString& table, - std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> types, - std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> rows, - EUploadRowsMode mode, - bool writeToPrivateTable) + const TString& table, + std::shared_ptr<TVector<std::pair<TString, Ydb::Type>>> types, + std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString>>> rows, + EUploadRowsMode mode, + bool writeToPrivateTable) { - return new TUploadRowsInternal(sender, - table, - types, - rows, - mode, - writeToPrivateTable); -} - -} // namespace NTxProxy -} // namespace NKikimr + return new TUploadRowsInternal(sender, + table, + types, + rows, + mode, + writeToPrivateTable); +} + +} // namespace NTxProxy +} // namespace NKikimr diff --git a/ydb/core/tx/tx_proxy/upload_rows.h b/ydb/core/tx/tx_proxy/upload_rows.h index ce5505725b..93d116b04e 100644 --- a/ydb/core/tx/tx_proxy/upload_rows.h +++ b/ydb/core/tx/tx_proxy/upload_rows.h @@ -1,26 +1,26 @@ -#pragma once -#include "defs.h" - -namespace Ydb { - class Type; -} - -namespace NKikimr { - -class TSerializedCellVec; - -namespace NTxProxy { - +#pragma once +#include "defs.h" + +namespace Ydb { + class Type; +} + +namespace NKikimr { + +class TSerializedCellVec; + +namespace NTxProxy { + enum class EUploadRowsMode { Normal, WriteToTableShadow, }; IActor* CreateUploadRowsInternal(const TActorId& sender, - const TString& table, - std::shared_ptr<TVector<std::pair<TString, Ydb::Type> > > types, - std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString> > > rows, + const TString& table, + std::shared_ptr<TVector<std::pair<TString, Ydb::Type> > > types, + std::shared_ptr<TVector<std::pair<TSerializedCellVec, TString> > > rows, EUploadRowsMode mode = EUploadRowsMode::Normal, bool writeToPrivateTable = false); -} // namespace NTxProxy -} // namespace NKikimr +} // namespace NTxProxy +} // namespace NKikimr diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.cpp b/ydb/core/tx/tx_proxy/upload_rows_common_impl.cpp index f7c24265a9..3e0a6401fc 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.cpp +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.cpp @@ -1 +1 @@ -#include "upload_rows_common_impl.h" +#include "upload_rows_common_impl.h" diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 0473daa606..0aaee7e7f5 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <ydb/core/actorlib_impl/long_timer.h> @@ -14,10 +14,10 @@ #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> - + #include <ydb/public/api/protos/ydb_status_codes.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> - + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #undef INCLUDE_YDB_INTERNAL_H @@ -89,13 +89,13 @@ private: } -namespace NTxProxy { +namespace NTxProxy { template <NKikimrServices::TActivity::EType DerivedActivityType> class TUploadRowsBase : public TActorBootstrapped<TUploadRowsBase<DerivedActivityType>> { using TBase = TActorBootstrapped<TUploadRowsBase<DerivedActivityType>>; using TThis = typename TBase::TThis; - + private: using TTabletId = ui64; @@ -108,11 +108,11 @@ private: TActorId TimeoutTimerActorId; bool WaitingResolveReply; bool Finished; - - TAutoPtr<NSchemeCache::TSchemeCacheRequest> ResolvePartitionsResult; + + TAutoPtr<NSchemeCache::TSchemeCacheRequest> ResolvePartitionsResult; TAutoPtr<NSchemeCache::TSchemeCacheNavigate> ResolveNamesResult; - TSerializedCellVec MinKey; - TSerializedCellVec MaxKey; + TSerializedCellVec MinKey; + TSerializedCellVec MaxKey; TVector<NScheme::TTypeId> KeyColumnTypes; TVector<NScheme::TTypeId> ValueColumnTypes; NSchemeCache::TSchemeCacheNavigate::EKind TableKind = NSchemeCache::TSchemeCacheNavigate::KindUnknown; @@ -123,7 +123,7 @@ private: NLongTxService::TLongTxId LongTxId; NThreading::TFuture<Ydb::LongTx::WriteResponse> WriteBatchResult; -protected: +protected: enum class EUploadSource { ProtoValues = 0, ArrowBatch = 1, @@ -147,7 +147,7 @@ protected: TVector<std::pair<TString, NScheme::TTypeId>> SrcColumns; // source columns in CSV could have any order TVector<std::pair<TString, NScheme::TTypeId>> YdbSchema; THashMap<ui32, size_t> Id2Position; // columnId -> its position in YdbSchema - + bool WriteToTableShadow = false; bool AllowWriteToPrivateTable = false; @@ -171,7 +171,7 @@ public: void Bootstrap(const NActors::TActorContext& ctx) { Deadline = AppData(ctx)->TimeProvider->Now() + Timeout; - ResolveTable(GetTable(), ctx); + ResolveTable(GetTable(), ctx); } void Die(const NActors::TActorContext& ctx) override { @@ -184,10 +184,10 @@ public: TBase::Die(ctx); } -protected: +protected: const NSchemeCache::TSchemeCacheNavigate* GetResolveNameResult() const { return ResolveNamesResult.Get(); - } + } const TKeyDesc* GetKeyRange() const { Y_VERIFY(ResolvePartitionsResult->ResultSet.size() == 1); @@ -233,14 +233,14 @@ protected: private: virtual TString GetDatabase() = 0; - virtual const TString& GetTable() = 0; - virtual const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const = 0; - virtual bool CheckAccess(TString& errorMessage) = 0; + virtual const TString& GetTable() = 0; + virtual const TVector<std::pair<TSerializedCellVec, TString>>& GetRows() const = 0; + virtual bool CheckAccess(TString& errorMessage) = 0; virtual TVector<std::pair<TString, Ydb::Type>> GetRequestColumns(TString& errorMessage) const = 0; virtual bool ExtractRows(TString& errorMessage) = 0; virtual bool ExtractBatch(TString& errorMessage) = 0; - virtual void RaiseIssue(const NYql::TIssue& issue) = 0; - virtual void SendResult(const NActors::TActorContext& ctx, const ::Ydb::StatusIds::StatusCode& status) = 0; + virtual void RaiseIssue(const NYql::TIssue& issue) = 0; + virtual void SendResult(const NActors::TActorContext& ctx, const ::Ydb::StatusIds::StatusCode& status) = 0; virtual EUploadSource GetSourceType() const { return EUploadSource::ProtoValues; @@ -256,7 +256,7 @@ private: return none; } -private: +private: STFUNC(StateWaitResolveTable) { switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); @@ -333,8 +333,8 @@ private: } } - for (size_t pos = 0; pos < reqColumns.size(); ++pos) { - auto& name = reqColumns[pos].first; + for (size_t pos = 0; pos < reqColumns.size(); ++pos) { + auto& name = reqColumns[pos].first; const auto* cp = columnByName.FindPtr(name); if (!cp) { errorMessage = Sprintf("Unknown column: %s", name.c_str()); @@ -343,7 +343,7 @@ private: ui32 colId = *cp; auto& ci = *entry.Columns.FindPtr(colId); - const auto& typeInProto = reqColumns[pos].second; + const auto& typeInProto = reqColumns[pos].second; if (typeInProto.type_id()) { NScheme::TTypeId typeInRequest = typeInProto.type_id(); @@ -418,60 +418,60 @@ private: return true; } - void ResolveTable(const TString& table, const NActors::TActorContext& ctx) { - // TODO: check all params; - // Cerr << *Request->GetProtoRequest() << Endl; + void ResolveTable(const TString& table, const NActors::TActorContext& ctx) { + // TODO: check all params; + // Cerr << *Request->GetProtoRequest() << Endl; - TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = ::NKikimr::SplitPath(table); - if (entry.Path.empty()) { - return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, "Invalid table path specified", ctx); + TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = ::NKikimr::SplitPath(table); + if (entry.Path.empty()) { + return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, "Invalid table path specified", ctx); } - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; entry.SyncVersion = true; entry.ShowPrivatePath = AllowWriteToPrivateTable; - request->ResultSet.emplace_back(entry); - ctx.Send(SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); + request->ResultSet.emplace_back(entry); + ctx.Send(SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); - TimeoutTimerActorId = CreateLongTimer(ctx, Timeout, - new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup())); + TimeoutTimerActorId = CreateLongTimer(ctx, Timeout, + new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup())); - TBase::Become(&TThis::StateWaitResolveTable); - WaitingResolveReply = true; - } + TBase::Become(&TThis::StateWaitResolveTable); + WaitingResolveReply = true; + } - void HandleTimeout(const TActorContext& ctx) { + void HandleTimeout(const TActorContext& ctx) { ShardRepliesLeft.clear(); - return ReplyWithError(Ydb::StatusIds::TIMEOUT, "Request timed out", ctx); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - WaitingResolveReply = false; - if (Finished) { - return Die(ctx); - } - - const NSchemeCache::TSchemeCacheNavigate& request = *ev->Get()->Request; - - Y_VERIFY(request.ResultSet.size() == 1); - switch (request.ResultSet.front().Status) { - case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: - case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + return ReplyWithError(Ydb::StatusIds::TIMEOUT, "Request timed out", ctx); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + WaitingResolveReply = false; + if (Finished) { + return Die(ctx); + } + + const NSchemeCache::TSchemeCacheNavigate& request = *ev->Get()->Request; + + Y_VERIFY(request.ResultSet.size() == 1); + switch (request.ResultSet.front().Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: return ReplyWithError(Ydb::StatusIds::UNAVAILABLE, Sprintf("Table '%s' unavaliable", GetTable().c_str()), ctx); - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotPath: - case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotPath: + case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Sprintf("Unknown table '%s'", GetTable().c_str()), ctx); - case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Sprintf("Unknown database for table '%s'", GetTable().c_str()), ctx); - case NSchemeCache::TSchemeCacheNavigate::EStatus::Unknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::Unknown: return ReplyWithError(Ydb::StatusIds::GENERIC_ERROR, Sprintf("Unknown error on table '%s'", GetTable().c_str()), ctx); } - + TableKind = request.ResultSet.front().Kind; bool isOlapTable = (TableKind == NSchemeCache::TSchemeCacheNavigate::KindOlapTable); @@ -480,10 +480,10 @@ private: Sprintf("Table '%s' is a system view. Bulk upsert is not supported.", GetTable().c_str()), ctx); } - ResolveNamesResult = ev->Get()->Request; + ResolveNamesResult = ev->Get()->Request; bool makeYdbSchema = isOlapTable || (GetSourceType() != EUploadSource::ProtoValues); - TString errorMessage; + TString errorMessage; if (!BuildSchema(ctx, errorMessage, makeYdbSchema)) { return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, errorMessage, ctx); } @@ -526,7 +526,7 @@ private: break; } - } + } if (TableKind == NSchemeCache::TSchemeCacheNavigate::KindTable) { ResolveShards(ctx); @@ -536,7 +536,7 @@ private: return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Sprintf("Table '%s': Bulk upsert is not supported for this table kind.", GetTable().c_str()), ctx); } - } + } void WriteToOlapTable(const NActors::TActorContext& ctx) { TString accessCheckError; @@ -739,10 +739,10 @@ private: return ReplyWithResult(msg->Record.GetStatus(), ctx); } - void FindMinMaxKeys() { + void FindMinMaxKeys() { - for (const auto& pair : GetRows()) { - const auto& serializedKey = pair.first; + for (const auto& pair : GetRows()) { + const auto& serializedKey = pair.first; if (MinKey.GetCells().empty()) { // Only for the first key @@ -841,7 +841,7 @@ private: // Group rows by shard id TVector<std::unique_ptr<TEvDataShard::TEvUploadRowsRequest>> shardRequests(keyRange->Partitions.size()); - for (const auto& keyValue : GetRows()) { + for (const auto& keyValue : GetRows()) { // Find partition for the key auto it = std::lower_bound(keyRange->Partitions.begin(), keyRange->Partitions.end(), keyValue.first.GetCells(), [this](const auto &partition, const auto& key) { @@ -999,7 +999,7 @@ private: } void ReplyWithResult(::Ydb::StatusIds::StatusCode status, const TActorContext& ctx) { - SendResult(ctx, status); + SendResult(ctx, status); LOG_DEBUG_S(ctx, NKikimrServices::MSGBUS_REQUEST, "Bulk upsert to table " << GetTable() << " completed with status " << status); @@ -1017,5 +1017,5 @@ private: } }; -} // namespace NTxProxy +} // namespace NTxProxy } // namespace NKikimr diff --git a/ydb/core/tx/tx_proxy/ya.make b/ydb/core/tx/tx_proxy/ya.make index c06594b92f..d412dc8988 100644 --- a/ydb/core/tx/tx_proxy/ya.make +++ b/ydb/core/tx/tx_proxy/ya.make @@ -13,8 +13,8 @@ SRCS( resolvereq.cpp snapshotreq.cpp commitreq.cpp - upload_rows_common_impl.cpp - upload_rows.cpp + upload_rows_common_impl.cpp + upload_rows.cpp ) GENERATE_ENUM_SERIALIZATION(read_table_impl.h) diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h index c030e697d9..c5716f46a7 100644 --- a/ydb/core/viewer/browse.h +++ b/ydb/core/viewer/browse.h @@ -239,7 +239,7 @@ public: auto handlers = Viewer->GetVirtualHandlers(CurrentType, CurrentPath); for (const auto* handler : handlers) { if (handler->BrowseHandler) { - TActorId browseActor = ctx.RegisterWithSameMailbox(handler->BrowseHandler(ctx.SelfID, BrowseContext)); + TActorId browseActor = ctx.RegisterWithSameMailbox(handler->BrowseHandler(ctx.SelfID, BrowseContext)); Handlers.insert(browseActor); ++Requests; } @@ -419,7 +419,7 @@ public: if (it != PipeClients.end()) { return it->second; } - TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); + TActorId pipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); return PipeClients.emplace(tabletId, pipeClient).first->second; } diff --git a/ydb/core/viewer/content/viewer.js b/ydb/core/viewer/content/viewer.js index c3adfb4a12..1108f42278 100644 --- a/ydb/core/viewer/content/viewer.js +++ b/ydb/core/viewer/content/viewer.js @@ -2462,7 +2462,7 @@ function onTreeDataComplete(result, obj, cb) { child.icon = "glyphicon glyphicon-folder-close schema-good"; break; case 2: // Table - child.children = true; + child.children = true; child.icon = "glyphicon glyphicon-list schema-good"; break; case 3: // PQ Group @@ -2485,21 +2485,21 @@ function onTreeDataComplete(result, obj, cb) { children.push(child); } obj.describe = result; - } else if (result.PathDescription.Table) { - if (result.PathDescription.Table.TableIndexes) { - var ilen = result.PathDescription.Table.TableIndexes.length; - for (var i = 0; i < ilen; i++) { - var child = {}; - child.id = concatPaths(result.Path, result.PathDescription.Table.TableIndexes[i].Name); - child.parent = result.Path; - child.text = result.PathDescription.Table.TableIndexes[i].Name; - child.children = true; - child.icon = "glyphicon glyphicon-book schema-good"; - child.data = function(obj1, cb1) { onTreeData(obj1, cb1); }; - children.push(child); - } - obj.describe = result; - } + } else if (result.PathDescription.Table) { + if (result.PathDescription.Table.TableIndexes) { + var ilen = result.PathDescription.Table.TableIndexes.length; + for (var i = 0; i < ilen; i++) { + var child = {}; + child.id = concatPaths(result.Path, result.PathDescription.Table.TableIndexes[i].Name); + child.parent = result.Path; + child.text = result.PathDescription.Table.TableIndexes[i].Name; + child.children = true; + child.icon = "glyphicon glyphicon-book schema-good"; + child.data = function(obj1, cb1) { onTreeData(obj1, cb1); }; + children.push(child); + } + obj.describe = result; + } if (result.PathDescription.Table.CdcStreams) { var ilen = result.PathDescription.Table.CdcStreams.length; for (var i = 0; i < ilen; i++) { @@ -2731,7 +2731,7 @@ function onTreeNodeComplete(result, obj) { } schemaHtml += "</table>"; panelSchema.innerHTML = schemaHtml; - + var indexes = table.TableIndexes; if (indexes !== undefined) { var ilen = indexes.length; @@ -2744,12 +2744,12 @@ function onTreeNodeComplete(result, obj) { if (j < jlen - 1) { schemaHtml += ", "; } - } + } schemaHtml += "</td></tr>" - } + } schemaHtml += "</table>"; panelSchema.innerHTML += schemaHtml; - } + } var cdcStreams = table.CdcStreams; if (cdcStreams !== undefined) { @@ -2764,7 +2764,7 @@ function onTreeNodeComplete(result, obj) { schemaHtml += "</table>"; panelSchema.innerHTML += schemaHtml; } - } + } var tabLen; var tablet; diff --git a/ydb/core/viewer/json_browse.h b/ydb/core/viewer/json_browse.h index abd2a2d4e5..cba672826e 100644 --- a/ydb/core/viewer/json_browse.h +++ b/ydb/core/viewer/json_browse.h @@ -86,7 +86,7 @@ public: if (n.empty() && p.empty()) { n = p = "/"; } - Paths.emplace_back(n, p, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, p, Event->Get()->UserToken))); + Paths.emplace_back(n, p, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, p, Event->Get()->UserToken))); ++pos; prevpos = pos; } else { @@ -96,7 +96,7 @@ public: if (pos != prevpos) { TString n = path.substr(prevpos, pos - prevpos); TString p = path.substr(0, pos); - Paths.emplace_back(n, p, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, p, Event->Get()->UserToken))); + Paths.emplace_back(n, p, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, p, Event->Get()->UserToken))); } } @@ -110,7 +110,7 @@ public: if (Recursive) { ParsePath(path, ctx); } else { - Paths.emplace_back(path, path, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, path, Event->Get()->UserToken))); + Paths.emplace_back(path, path, ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, path, Event->Get()->UserToken))); } Become(&TThis::StateWait, ctx, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); } diff --git a/ydb/core/viewer/json_content.h b/ydb/core/viewer/json_content.h index 8e7031c432..79b230f128 100644 --- a/ydb/core/viewer/json_content.h +++ b/ydb/core/viewer/json_content.h @@ -49,7 +49,7 @@ public: ContentRequestContext.UserToken = Event->Get()->UserToken; } BrowseStarted = ctx.Now(); - ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, ContentRequestContext.Path, Event->Get()->UserToken)); + ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, ContentRequestContext.Path, Event->Get()->UserToken)); TBase::Become( &TThis::StateWaitingBrowse, @@ -117,7 +117,7 @@ private: ContentRequestContext.Timeout -= (ctx.Now() - BrowseStarted); // spawn content retrieval actor - ctx.RegisterWithSameMailbox(contentHandler(Initiator, ContentRequestContext)); + ctx.RegisterWithSameMailbox(contentHandler(Initiator, ContentRequestContext)); Die(ctx); } diff --git a/ydb/core/viewer/json_describe.h b/ydb/core/viewer/json_describe.h index d13d84122f..05ef441f8f 100644 --- a/ydb/core/viewer/json_describe.h +++ b/ydb/core/viewer/json_describe.h @@ -50,7 +50,7 @@ public: record->MutableOptions()->SetReturnPartitionStats(FromStringWithDefault<bool>(params.Get("partition_stats"), false)); record->MutableOptions()->SetReturnPartitioningInfo(FromStringWithDefault<bool>(params.Get("partitioning_info"), true)); } - + void Bootstrap() { const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); diff --git a/ydb/core/viewer/json_metainfo.h b/ydb/core/viewer/json_metainfo.h index a566c64f36..54064d7403 100644 --- a/ydb/core/viewer/json_metainfo.h +++ b/ydb/core/viewer/json_metainfo.h @@ -53,7 +53,7 @@ public: Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); Counters = FromStringWithDefault(params.Get("counters"), false); TString path = params.Get("path"); - BrowseActorID = ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, path, Event->Get()->UserToken)); + BrowseActorID = ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, path, Event->Get()->UserToken)); Become(&TThis::StateWait, ctx, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); } diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h index 464b12adf5..600d8088e1 100644 --- a/ydb/core/viewer/json_query.h +++ b/ydb/core/viewer/json_query.h @@ -95,10 +95,10 @@ public: } NKikimrKqp::TQueryRequest& request = *event->Record.MutableRequest(); request.SetQuery(query); - if (Action.empty() || Action == "execute-script" || Action == "execute") { + if (Action.empty() || Action == "execute-script" || Action == "execute") { request.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - request.SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT); - request.SetKeepSession(false); + request.SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT); + request.SetKeepSession(false); } else if (Action == "execute-scan") { request.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); request.SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN); diff --git a/ydb/core/viewer/json_tabletcounters.h b/ydb/core/viewer/json_tabletcounters.h index d8e108ef6b..7f39910441 100644 --- a/ydb/core/viewer/json_tabletcounters.h +++ b/ydb/core/viewer/json_tabletcounters.h @@ -72,7 +72,7 @@ public: TTabletId tabletId = FromStringWithDefault<TTabletId>(params.Get("tablet_id"), 0); if (tabletId != 0) { Tablets.emplace_back(tabletId); - TActorId PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); + TActorId PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); NTabletPipe::SendData(ctx, PipeClient, new TEvTablet::TEvGetCounters(), tabletId); PipeClients.emplace_back(PipeClient); Become(&TThis::StateRequestedGetCounters, ctx, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); @@ -120,7 +120,7 @@ public: Tablets.erase(std::unique(Tablets.begin(), Tablets.end()), Tablets.end()); } for (auto tabletId : Tablets) { - TActorId PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); + TActorId PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, tabletId, GetPipeClientConfig())); NTabletPipe::SendData(ctx, PipeClient, new TEvTablet::TEvGetCounters(), tabletId); PipeClients.emplace_back(PipeClient); } diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h index 676e5d452a..749422ba39 100644 --- a/ydb/core/viewer/json_tenantinfo.h +++ b/ydb/core/viewer/json_tenantinfo.h @@ -124,7 +124,7 @@ public: } void Handle(NConsole::TEvConsole::TEvListTenantsResponse::TPtr& ev) { - Ydb::Cms::ListDatabasesResult listTenantsResult; + Ydb::Cms::ListDatabasesResult listTenantsResult; ev->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult); for (const TString& path : listTenantsResult.paths()) { if (!Path.empty() && path != Path) { @@ -137,7 +137,7 @@ public: } void Handle(NConsole::TEvConsole::TEvGetTenantStatusResponse::TPtr& ev) { - Ydb::Cms::GetDatabaseStatusResult getTenantStatusResult; + Ydb::Cms::GetDatabaseStatusResult getTenantStatusResult; ev->Get()->Record.GetResponse().operation().result().UnpackTo(&getTenantStatusResult); TString path = getTenantStatusResult.path(); NKikimrViewer::TTenant& tenant = TenantByPath[path]; diff --git a/ydb/core/viewer/json_tenants.h b/ydb/core/viewer/json_tenants.h index ea3ddbdfc9..762d5e3b43 100644 --- a/ydb/core/viewer/json_tenants.h +++ b/ydb/core/viewer/json_tenants.h @@ -63,7 +63,7 @@ public: } void Handle(NConsole::TEvConsole::TEvListTenantsResponse::TPtr& ev) { - Ydb::Cms::ListDatabasesResult listTenantsResult; + Ydb::Cms::ListDatabasesResult listTenantsResult; ev->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult); for (const TString& path : listTenantsResult.paths()) { NKikimrViewer::TTenant& tenant = *Result.AddTenants(); @@ -77,7 +77,7 @@ public: } void Handle(NConsole::TEvConsole::TEvGetTenantStatusResponse::TPtr& ev) { - Ydb::Cms::GetDatabaseStatusResult getTenantStatusResult; + Ydb::Cms::GetDatabaseStatusResult getTenantStatusResult; ev->Get()->Record.GetResponse().operation().result().UnpackTo(&getTenantStatusResult); auto itTenant = TenantIndex.find(getTenantStatusResult.path()); if (itTenant != TenantIndex.end()) { diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 1ad0501863..9f6c822a89 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -332,7 +332,7 @@ message TTenant { string Name = 1; string Id = 2; ETenantType Type = 3; - Ydb.Cms.GetDatabaseStatusResult.State State = 4; + Ydb.Cms.GetDatabaseStatusResult.State State = 4; repeated NKikimrHive.THiveDomainStatsStateCount StateStats = 5; NKikimrTabletBase.TMetrics Metrics = 6; repeated uint32 NodeIds = 7; diff --git a/ydb/core/viewer/ut/ya.make b/ydb/core/viewer/ut/ya.make index d0dcb1bda9..18657a1f66 100644 --- a/ydb/core/viewer/ut/ya.make +++ b/ydb/core/viewer/ut/ya.make @@ -14,8 +14,8 @@ SRCS( viewer_ut.cpp ) -PEERDIR( +PEERDIR( ydb/core/testlib -) - +) + END() diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 0a56c66528..0d10bb8450 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -2,27 +2,27 @@ #include "table_description.h" #include "table_settings.h" #include "ydb_convert.h" - + #include <ydb/core/base/appdata.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/issue_id.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> - + #include <util/generic/hash.h> -namespace NKikimr { - -static NProtoBuf::Timestamp MillisecToProtoTimeStamp(ui64 ms) { - NProtoBuf::Timestamp timestamp; - timestamp.set_seconds((i64)(ms / 1000)); - timestamp.set_nanos((i32)((ms % 1000) * 1000000)); - return timestamp; -} - +namespace NKikimr { + +static NProtoBuf::Timestamp MillisecToProtoTimeStamp(ui64 ms) { + NProtoBuf::Timestamp timestamp; + timestamp.set_seconds((i64)(ms / 1000)); + timestamp.set_nanos((i32)((ms % 1000) * 1000000)); + return timestamp; +} + template <typename TYdbProto> void FillColumnDescriptionImpl(TYdbProto& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { - + splitKeyType.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); splitKeyType.MutableTuple()->MutableElement()->Reserve(in.KeyColumnIdsSize()); THashMap<ui32, size_t> columnIdToKeyPos; @@ -33,35 +33,35 @@ void FillColumnDescriptionImpl(TYdbProto& out, } for (const auto& column : in.GetColumns()) { - NYql::NProto::TypeIds protoType; - if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Got invalid type: " << column.GetType() << " for column: " << column.GetName(); - } - - auto newColumn = out.add_columns(); - newColumn->set_name(column.GetName()); - auto& item = *newColumn->mutable_type()->mutable_optional_type()->mutable_item(); - if (protoType == NYql::NProto::TypeIds::Decimal) { - auto typeParams = item.mutable_decimal_type(); - // TODO: Change TEvDescribeSchemeResult to return decimal params - typeParams->set_precision(22); - typeParams->set_scale(9); - } else { - NMiniKQL::ExportPrimitiveTypeToProto(protoType, item); - } + NYql::NProto::TypeIds protoType; + if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Got invalid type: " << column.GetType() << " for column: " << column.GetName(); + } + + auto newColumn = out.add_columns(); + newColumn->set_name(column.GetName()); + auto& item = *newColumn->mutable_type()->mutable_optional_type()->mutable_item(); + if (protoType == NYql::NProto::TypeIds::Decimal) { + auto typeParams = item.mutable_decimal_type(); + // TODO: Change TEvDescribeSchemeResult to return decimal params + typeParams->set_precision(22); + typeParams->set_scale(9); + } else { + NMiniKQL::ExportPrimitiveTypeToProto(protoType, item); + } if (columnIdToKeyPos.count(column.GetId())) { size_t keyPos = columnIdToKeyPos[column.GetId()]; auto tupleElement = splitKeyType.MutableTuple()->MutableElement(keyPos); tupleElement->SetKind(NKikimrMiniKQL::ETypeKind::Optional); ConvertYdbTypeToMiniKQLType(item, *tupleElement->MutableOptional()->MutableItem()); - } + } - if (column.HasFamilyName()) { - newColumn->set_family(column.GetFamilyName()); - } - } + if (column.HasFamilyName()) { + newColumn->set_family(column.GetFamilyName()); + } + } if (in.HasTTLSettings() && in.GetTTLSettings().HasEnabled()) { const auto& inTTL = in.GetTTLSettings().GetEnabled(); @@ -89,13 +89,13 @@ void FillColumnDescriptionImpl(TYdbProto& out, break; } } -} - +} + void FillColumnDescription(Ydb::Table::DescribeTableResult& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { FillColumnDescriptionImpl(out, splitKeyType, in); } - + void FillColumnDescription(Ydb::Table::CreateTableRequest& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { FillColumnDescriptionImpl(out, splitKeyType, in); @@ -147,9 +147,9 @@ bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusId } bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, - const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error) { + const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error) { - for (const auto& column : in) { + for (const auto& column : in) { NKikimrSchemeOp:: TColumnDescription* cd = out.AddColumns(); cd->SetName(column.name()); if (!column.type().has_optional_type()) { @@ -180,13 +180,13 @@ template <typename TYdbProto> void FillTableBoundaryImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { - for (const auto& boundary : in.GetSplitBoundary()) { - if (boundary.HasSerializedKeyPrefix()) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Unexpected serialized response from txProxy"; - } else if (boundary.HasKeyPrefix()) { + for (const auto& boundary : in.GetSplitBoundary()) { + if (boundary.HasSerializedKeyPrefix()) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Unexpected serialized response from txProxy"; + } else if (boundary.HasKeyPrefix()) { Ydb::TypedValue* ydbValue = nullptr; - + if constexpr (std::is_same<TYdbProto, Ydb::Table::DescribeTableResult>::value) { ydbValue = out.add_shard_key_bounds(); } else if constexpr (std::is_same<TYdbProto, Ydb::Table::CreateTableRequest>::value) { @@ -195,26 +195,26 @@ void FillTableBoundaryImpl(TYdbProto& out, Y_FAIL("Unknown proto type"); } - ConvertMiniKQLTypeToYdbType( - splitKeyType, - *ydbValue->mutable_type()); - - ConvertMiniKQLValueToYdbValue( - splitKeyType, - boundary.GetKeyPrefix(), - *ydbValue->mutable_value()); - } else { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Got invalid boundary"; - } - } -} - + ConvertMiniKQLTypeToYdbType( + splitKeyType, + *ydbValue->mutable_type()); + + ConvertMiniKQLValueToYdbValue( + splitKeyType, + boundary.GetKeyPrefix(), + *ydbValue->mutable_value()); + } else { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Got invalid boundary"; + } + } +} + void FillTableBoundary(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { FillTableBoundaryImpl<Ydb::Table::DescribeTableResult>(out, in, splitKeyType); } - + void FillTableBoundary(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { FillTableBoundaryImpl<Ydb::Table::CreateTableRequest>(out, in, splitKeyType); @@ -224,27 +224,27 @@ template <typename TYdbProto> void FillIndexDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - for (const auto& tableIndex : in.GetTableIndexes()) { - auto index = out.add_indexes(); - - index->set_name(tableIndex.GetName()); - - *index->mutable_index_columns() = { - tableIndex.GetKeyColumnNames().begin(), - tableIndex.GetKeyColumnNames().end() - }; - - *index->mutable_data_columns() = { - tableIndex.GetDataColumnNames().begin(), - tableIndex.GetDataColumnNames().end() - }; - + for (const auto& tableIndex : in.GetTableIndexes()) { + auto index = out.add_indexes(); + + index->set_name(tableIndex.GetName()); + + *index->mutable_index_columns() = { + tableIndex.GetKeyColumnNames().begin(), + tableIndex.GetKeyColumnNames().end() + }; + + *index->mutable_data_columns() = { + tableIndex.GetDataColumnNames().begin(), + tableIndex.GetDataColumnNames().end() + }; + switch (tableIndex.GetType()) { case NKikimrSchemeOp::EIndexType::EIndexTypeGlobal: - *index->mutable_global_index() = Ydb::Table::GlobalIndex(); + *index->mutable_global_index() = Ydb::Table::GlobalIndex(); break; case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: - *index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + *index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); break; default: break; @@ -257,10 +257,10 @@ void FillIndexDescriptionImpl(TYdbProto& out, index->set_status(Ydb::Table::TableIndexDescription::STATUS_BUILDING); } index->set_size_bytes(tableIndex.GetDataSize()); - } - } -} - + } + } +} + void FillIndexDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { FillIndexDescriptionImpl(out, in); @@ -280,24 +280,24 @@ bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, return false; }; - for (const auto& index : in.indexes()) { - auto indexDesc = out.MutableIndexDescription()->Add(); + for (const auto& index : in.indexes()) { + auto indexDesc = out.MutableIndexDescription()->Add(); if (!index.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { - return returnError(Ydb::StatusIds::UNSUPPORTED, "Data column feature is not supported yet"); + return returnError(Ydb::StatusIds::UNSUPPORTED, "Data column feature is not supported yet"); } // common fields indexDesc->SetName(index.name()); - + for (const auto& col : index.index_columns()) { indexDesc->AddKeyColumnNames(col); } - for (const auto& col : index.data_columns()) { - indexDesc->AddDataColumnNames(col); - } - + for (const auto& col : index.data_columns()) { + indexDesc->AddDataColumnNames(col); + } + // specific fields switch (index.type_case()) { case Ydb::Table::TableIndex::kGlobalIndex: @@ -307,7 +307,7 @@ bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, case Ydb::Table::TableIndex::kGlobalAsyncIndex: if (!AppData()->FeatureFlags.GetEnableAsyncIndexes()) { return returnError(Ydb::StatusIds::UNSUPPORTED, "Async indexes are not supported yet"); - } + } indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync); break; @@ -344,37 +344,37 @@ bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, void FillTableStats(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in, bool withPartitionStatistic) { - auto stats = out.mutable_table_stats(); - - if (withPartitionStatistic) { - for (const auto& tablePartitionStat : in.GetTablePartitionStats()) { - auto partition = stats->add_partition_stats(); - partition->set_rows_estimate(tablePartitionStat.GetRowCount()); + auto stats = out.mutable_table_stats(); + + if (withPartitionStatistic) { + for (const auto& tablePartitionStat : in.GetTablePartitionStats()) { + auto partition = stats->add_partition_stats(); + partition->set_rows_estimate(tablePartitionStat.GetRowCount()); partition->set_store_size(tablePartitionStat.GetDataSize() + tablePartitionStat.GetIndexSize()); - } - } - - stats->set_rows_estimate(in.GetTableStats().GetRowCount()); - stats->set_partitions(in.GetTableStats().GetPartCount()); - + } + } + + stats->set_rows_estimate(in.GetTableStats().GetRowCount()); + stats->set_partitions(in.GetTableStats().GetPartCount()); + stats->set_store_size(in.GetTableStats().GetDataSize() + in.GetTableStats().GetIndexSize()); for (const auto& index : in.GetTable().GetTableIndexes()) { stats->set_store_size(stats->store_size() + index.GetDataSize()); } - ui64 modificationTimeMs = in.GetTableStats().GetLastUpdateTime(); + ui64 modificationTimeMs = in.GetTableStats().GetLastUpdateTime(); if (modificationTimeMs) { auto modificationTime = MillisecToProtoTimeStamp(modificationTimeMs); stats->mutable_modification_time()->CopyFrom(modificationTime); } - + ui64 creationTimeMs = in.GetSelf().GetCreateStep(); if (creationTimeMs) { auto creationTime = MillisecToProtoTimeStamp(creationTimeMs); stats->mutable_creation_time()->CopyFrom(creationTime); } -} - +} + static bool IsDefaultFamily(const NKikimrSchemeOp::TFamilyDescription& family) { if (family.HasId() && family.GetId() == 0) { return true; // explicit id 0 @@ -694,8 +694,8 @@ void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, void FillReadReplicasSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { FillReadReplicasSettingsImpl(out, in); -} - +} + bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error) { @@ -731,4 +731,4 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, return true; } -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index adf35ec962..af77a93776 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -13,7 +13,7 @@ void FillColumnDescription(Ydb::Table::CreateTableRequest& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in); // in bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, - const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error); + const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error); bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); // out diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp index aeb3c22ee7..11c804cff5 100644 --- a/ydb/core/ydb_convert/ydb_convert.cpp +++ b/ydb/core/ydb_convert/ydb_convert.cpp @@ -12,262 +12,262 @@ #include <ydb/library/yql/public/udf/udf_types.h> #include <ydb/library/yql/utils/utf8.h> -namespace NKikimr { - -template<typename TOut> -Y_FORCE_INLINE void ConvertMiniKQLTupleTypeToYdbType(const NKikimrMiniKQL::TTupleType& protoTupleType, TOut& output) { - const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); - for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { - const NKikimrMiniKQL::TType& elementProtoType = protoTupleType.GetElement(elementIdx); - ConvertMiniKQLTypeToYdbType(elementProtoType, *output.add_elements()); - } -} - -template<typename TOut> -Y_FORCE_INLINE void ConvertMiniKQLStructTypeToYdbType(const NKikimrMiniKQL::TStructType& protoStructType, TOut& output) { - const ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { - const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); - auto newMember = output.add_members(); - newMember->set_name(protoMember.GetName()); - ConvertMiniKQLTypeToYdbType(protoMember.GetType(), *newMember->mutable_type()); - } -} - -template<typename TOut> -Y_FORCE_INLINE void ConvertYdbTupleTypeToMiniKQLType(const Ydb::TupleType& protoTupleType, TOut& output) { - const ui32 elementsCount = static_cast<ui32>(protoTupleType.elementsSize()); - for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { - const Ydb::Type& elementProtoType = protoTupleType.elements(elementIdx); - ConvertYdbTypeToMiniKQLType(elementProtoType, *output.AddElement()); - } -} - -template<typename TOut> -Y_FORCE_INLINE void ConvertYdbStructTypeToMiniKQLType(const Ydb::StructType& protoStructType, TOut& output) { - const ui32 membersCount = static_cast<ui32>(protoStructType.membersSize()); - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { - const Ydb::StructMember& protoMember = protoStructType.members(memberNum); - auto newMember = output.AddMember(); - newMember->SetName(protoMember.name()); - ConvertYdbTypeToMiniKQLType(protoMember.type(), *newMember->MutableType()); - } -} - +namespace NKikimr { + +template<typename TOut> +Y_FORCE_INLINE void ConvertMiniKQLTupleTypeToYdbType(const NKikimrMiniKQL::TTupleType& protoTupleType, TOut& output) { + const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); + for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { + const NKikimrMiniKQL::TType& elementProtoType = protoTupleType.GetElement(elementIdx); + ConvertMiniKQLTypeToYdbType(elementProtoType, *output.add_elements()); + } +} + +template<typename TOut> +Y_FORCE_INLINE void ConvertMiniKQLStructTypeToYdbType(const NKikimrMiniKQL::TStructType& protoStructType, TOut& output) { + const ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); + auto newMember = output.add_members(); + newMember->set_name(protoMember.GetName()); + ConvertMiniKQLTypeToYdbType(protoMember.GetType(), *newMember->mutable_type()); + } +} + +template<typename TOut> +Y_FORCE_INLINE void ConvertYdbTupleTypeToMiniKQLType(const Ydb::TupleType& protoTupleType, TOut& output) { + const ui32 elementsCount = static_cast<ui32>(protoTupleType.elementsSize()); + for (ui32 elementIdx = 0; elementIdx < elementsCount; ++elementIdx) { + const Ydb::Type& elementProtoType = protoTupleType.elements(elementIdx); + ConvertYdbTypeToMiniKQLType(elementProtoType, *output.AddElement()); + } +} + +template<typename TOut> +Y_FORCE_INLINE void ConvertYdbStructTypeToMiniKQLType(const Ydb::StructType& protoStructType, TOut& output) { + const ui32 membersCount = static_cast<ui32>(protoStructType.membersSize()); + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + const Ydb::StructMember& protoMember = protoStructType.members(memberNum); + auto newMember = output.AddMember(); + newMember->SetName(protoMember.name()); + ConvertYdbTypeToMiniKQLType(protoMember.type(), *newMember->MutableType()); + } +} + void ConvertMiniKQLTypeToYdbType(const NKikimrMiniKQL::TType& input, Ydb::Type& output) { - switch (input.GetKind()) { - case NKikimrMiniKQL::ETypeKind::Void: { - output.set_void_type(::google::protobuf::NULL_VALUE); - break; - } + switch (input.GetKind()) { + case NKikimrMiniKQL::ETypeKind::Void: { + output.set_void_type(::google::protobuf::NULL_VALUE); + break; + } case NKikimrMiniKQL::ETypeKind::Null: { output.set_null_type(::google::protobuf::NULL_VALUE); break; } - case NKikimrMiniKQL::ETypeKind::Data: { - const NKikimrMiniKQL::TDataType& protoData = input.GetData(); - NUdf::TDataTypeId schemeType = protoData.GetScheme(); - Y_VERIFY(NUdf::FindDataSlot(schemeType), "unknown type id: %d", (int) schemeType); - if (schemeType == NYql::NProto::TypeIds::Decimal) { - auto typeParams = output.mutable_decimal_type(); - typeParams->set_precision(protoData.GetDecimalParams().GetPrecision()); - typeParams->set_scale(protoData.GetDecimalParams().GetScale()); - } else { - NMiniKQL::ExportPrimitiveTypeToProto(schemeType, output); - } - break; - } - case NKikimrMiniKQL::ETypeKind::Optional: { - const NKikimrMiniKQL::TOptionalType& protoOptionalType = input.GetOptional(); - ConvertMiniKQLTypeToYdbType(protoOptionalType.GetItem(), *output.mutable_optional_type()->mutable_item()); - break; - } - case NKikimrMiniKQL::ETypeKind::List: { - const NKikimrMiniKQL::TListType& protoListType = input.GetList(); - ConvertMiniKQLTypeToYdbType(protoListType.GetItem(), *output.mutable_list_type()->mutable_item()); - break; - } - case NKikimrMiniKQL::ETypeKind::Tuple: { - const NKikimrMiniKQL::TTupleType& protoTupleType = input.GetTuple(); - ConvertMiniKQLTupleTypeToYdbType(protoTupleType, *output.mutable_tuple_type()); - break; - } - case NKikimrMiniKQL::ETypeKind::Struct: { - const NKikimrMiniKQL::TStructType& protoStructType = input.GetStruct(); - ConvertMiniKQLStructTypeToYdbType(protoStructType, *output.mutable_struct_type()); - break; - } - case NKikimrMiniKQL::ETypeKind::Dict: { - const NKikimrMiniKQL::TDictType& protoDictType = input.GetDict(); - ConvertMiniKQLTypeToYdbType(protoDictType.GetKey(), *output.mutable_dict_type()->mutable_key()); - ConvertMiniKQLTypeToYdbType(protoDictType.GetPayload(), *output.mutable_dict_type()->mutable_payload()); - break; - } - case NKikimrMiniKQL::ETypeKind::Variant: { - const NKikimrMiniKQL::TVariantType& protoVariantType = input.GetVariant(); - auto variantOut = output.mutable_variant_type(); - switch (protoVariantType.type_case()) { - case NKikimrMiniKQL::TVariantType::kTupleItems: { - const NKikimrMiniKQL::TTupleType& protoTupleType = protoVariantType.GetTupleItems(); - ConvertMiniKQLTupleTypeToYdbType(protoTupleType, *variantOut->mutable_tuple_items()); - break; - } - case NKikimrMiniKQL::TVariantType::kStructItems: { - const NKikimrMiniKQL::TStructType& protoStructType = protoVariantType.GetStructItems(); - ConvertMiniKQLStructTypeToYdbType(protoStructType, *variantOut->mutable_struct_items()); - break; - } - default: - ythrow yexception() << "Unknown variant type representation: " - << protoVariantType.DebugString(); - } - break; - } - default: { - Y_FAIL("Unknown protobuf type: %s", input.DebugString().c_str()); - } - } -} - + case NKikimrMiniKQL::ETypeKind::Data: { + const NKikimrMiniKQL::TDataType& protoData = input.GetData(); + NUdf::TDataTypeId schemeType = protoData.GetScheme(); + Y_VERIFY(NUdf::FindDataSlot(schemeType), "unknown type id: %d", (int) schemeType); + if (schemeType == NYql::NProto::TypeIds::Decimal) { + auto typeParams = output.mutable_decimal_type(); + typeParams->set_precision(protoData.GetDecimalParams().GetPrecision()); + typeParams->set_scale(protoData.GetDecimalParams().GetScale()); + } else { + NMiniKQL::ExportPrimitiveTypeToProto(schemeType, output); + } + break; + } + case NKikimrMiniKQL::ETypeKind::Optional: { + const NKikimrMiniKQL::TOptionalType& protoOptionalType = input.GetOptional(); + ConvertMiniKQLTypeToYdbType(protoOptionalType.GetItem(), *output.mutable_optional_type()->mutable_item()); + break; + } + case NKikimrMiniKQL::ETypeKind::List: { + const NKikimrMiniKQL::TListType& protoListType = input.GetList(); + ConvertMiniKQLTypeToYdbType(protoListType.GetItem(), *output.mutable_list_type()->mutable_item()); + break; + } + case NKikimrMiniKQL::ETypeKind::Tuple: { + const NKikimrMiniKQL::TTupleType& protoTupleType = input.GetTuple(); + ConvertMiniKQLTupleTypeToYdbType(protoTupleType, *output.mutable_tuple_type()); + break; + } + case NKikimrMiniKQL::ETypeKind::Struct: { + const NKikimrMiniKQL::TStructType& protoStructType = input.GetStruct(); + ConvertMiniKQLStructTypeToYdbType(protoStructType, *output.mutable_struct_type()); + break; + } + case NKikimrMiniKQL::ETypeKind::Dict: { + const NKikimrMiniKQL::TDictType& protoDictType = input.GetDict(); + ConvertMiniKQLTypeToYdbType(protoDictType.GetKey(), *output.mutable_dict_type()->mutable_key()); + ConvertMiniKQLTypeToYdbType(protoDictType.GetPayload(), *output.mutable_dict_type()->mutable_payload()); + break; + } + case NKikimrMiniKQL::ETypeKind::Variant: { + const NKikimrMiniKQL::TVariantType& protoVariantType = input.GetVariant(); + auto variantOut = output.mutable_variant_type(); + switch (protoVariantType.type_case()) { + case NKikimrMiniKQL::TVariantType::kTupleItems: { + const NKikimrMiniKQL::TTupleType& protoTupleType = protoVariantType.GetTupleItems(); + ConvertMiniKQLTupleTypeToYdbType(protoTupleType, *variantOut->mutable_tuple_items()); + break; + } + case NKikimrMiniKQL::TVariantType::kStructItems: { + const NKikimrMiniKQL::TStructType& protoStructType = protoVariantType.GetStructItems(); + ConvertMiniKQLStructTypeToYdbType(protoStructType, *variantOut->mutable_struct_items()); + break; + } + default: + ythrow yexception() << "Unknown variant type representation: " + << protoVariantType.DebugString(); + } + break; + } + default: { + Y_FAIL("Unknown protobuf type: %s", input.DebugString().c_str()); + } + } +} + void ConvertYdbTypeToMiniKQLType(const Ydb::Type& input, NKikimrMiniKQL::TType& output) { switch (input.type_case()) { case Ydb::Type::kVoidType: - output.SetKind(NKikimrMiniKQL::ETypeKind::Void); - break; + output.SetKind(NKikimrMiniKQL::ETypeKind::Void); + break; case Ydb::Type::kNullType: output.SetKind(NKikimrMiniKQL::ETypeKind::Null); break; case Ydb::Type::kTypeId: - output.SetKind(NKikimrMiniKQL::ETypeKind::Data); + output.SetKind(NKikimrMiniKQL::ETypeKind::Data); output.MutableData()->SetScheme(input.type_id()); - break; - case Ydb::Type::kDecimalType: { + break; + case Ydb::Type::kDecimalType: { // TODO: Decimal parameters output.SetKind(NKikimrMiniKQL::ETypeKind::Data); - auto data = output.MutableData(); - data->SetScheme(NYql::NProto::TypeIds::Decimal); - data->MutableDecimalParams()->SetPrecision(input.decimal_type().precision()); - data->MutableDecimalParams()->SetScale(input.decimal_type().scale()); + auto data = output.MutableData(); + data->SetScheme(NYql::NProto::TypeIds::Decimal); + data->MutableDecimalParams()->SetPrecision(input.decimal_type().precision()); + data->MutableDecimalParams()->SetScale(input.decimal_type().scale()); break; - } + } case Ydb::Type::kOptionalType: - output.SetKind(NKikimrMiniKQL::ETypeKind::Optional); - ConvertYdbTypeToMiniKQLType(input.optional_type().item(), *output.MutableOptional()->MutableItem()); - break; + output.SetKind(NKikimrMiniKQL::ETypeKind::Optional); + ConvertYdbTypeToMiniKQLType(input.optional_type().item(), *output.MutableOptional()->MutableItem()); + break; case Ydb::Type::kListType: - output.SetKind(NKikimrMiniKQL::ETypeKind::List); - ConvertYdbTypeToMiniKQLType(input.list_type().item(), *output.MutableList()->MutableItem()); - break; + output.SetKind(NKikimrMiniKQL::ETypeKind::List); + ConvertYdbTypeToMiniKQLType(input.list_type().item(), *output.MutableList()->MutableItem()); + break; case Ydb::Type::kTupleType: { - output.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); + output.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); const Ydb::TupleType& protoTupleType = input.tuple_type(); - ConvertYdbTupleTypeToMiniKQLType(protoTupleType, *output.MutableTuple()); - break; - } + ConvertYdbTupleTypeToMiniKQLType(protoTupleType, *output.MutableTuple()); + break; + } case Ydb::Type::kStructType: { - output.SetKind(NKikimrMiniKQL::ETypeKind::Struct); + output.SetKind(NKikimrMiniKQL::ETypeKind::Struct); const Ydb::StructType& protoStructType = input.struct_type(); - ConvertYdbStructTypeToMiniKQLType(protoStructType, *output.MutableStruct()); - break; - } + ConvertYdbStructTypeToMiniKQLType(protoStructType, *output.MutableStruct()); + break; + } case Ydb::Type::kDictType: { - output.SetKind(NKikimrMiniKQL::ETypeKind::Dict); + output.SetKind(NKikimrMiniKQL::ETypeKind::Dict); const Ydb::DictType& protoDictType = input.dict_type(); - ConvertYdbTypeToMiniKQLType(protoDictType.key(), *output.MutableDict()->MutableKey()); - ConvertYdbTypeToMiniKQLType(protoDictType.payload(), *output.MutableDict()->MutablePayload()); - break; - } - case Ydb::Type::kVariantType: { - output.SetKind(NKikimrMiniKQL::ETypeKind::Variant); - const Ydb::VariantType& protoVariantType = input.variant_type(); - auto variantOut = output.MutableVariant(); - switch (protoVariantType.type_case()) { - case Ydb::VariantType::kTupleItems: { - const Ydb::TupleType& protoTupleType = protoVariantType.tuple_items(); - ConvertYdbTupleTypeToMiniKQLType(protoTupleType, *variantOut->MutableTupleItems()); - break; - } - case Ydb::VariantType::kStructItems: { - const Ydb::StructType& protoStructType = protoVariantType.struct_items(); - ConvertYdbStructTypeToMiniKQLType(protoStructType, *variantOut->MutableStructItems()); - break; - } - default: - ythrow yexception() << "Unknown variant type representation: " - << protoVariantType.DebugString(); - } - break; - } - default: { - ythrow yexception() << "Unknown protobuf type: " - << input.DebugString(); - } - } - -} - + ConvertYdbTypeToMiniKQLType(protoDictType.key(), *output.MutableDict()->MutableKey()); + ConvertYdbTypeToMiniKQLType(protoDictType.payload(), *output.MutableDict()->MutablePayload()); + break; + } + case Ydb::Type::kVariantType: { + output.SetKind(NKikimrMiniKQL::ETypeKind::Variant); + const Ydb::VariantType& protoVariantType = input.variant_type(); + auto variantOut = output.MutableVariant(); + switch (protoVariantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + const Ydb::TupleType& protoTupleType = protoVariantType.tuple_items(); + ConvertYdbTupleTypeToMiniKQLType(protoTupleType, *variantOut->MutableTupleItems()); + break; + } + case Ydb::VariantType::kStructItems: { + const Ydb::StructType& protoStructType = protoVariantType.struct_items(); + ConvertYdbStructTypeToMiniKQLType(protoStructType, *variantOut->MutableStructItems()); + break; + } + default: + ythrow yexception() << "Unknown variant type representation: " + << protoVariantType.DebugString(); + } + break; + } + default: { + ythrow yexception() << "Unknown protobuf type: " + << input.DebugString(); + } + } + +} + Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const NKikimrMiniKQL::TValue& value, Ydb::Value& res) { - switch (typeId) { - case NUdf::TDataType<bool>::Id: - res.set_bool_value(value.GetBool()); - break; - case NUdf::TDataType<ui8>::Id: - res.set_uint32_value(value.GetUint32()); - break; - case NUdf::TDataType<i8>::Id: - res.set_int32_value(value.GetInt32()); - break; - case NUdf::TDataType<ui16>::Id: - res.set_uint32_value(value.GetUint32()); - break; - case NUdf::TDataType<i16>::Id: - res.set_int32_value(value.GetInt32()); - break; - case NUdf::TDataType<i32>::Id: - res.set_int32_value(value.GetInt32()); - break; - case NUdf::TDataType<ui32>::Id: - res.set_uint32_value(value.GetUint32()); - break; - case NUdf::TDataType<i64>::Id: - res.set_int64_value(value.GetInt64()); - break; - case NUdf::TDataType<ui64>::Id: - res.set_uint64_value(value.GetUint64()); - break; - case NUdf::TDataType<float>::Id: - res.set_float_value(value.GetFloat()); - break; - case NUdf::TDataType<double>::Id: - res.set_double_value(value.GetDouble()); - break; - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: { - const auto& stringRef = value.GetText(); + switch (typeId) { + case NUdf::TDataType<bool>::Id: + res.set_bool_value(value.GetBool()); + break; + case NUdf::TDataType<ui8>::Id: + res.set_uint32_value(value.GetUint32()); + break; + case NUdf::TDataType<i8>::Id: + res.set_int32_value(value.GetInt32()); + break; + case NUdf::TDataType<ui16>::Id: + res.set_uint32_value(value.GetUint32()); + break; + case NUdf::TDataType<i16>::Id: + res.set_int32_value(value.GetInt32()); + break; + case NUdf::TDataType<i32>::Id: + res.set_int32_value(value.GetInt32()); + break; + case NUdf::TDataType<ui32>::Id: + res.set_uint32_value(value.GetUint32()); + break; + case NUdf::TDataType<i64>::Id: + res.set_int64_value(value.GetInt64()); + break; + case NUdf::TDataType<ui64>::Id: + res.set_uint64_value(value.GetUint64()); + break; + case NUdf::TDataType<float>::Id: + res.set_float_value(value.GetFloat()); + break; + case NUdf::TDataType<double>::Id: + res.set_double_value(value.GetDouble()); + break; + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: { + const auto& stringRef = value.GetText(); res.set_text_value(stringRef.data(), stringRef.size()); - break; - } - case NUdf::TDataType<NUdf::TDate>::Id: - res.set_uint32_value(value.GetUint32()); - break; - case NUdf::TDataType<NUdf::TDatetime>::Id: - res.set_uint32_value(value.GetUint32()); - break; - case NUdf::TDataType<NUdf::TTimestamp>::Id: - res.set_uint64_value(value.GetUint64()); - break; - case NUdf::TDataType<NUdf::TInterval>::Id: - res.set_int64_value(value.GetInt64()); - break; - case NUdf::TDataType<NUdf::TDecimal>::Id: - case NUdf::TDataType<NUdf::TUuid>::Id: { - res.set_low_128(value.GetLow128()); - res.set_high_128(value.GetHi128()); - break; - } + break; + } + case NUdf::TDataType<NUdf::TDate>::Id: + res.set_uint32_value(value.GetUint32()); + break; + case NUdf::TDataType<NUdf::TDatetime>::Id: + res.set_uint32_value(value.GetUint32()); + break; + case NUdf::TDataType<NUdf::TTimestamp>::Id: + res.set_uint64_value(value.GetUint64()); + break; + case NUdf::TDataType<NUdf::TInterval>::Id: + res.set_int64_value(value.GetInt64()); + break; + case NUdf::TDataType<NUdf::TDecimal>::Id: + case NUdf::TDataType<NUdf::TUuid>::Id: { + res.set_low_128(value.GetLow128()); + res.set_high_128(value.GetHi128()); + break; + } case NUdf::TDataType<NUdf::TJsonDocument>::Id: { const auto json = NBinaryJson::SerializeToJson(value.GetBytes()); res.set_text_value(json); @@ -279,12 +279,12 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const NKikimrMiniKQL:: res.set_text_value(*number); break; } - default: - const auto& stringRef = value.GetBytes(); + default: + const auto& stringRef = value.GetBytes(); res.set_bytes_value(stringRef.data(), stringRef.size()); - } -} - + } +} + Y_FORCE_INLINE void CheckTypeId(i32 id, i32 expected, std::string_view typeName) { if (id != expected) { throw yexception() << "Invalid value representation for type: " << typeName; @@ -292,51 +292,51 @@ Y_FORCE_INLINE void CheckTypeId(i32 id, i32 expected, std::string_view typeName) } Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& value, NKikimrMiniKQL::TValue& res) { - switch (typeId) { - case NUdf::TDataType<bool>::Id: + switch (typeId) { + case NUdf::TDataType<bool>::Id: CheckTypeId(value.value_case(), Ydb::Value::kBoolValue, "Bool"); - res.SetBool(value.bool_value()); - break; - case NUdf::TDataType<ui8>::Id: + res.SetBool(value.bool_value()); + break; + case NUdf::TDataType<ui8>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint32Value, "Uint8"); - res.SetUint32(value.uint32_value()); - break; - case NUdf::TDataType<i8>::Id: + res.SetUint32(value.uint32_value()); + break; + case NUdf::TDataType<i8>::Id: CheckTypeId(value.value_case(), Ydb::Value::kInt32Value, "Int8"); - res.SetInt32(value.int32_value()); - break; - case NUdf::TDataType<ui16>::Id: + res.SetInt32(value.int32_value()); + break; + case NUdf::TDataType<ui16>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint32Value, "Uint16"); - res.SetUint32(value.uint32_value()); - break; - case NUdf::TDataType<i16>::Id: + res.SetUint32(value.uint32_value()); + break; + case NUdf::TDataType<i16>::Id: CheckTypeId(value.value_case(), Ydb::Value::kInt32Value, "Int16"); - res.SetInt32(value.int32_value()); - break; - case NUdf::TDataType<i32>::Id: + res.SetInt32(value.int32_value()); + break; + case NUdf::TDataType<i32>::Id: CheckTypeId(value.value_case(), Ydb::Value::kInt32Value, "Int32"); - res.SetInt32(value.int32_value()); - break; - case NUdf::TDataType<ui32>::Id: + res.SetInt32(value.int32_value()); + break; + case NUdf::TDataType<ui32>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint32Value, "Uint32"); - res.SetUint32(value.uint32_value()); - break; - case NUdf::TDataType<i64>::Id: + res.SetUint32(value.uint32_value()); + break; + case NUdf::TDataType<i64>::Id: CheckTypeId(value.value_case(), Ydb::Value::kInt64Value, "Int64"); - res.SetInt64(value.int64_value()); - break; - case NUdf::TDataType<ui64>::Id: + res.SetInt64(value.int64_value()); + break; + case NUdf::TDataType<ui64>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint64Value, "Uint64"); - res.SetUint64(value.uint64_value()); - break; - case NUdf::TDataType<float>::Id: + res.SetUint64(value.uint64_value()); + break; + case NUdf::TDataType<float>::Id: CheckTypeId(value.value_case(), Ydb::Value::kFloatValue, "Float"); - res.SetFloat(value.float_value()); - break; - case NUdf::TDataType<double>::Id: + res.SetFloat(value.float_value()); + break; + case NUdf::TDataType<double>::Id: CheckTypeId(value.value_case(), Ydb::Value::kDoubleValue, "Double"); - res.SetDouble(value.double_value()); - break; + res.SetDouble(value.double_value()); + break; case NUdf::TDataType<NUdf::TTzDate>::Id: { CheckTypeId(value.value_case(), Ydb::Value::kTextValue, "TzDate"); const auto& stringRef = value.text_value(); @@ -364,48 +364,48 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& valu res.SetText(stringRef.data(), stringRef.size()); break; } - case NUdf::TDataType<NUdf::TUtf8>::Id: { + case NUdf::TDataType<NUdf::TUtf8>::Id: { CheckTypeId(value.value_case(), Ydb::Value::kTextValue, "Utf8"); - const auto& stringRef = value.text_value(); + const auto& stringRef = value.text_value(); if (!NYql::IsUtf8(stringRef)) { throw yexception() << "Invalid Utf8 value"; } res.SetText(stringRef.data(), stringRef.size()); - break; + break; } case NUdf::TDataType<NUdf::TDate>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint32Value, "Date"); if (value.uint32_value() >= NUdf::MAX_DATE) { throw yexception() << "Invalid Date value"; - } - res.SetUint32(value.uint32_value()); - break; - case NUdf::TDataType<NUdf::TDatetime>::Id: + } + res.SetUint32(value.uint32_value()); + break; + case NUdf::TDataType<NUdf::TDatetime>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint32Value, "Datetime"); if (value.uint32_value() >= NUdf::MAX_DATETIME) { throw yexception() << "Invalid Datetime value"; } - res.SetUint32(value.uint32_value()); - break; - case NUdf::TDataType<NUdf::TTimestamp>::Id: + res.SetUint32(value.uint32_value()); + break; + case NUdf::TDataType<NUdf::TTimestamp>::Id: CheckTypeId(value.value_case(), Ydb::Value::kUint64Value, "Timestamp"); if (value.uint64_value() >= NUdf::MAX_TIMESTAMP) { throw yexception() << "Invalid Timestamp value"; } - res.SetUint64(value.uint64_value()); - break; - case NUdf::TDataType<NUdf::TInterval>::Id: + res.SetUint64(value.uint64_value()); + break; + case NUdf::TDataType<NUdf::TInterval>::Id: CheckTypeId(value.value_case(), Ydb::Value::kInt64Value, "Interval"); if ((ui64)std::abs(value.int64_value()) >= NUdf::MAX_TIMESTAMP) { throw yexception() << "Invalid Interval value"; } - res.SetInt64(value.int64_value()); - break; - case NUdf::TDataType<NUdf::TUuid>::Id: + res.SetInt64(value.int64_value()); + break; + case NUdf::TDataType<NUdf::TUuid>::Id: CheckTypeId(value.value_case(), Ydb::Value::kLow128, "Uuid"); - res.SetLow128(value.low_128()); - res.SetHi128(value.high_128()); - break; + res.SetLow128(value.low_128()); + res.SetHi128(value.high_128()); + break; case NUdf::TDataType<NUdf::TJsonDocument>::Id: { CheckTypeId(value.value_case(), Ydb::Value::kTextValue, "JsonDocument"); const auto binaryJson = NBinaryJson::SerializeToBinaryJson(value.text_value()); @@ -426,7 +426,7 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& valu } case NUdf::TDataType<char*>::Id: { CheckTypeId(value.value_case(), Ydb::Value::kBytesValue, "String"); - const auto& stringRef = value.bytes_value(); + const auto& stringRef = value.bytes_value(); res.SetBytes(stringRef.data(), stringRef.size()); break; } @@ -441,145 +441,145 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& valu } default: throw yexception() << "Unsupported data type: " << typeId; - } -} - -void ConvertMiniKQLValueToYdbValue(const NKikimrMiniKQL::TType& inputType, - const NKikimrMiniKQL::TValue& inputValue, + } +} + +void ConvertMiniKQLValueToYdbValue(const NKikimrMiniKQL::TType& inputType, + const NKikimrMiniKQL::TValue& inputValue, Ydb::Value& output) { - switch (inputType.GetKind()) { - case NKikimrMiniKQL::ETypeKind::Void: { - break; - } + switch (inputType.GetKind()) { + case NKikimrMiniKQL::ETypeKind::Void: { + break; + } case NKikimrMiniKQL::ETypeKind::Null: { output.set_null_flag_value(::google::protobuf::NULL_VALUE); break; } - case NKikimrMiniKQL::ETypeKind::Data: { - const NKikimrMiniKQL::TDataType& protoData = inputType.GetData(); - const NUdf::TDataTypeId typeId = protoData.GetScheme(); - ConvertData(typeId, inputValue, output); - break; - } - case NKikimrMiniKQL::ETypeKind::Optional: { - const NKikimrMiniKQL::TOptionalType& protoOptionalType = inputType.GetOptional(); - if (inputValue.HasOptional()) { - // Optional type, and there is somthing inside - keep going - const NKikimrMiniKQL::TType* nextType = &protoOptionalType.GetItem(); - const NKikimrMiniKQL::TValue* curValue = &inputValue; - ui32 optionalCounter = 0; - while (nextType->GetKind() == NKikimrMiniKQL::ETypeKind::Optional && curValue->HasOptional()) { - optionalCounter++; - nextType = &nextType->GetOptional().GetItem(); - curValue = &curValue->GetOptional(); - } - if (curValue->HasOptional()) { - // Next type is not optional and we have optional value - write this value (without nested Item messages) - ConvertMiniKQLValueToYdbValue(*nextType, curValue->GetOptional(), output); - } else { - // Next type is not optional and we have no optional value - Optional<Optional<T>>(Null) case. - // Write corresponding number of nested Value messages, and NullFlag to the last one + case NKikimrMiniKQL::ETypeKind::Data: { + const NKikimrMiniKQL::TDataType& protoData = inputType.GetData(); + const NUdf::TDataTypeId typeId = protoData.GetScheme(); + ConvertData(typeId, inputValue, output); + break; + } + case NKikimrMiniKQL::ETypeKind::Optional: { + const NKikimrMiniKQL::TOptionalType& protoOptionalType = inputType.GetOptional(); + if (inputValue.HasOptional()) { + // Optional type, and there is somthing inside - keep going + const NKikimrMiniKQL::TType* nextType = &protoOptionalType.GetItem(); + const NKikimrMiniKQL::TValue* curValue = &inputValue; + ui32 optionalCounter = 0; + while (nextType->GetKind() == NKikimrMiniKQL::ETypeKind::Optional && curValue->HasOptional()) { + optionalCounter++; + nextType = &nextType->GetOptional().GetItem(); + curValue = &curValue->GetOptional(); + } + if (curValue->HasOptional()) { + // Next type is not optional and we have optional value - write this value (without nested Item messages) + ConvertMiniKQLValueToYdbValue(*nextType, curValue->GetOptional(), output); + } else { + // Next type is not optional and we have no optional value - Optional<Optional<T>>(Null) case. + // Write corresponding number of nested Value messages, and NullFlag to the last one Ydb::Value* resValue = &output; - while (optionalCounter--) { - resValue = resValue->mutable_nested_value(); - } - resValue->set_null_flag_value(::google::protobuf::NULL_VALUE); - } - } else { - // Optional type, but there isn't optional value - single empty level - convert to NullFlag - output.set_null_flag_value(::google::protobuf::NULL_VALUE); - } - break; - } - case NKikimrMiniKQL::ETypeKind::List: { - const NKikimrMiniKQL::TListType& protoListType = inputType.GetList(); - const NKikimrMiniKQL::TType& protoItemType = protoListType.GetItem(); - for (const auto& x : inputValue.GetList()) { - ConvertMiniKQLValueToYdbValue(protoItemType, x, *output.mutable_items()->Add()); - } - break; - } - case NKikimrMiniKQL::ETypeKind::Struct: { - const NKikimrMiniKQL::TStructType& protoStructType = inputType.GetStruct(); - ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); - if (membersCount != inputValue.StructSize()) { - ythrow yexception() << "Number of struct fields and their types mismatched"; - } - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { - const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); - ConvertMiniKQLValueToYdbValue(protoMember.GetType(), inputValue.GetStruct(memberNum), *output.mutable_items()->Add()); - } - break; - } - case NKikimrMiniKQL::ETypeKind::Dict: { - const NKikimrMiniKQL::TDictType& protoDictType = inputType.GetDict(); - for (const auto& x : inputValue.GetDict()) { - auto pair = output.mutable_pairs()->Add(); - ConvertMiniKQLValueToYdbValue(protoDictType.GetKey(), x.GetKey(), *pair->mutable_key()); - ConvertMiniKQLValueToYdbValue(protoDictType.GetPayload(), x.GetPayload(), *pair->mutable_payload()); - } - break; - } - case NKikimrMiniKQL::ETypeKind::Tuple: { - const NKikimrMiniKQL::TTupleType& protoTupleType = inputType.GetTuple(); - ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); - if (elementsCount != inputValue.TupleSize()) { - throw yexception() << "Number of tuple elements and their types mismatched"; - } - for (ui32 elementNum = 0; elementNum < elementsCount; ++elementNum) { - const NKikimrMiniKQL::TType& protoMember = protoTupleType.GetElement(elementNum); - ConvertMiniKQLValueToYdbValue(protoMember, inputValue.GetTuple(elementNum), *output.mutable_items()->Add()); - } - break; - } - case NKikimrMiniKQL::ETypeKind::Variant: { - const NKikimrMiniKQL::TVariantType& protoVariantType = inputType.GetVariant(); - const ui32 variantIndex = inputValue.GetVariantIndex(); - output.set_variant_index(variantIndex); - switch (protoVariantType.type_case()) { - case NKikimrMiniKQL::TVariantType::kTupleItems: { - if (variantIndex >= protoVariantType.GetTupleItems().ElementSize()) { - throw yexception() << "Variant index out of type range"; - } - const NKikimrMiniKQL::TType& nextType = protoVariantType.GetTupleItems().GetElement(variantIndex); - ConvertMiniKQLValueToYdbValue(nextType, inputValue.GetOptional(), *output.mutable_nested_value()); - break; - } - case NKikimrMiniKQL::TVariantType::kStructItems: { - if (variantIndex >= protoVariantType.GetStructItems().MemberSize()) { - throw yexception() << "Variant index out of type range"; - } - const NKikimrMiniKQL::TType& nextType = protoVariantType.GetStructItems().GetMember(variantIndex).GetType(); - ConvertMiniKQLValueToYdbValue(nextType, inputValue.GetOptional(), *output.mutable_nested_value()); - break; - } - default: - ythrow yexception() << "Unknown variant type representation: " - << protoVariantType.DebugString(); - } - break; - } - default: { - ythrow yexception() << "Unknown protobuf type: " - << inputType.DebugString(); - } - } -} - + while (optionalCounter--) { + resValue = resValue->mutable_nested_value(); + } + resValue->set_null_flag_value(::google::protobuf::NULL_VALUE); + } + } else { + // Optional type, but there isn't optional value - single empty level - convert to NullFlag + output.set_null_flag_value(::google::protobuf::NULL_VALUE); + } + break; + } + case NKikimrMiniKQL::ETypeKind::List: { + const NKikimrMiniKQL::TListType& protoListType = inputType.GetList(); + const NKikimrMiniKQL::TType& protoItemType = protoListType.GetItem(); + for (const auto& x : inputValue.GetList()) { + ConvertMiniKQLValueToYdbValue(protoItemType, x, *output.mutable_items()->Add()); + } + break; + } + case NKikimrMiniKQL::ETypeKind::Struct: { + const NKikimrMiniKQL::TStructType& protoStructType = inputType.GetStruct(); + ui32 membersCount = static_cast<ui32>(protoStructType.MemberSize()); + if (membersCount != inputValue.StructSize()) { + ythrow yexception() << "Number of struct fields and their types mismatched"; + } + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + const NKikimrMiniKQL::TMember& protoMember = protoStructType.GetMember(memberNum); + ConvertMiniKQLValueToYdbValue(protoMember.GetType(), inputValue.GetStruct(memberNum), *output.mutable_items()->Add()); + } + break; + } + case NKikimrMiniKQL::ETypeKind::Dict: { + const NKikimrMiniKQL::TDictType& protoDictType = inputType.GetDict(); + for (const auto& x : inputValue.GetDict()) { + auto pair = output.mutable_pairs()->Add(); + ConvertMiniKQLValueToYdbValue(protoDictType.GetKey(), x.GetKey(), *pair->mutable_key()); + ConvertMiniKQLValueToYdbValue(protoDictType.GetPayload(), x.GetPayload(), *pair->mutable_payload()); + } + break; + } + case NKikimrMiniKQL::ETypeKind::Tuple: { + const NKikimrMiniKQL::TTupleType& protoTupleType = inputType.GetTuple(); + ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); + if (elementsCount != inputValue.TupleSize()) { + throw yexception() << "Number of tuple elements and their types mismatched"; + } + for (ui32 elementNum = 0; elementNum < elementsCount; ++elementNum) { + const NKikimrMiniKQL::TType& protoMember = protoTupleType.GetElement(elementNum); + ConvertMiniKQLValueToYdbValue(protoMember, inputValue.GetTuple(elementNum), *output.mutable_items()->Add()); + } + break; + } + case NKikimrMiniKQL::ETypeKind::Variant: { + const NKikimrMiniKQL::TVariantType& protoVariantType = inputType.GetVariant(); + const ui32 variantIndex = inputValue.GetVariantIndex(); + output.set_variant_index(variantIndex); + switch (protoVariantType.type_case()) { + case NKikimrMiniKQL::TVariantType::kTupleItems: { + if (variantIndex >= protoVariantType.GetTupleItems().ElementSize()) { + throw yexception() << "Variant index out of type range"; + } + const NKikimrMiniKQL::TType& nextType = protoVariantType.GetTupleItems().GetElement(variantIndex); + ConvertMiniKQLValueToYdbValue(nextType, inputValue.GetOptional(), *output.mutable_nested_value()); + break; + } + case NKikimrMiniKQL::TVariantType::kStructItems: { + if (variantIndex >= protoVariantType.GetStructItems().MemberSize()) { + throw yexception() << "Variant index out of type range"; + } + const NKikimrMiniKQL::TType& nextType = protoVariantType.GetStructItems().GetMember(variantIndex).GetType(); + ConvertMiniKQLValueToYdbValue(nextType, inputValue.GetOptional(), *output.mutable_nested_value()); + break; + } + default: + ythrow yexception() << "Unknown variant type representation: " + << protoVariantType.DebugString(); + } + break; + } + default: { + ythrow yexception() << "Unknown protobuf type: " + << inputType.DebugString(); + } + } +} + void ConvertYdbValueToMiniKQLValue(const Ydb::Type& inputType, const Ydb::Value& inputValue, - NKikimrMiniKQL::TValue& output) { - + NKikimrMiniKQL::TValue& output) { + switch (inputType.type_case()) { case Ydb::Type::kVoidType: - break; + break; case Ydb::Type::kTypeId: ConvertData(inputType.type_id(), inputValue, output); - break; + break; case Ydb::Type::kDecimalType: - Y_ENSURE(inputValue.value_case() == Ydb::Value::kLow128); - output.SetLow128(inputValue.low_128()); - output.SetHi128(inputValue.high_128()); + Y_ENSURE(inputValue.value_case() == Ydb::Value::kLow128); + output.SetLow128(inputValue.low_128()); + output.SetHi128(inputValue.high_128()); if (NYql::NDecimal::IsError(NYql::NDecimal::FromProto(output))) { throw yexception() << "Invalid decimal value"; } @@ -587,201 +587,201 @@ void ConvertYdbValueToMiniKQLValue(const Ydb::Type& inputType, case Ydb::Type::kOptionalType: { switch (inputValue.value_case()) { case Ydb::Value::kNullFlagValue: - break; + break; case Ydb::Value::kNestedValue: - ConvertYdbValueToMiniKQLValue(inputType.optional_type().item(), inputValue.nested_value(), *output.MutableOptional()); - break; - default: - ConvertYdbValueToMiniKQLValue(inputType.optional_type().item(), inputValue, *output.MutableOptional()); - } - break; - } + ConvertYdbValueToMiniKQLValue(inputType.optional_type().item(), inputValue.nested_value(), *output.MutableOptional()); + break; + default: + ConvertYdbValueToMiniKQLValue(inputType.optional_type().item(), inputValue, *output.MutableOptional()); + } + break; + } case Ydb::Type::kListType: { const Ydb::ListType& protoListType = inputType.list_type(); const Ydb::Type& protoItemType = protoListType.item(); - for (const auto& x : inputValue.items()) { - ConvertYdbValueToMiniKQLValue(protoItemType, x, *output.MutableList()->Add()); - } - break; - } + for (const auto& x : inputValue.items()) { + ConvertYdbValueToMiniKQLValue(protoItemType, x, *output.MutableList()->Add()); + } + break; + } case Ydb::Type::kStructType: { const Ydb::StructType& protoStructType = inputType.struct_type(); - ui32 membersCount = static_cast<ui32>(protoStructType.membersSize()); - if (membersCount != inputValue.itemsSize()) { + ui32 membersCount = static_cast<ui32>(protoStructType.membersSize()); + if (membersCount != inputValue.itemsSize()) { throw yexception() << "Number of struct fields and their types mismatched"; - } - for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { + } + for (ui32 memberNum = 0; memberNum < membersCount; ++memberNum) { const Ydb::StructMember& protoMember = protoStructType.members(memberNum); - ConvertYdbValueToMiniKQLValue(protoMember.type(), inputValue.items(memberNum), *output.MutableStruct()->Add()); - } - break; - } + ConvertYdbValueToMiniKQLValue(protoMember.type(), inputValue.items(memberNum), *output.MutableStruct()->Add()); + } + break; + } case Ydb::Type::kDictType: { const Ydb::DictType& protoDictType = inputType.dict_type(); - for (const auto& x : inputValue.pairs()) { - auto pair = output.MutableDict()->Add(); - ConvertYdbValueToMiniKQLValue(protoDictType.key(), x.key(), *pair->MutableKey()); - ConvertYdbValueToMiniKQLValue(protoDictType.payload(), x.payload(), *pair->MutablePayload()); - } - break; - } + for (const auto& x : inputValue.pairs()) { + auto pair = output.MutableDict()->Add(); + ConvertYdbValueToMiniKQLValue(protoDictType.key(), x.key(), *pair->MutableKey()); + ConvertYdbValueToMiniKQLValue(protoDictType.payload(), x.payload(), *pair->MutablePayload()); + } + break; + } case Ydb::Type::kTupleType: { const Ydb::TupleType& protoTupleType = inputType.tuple_type(); - ui32 elementsCount = static_cast<ui32>(protoTupleType.elementsSize()); - if (elementsCount != inputValue.itemsSize()) { + ui32 elementsCount = static_cast<ui32>(protoTupleType.elementsSize()); + if (elementsCount != inputValue.itemsSize()) { throw yexception() << "Number of tuple elements and their types mismatched"; - } - for (ui32 elementNum = 0; elementNum < elementsCount; ++elementNum) { + } + for (ui32 elementNum = 0; elementNum < elementsCount; ++elementNum) { const Ydb::Type& elementType = protoTupleType.elements(elementNum); - ConvertYdbValueToMiniKQLValue(elementType, inputValue.items(elementNum), *output.MutableTuple()->Add()); - } - break; - } - case Ydb::Type::kVariantType: { - const Ydb::VariantType& protoVariantType = inputType.variant_type(); - const ui32 variantIndex = inputValue.variant_index(); - output.SetVariantIndex(variantIndex); - switch (protoVariantType.type_case()) { - case Ydb::VariantType::kTupleItems: { - Y_ENSURE(protoVariantType.tuple_items().elements_size() >= 0); - if (variantIndex >= (ui32)protoVariantType.tuple_items().elements_size()) { + ConvertYdbValueToMiniKQLValue(elementType, inputValue.items(elementNum), *output.MutableTuple()->Add()); + } + break; + } + case Ydb::Type::kVariantType: { + const Ydb::VariantType& protoVariantType = inputType.variant_type(); + const ui32 variantIndex = inputValue.variant_index(); + output.SetVariantIndex(variantIndex); + switch (protoVariantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + Y_ENSURE(protoVariantType.tuple_items().elements_size() >= 0); + if (variantIndex >= (ui32)protoVariantType.tuple_items().elements_size()) { throw yexception() << "Variant index out of type range"; - } - const Ydb::Type& nextType = protoVariantType.tuple_items().elements(variantIndex); - ConvertYdbValueToMiniKQLValue(nextType, inputValue.nested_value(), *output.MutableOptional()); - break; - } - case Ydb::VariantType::kStructItems: { - Y_ENSURE(protoVariantType.struct_items().members_size() >= 0); - if (variantIndex >= (ui32)protoVariantType.struct_items().members_size()) { + } + const Ydb::Type& nextType = protoVariantType.tuple_items().elements(variantIndex); + ConvertYdbValueToMiniKQLValue(nextType, inputValue.nested_value(), *output.MutableOptional()); + break; + } + case Ydb::VariantType::kStructItems: { + Y_ENSURE(protoVariantType.struct_items().members_size() >= 0); + if (variantIndex >= (ui32)protoVariantType.struct_items().members_size()) { throw yexception() << "Variant index out of type range"; - } - const Ydb::Type& nextType = protoVariantType.struct_items().members(variantIndex).type(); - ConvertYdbValueToMiniKQLValue(nextType, inputValue.nested_value(), *output.MutableOptional()); - break; - } - default: + } + const Ydb::Type& nextType = protoVariantType.struct_items().members(variantIndex).type(); + ConvertYdbValueToMiniKQLValue(nextType, inputValue.nested_value(), *output.MutableOptional()); + break; + } + default: throw yexception() << "Unknown variant type representation: " - << protoVariantType.DebugString(); - } - break; - } - - default: { + << protoVariantType.DebugString(); + } + break; + } + + default: { throw yexception() << "Unknown protobuf type: " - << inputType.DebugString(); - } - } -} - -void ConvertYdbParamsToMiniKQLParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, - NKikimrMiniKQL::TParams& output) { - output.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Struct); - auto type = output.MutableType()->MutableStruct(); - auto value = output.MutableValue(); - for (const auto& p : input) { - auto typeMember = type->AddMember(); - auto valueItem = value->AddStruct(); - typeMember->SetName(p.first); - ConvertYdbTypeToMiniKQLType(p.second.type(), *typeMember->MutableType()); - ConvertYdbValueToMiniKQLValue(p.second.type(), p.second.value(), *valueItem); - } -} - -void ConvertAclToYdb(const TString& owner, const TString& acl, bool isContainer, - google::protobuf::RepeatedPtrField<Ydb::Scheme::Permissions>* permissions) { - const auto& securityObject = TSecurityObject(owner, acl, isContainer); - for (const auto& ace : securityObject.GetACL().GetACE()) { - auto entry = permissions->Add(); - entry->set_subject(ace.GetSID()); - auto str = ConvertACLMaskToYdbPermissionNames(ace.GetAccessRight()); - for (auto n : str) { - entry->add_permission_names(n); - } - } - -} - -using namespace NACLib; + << inputType.DebugString(); + } + } +} + +void ConvertYdbParamsToMiniKQLParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, + NKikimrMiniKQL::TParams& output) { + output.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Struct); + auto type = output.MutableType()->MutableStruct(); + auto value = output.MutableValue(); + for (const auto& p : input) { + auto typeMember = type->AddMember(); + auto valueItem = value->AddStruct(); + typeMember->SetName(p.first); + ConvertYdbTypeToMiniKQLType(p.second.type(), *typeMember->MutableType()); + ConvertYdbValueToMiniKQLValue(p.second.type(), p.second.value(), *valueItem); + } +} + +void ConvertAclToYdb(const TString& owner, const TString& acl, bool isContainer, + google::protobuf::RepeatedPtrField<Ydb::Scheme::Permissions>* permissions) { + const auto& securityObject = TSecurityObject(owner, acl, isContainer); + for (const auto& ace : securityObject.GetACL().GetACE()) { + auto entry = permissions->Add(); + entry->set_subject(ace.GetSID()); + auto str = ConvertACLMaskToYdbPermissionNames(ace.GetAccessRight()); + for (auto n : str) { + entry->add_permission_names(n); + } + } + +} + +using namespace NACLib; const THashMap<TString, TACLAttrs> AccessMap_ = { { "ydb.database.connect", TACLAttrs(EAccessRights::ConnectDatabase, EInheritanceType::InheritNone) }, { "ydb.tables.modify", TACLAttrs(EAccessRights(UpdateRow | EraseRow)) }, { "ydb.tables.read", TACLAttrs(EAccessRights::SelectRow | EAccessRights::ReadAttributes) }, - { "ydb.generic.read", EAccessRights::GenericRead }, - { "ydb.generic.write", EAccessRights::GenericWrite }, + { "ydb.generic.read", EAccessRights::GenericRead }, + { "ydb.generic.write", EAccessRights::GenericWrite }, { "ydb.generic.use", EAccessRights::GenericUse }, { "ydb.generic.manage", EAccessRights::GenericManage }, - { "ydb.generic.full", EAccessRights::GenericFull }, + { "ydb.generic.full", EAccessRights::GenericFull }, { "ydb.database.create", EAccessRights::CreateDatabase }, { "ydb.database.drop", EAccessRights::DropDatabase }, { "ydb.access.grant", EAccessRights::GrantAccessRights }, - { "ydb.deprecated.select_row", EAccessRights::SelectRow }, - { "ydb.deprecated.update_row", EAccessRights::UpdateRow }, - { "ydb.deprecated.erase_row", EAccessRights::EraseRow }, - { "ydb.deprecated.read_attributes", EAccessRights::ReadAttributes }, - { "ydb.deprecated.write_attributes", EAccessRights::WriteAttributes }, - { "ydb.deprecated.create_directory", EAccessRights::CreateDirectory }, - { "ydb.deprecated.create_table", EAccessRights::CreateTable }, - { "ydb.deprecated.create_queue", EAccessRights::CreateQueue }, - { "ydb.deprecated.remove_schema", EAccessRights::RemoveSchema }, - { "ydb.deprecated.describe_schema", EAccessRights::DescribeSchema }, - { "ydb.deprecated.alter_schema", EAccessRights::AlterSchema } - -}; - -static ui32 BitCount(ui32 in) { - ui32 res = 0; - while (in) { - res++; - in &= in - 1; - } - return res; -} - -static TVector<std::pair<ui32, TString>> CalcMaskByPower() { - TVector<std::pair<ui32, TString>> result; - for (const auto& it : AccessMap_) { + { "ydb.deprecated.select_row", EAccessRights::SelectRow }, + { "ydb.deprecated.update_row", EAccessRights::UpdateRow }, + { "ydb.deprecated.erase_row", EAccessRights::EraseRow }, + { "ydb.deprecated.read_attributes", EAccessRights::ReadAttributes }, + { "ydb.deprecated.write_attributes", EAccessRights::WriteAttributes }, + { "ydb.deprecated.create_directory", EAccessRights::CreateDirectory }, + { "ydb.deprecated.create_table", EAccessRights::CreateTable }, + { "ydb.deprecated.create_queue", EAccessRights::CreateQueue }, + { "ydb.deprecated.remove_schema", EAccessRights::RemoveSchema }, + { "ydb.deprecated.describe_schema", EAccessRights::DescribeSchema }, + { "ydb.deprecated.alter_schema", EAccessRights::AlterSchema } + +}; + +static ui32 BitCount(ui32 in) { + ui32 res = 0; + while (in) { + res++; + in &= in - 1; + } + return res; +} + +static TVector<std::pair<ui32, TString>> CalcMaskByPower() { + TVector<std::pair<ui32, TString>> result; + for (const auto& it : AccessMap_) { result.push_back({it.second.AccessMask, it.first}); - } - - //Sort this vector by number of set bits in mask - //max is first - auto comp = [](const std::pair<ui32, TString>& a, const std::pair<ui32, TString>& b) -> bool { - return BitCount(a.first) > BitCount(b.first); - }; - std::sort (result.begin(), result.end(), comp); - return result; -} - + } + + //Sort this vector by number of set bits in mask + //max is first + auto comp = [](const std::pair<ui32, TString>& a, const std::pair<ui32, TString>& b) -> bool { + return BitCount(a.first) > BitCount(b.first); + }; + std::sort (result.begin(), result.end(), comp); + return result; +} + TACLAttrs ConvertYdbPermissionNameToACLAttrs(const TString& name) { - auto it = AccessMap_.find(name); - if (it == AccessMap_.end()) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Unknown permission name: " << name; - } - return it->second; -} - -TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32 mask) { - static const TVector<std::pair<ui32, TString>> maskByPower = CalcMaskByPower(); - TVector<TString> result; + auto it = AccessMap_.find(name); + if (it == AccessMap_.end()) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Unknown permission name: " << name; + } + return it->second; +} + +TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32 mask) { + static const TVector<std::pair<ui32, TString>> maskByPower = CalcMaskByPower(); + TVector<TString> result; for (const auto& pair : maskByPower) { - if ((mask & pair.first ^ pair.first) == 0) { - result.push_back(pair.second); - mask &= ~pair.first; - } - - if (!mask) { - break; - } - } - return result; -} - + if ((mask & pair.first ^ pair.first) == 0) { + result.push_back(pair.second); + mask &= ~pair.first; + } + + if (!mask) { + break; + } + } + return result; +} + void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme::Entry* to, bool processAcl) { - to->set_name(from.GetName()); - to->set_owner(from.GetOwner()); + to->set_name(from.GetName()); + to->set_owner(from.GetOwner()); switch (from.GetPathType()) { case NKikimrSchemeOp::EPathTypeExtSubDomain: @@ -791,13 +791,13 @@ void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme:: to->set_type(static_cast<Ydb::Scheme::Entry::Type>(from.GetPathType())); } - if (processAcl) { + if (processAcl) { const bool isDir = from.GetPathType() == NKikimrSchemeOp::EPathTypeDir; ConvertAclToYdb(from.GetOwner(), from.GetEffectiveACL(), isDir, to->mutable_effective_permissions()); ConvertAclToYdb(from.GetOwner(), from.GetACL(), isDir, to->mutable_permissions()); } } - + void ConvertDirectoryEntry(const NKikimrSchemeOp::TPathDescription& from, Ydb::Scheme::Entry* to, bool processAcl) { ConvertDirectoryEntry(from.GetSelf(), to, processAcl); @@ -814,9 +814,9 @@ void ConvertDirectoryEntry(const NKikimrSchemeOp::TPathDescription& from, Ydb::S break; default: break; - } -} - + } +} + void ConvertYdbResultToKqpResult(const Ydb::ResultSet& input, NKikimrMiniKQL::TResult& output) { auto& outputType = *output.MutableType(); outputType.SetKind(NKikimrMiniKQL::ETypeKind::Struct); @@ -863,4 +863,4 @@ TACLAttrs::TACLAttrs(ui32 access) , InheritanceType(EInheritanceType::InheritObject | EInheritanceType::InheritContainer) {} -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/ydb_convert/ydb_convert.h b/ydb/core/ydb_convert/ydb_convert.h index 9689291a86..2cf74135af 100644 --- a/ydb/core/ydb_convert/ydb_convert.h +++ b/ydb/core/ydb_convert/ydb_convert.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> #include <ydb/public/api/protos/ydb_scheme.pb.h> - -namespace NKikimr { - + +namespace NKikimr { + void ConvertMiniKQLTypeToYdbType(const NKikimrMiniKQL::TType& input, Ydb::Type& output); -void ConvertMiniKQLValueToYdbValue(const NKikimrMiniKQL::TType& inputType, - const NKikimrMiniKQL::TValue& inputValue, - Ydb::Value& output); +void ConvertMiniKQLValueToYdbValue(const NKikimrMiniKQL::TType& inputType, + const NKikimrMiniKQL::TValue& inputValue, + Ydb::Value& output); void ConvertYdbTypeToMiniKQLType(const Ydb::Type& input, NKikimrMiniKQL::TType& output); void ConvertYdbValueToMiniKQLValue(const Ydb::Type& inputType, - const Ydb::Value& inputValue, - NKikimrMiniKQL::TValue& output); - + const Ydb::Value& inputValue, + NKikimrMiniKQL::TValue& output); + void ConvertYdbResultToKqpResult(const Ydb::ResultSet& input, NKikimrMiniKQL::TResult& output); -void ConvertYdbParamsToMiniKQLParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, - NKikimrMiniKQL::TParams& output); - -void ConvertAclToYdb(const TString& owner, const TString& acl, bool isContainer, - google::protobuf::RepeatedPtrField<Ydb::Scheme::Permissions> *permissions); +void ConvertYdbParamsToMiniKQLParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, + NKikimrMiniKQL::TParams& output); + +void ConvertAclToYdb(const TString& owner, const TString& acl, bool isContainer, + google::protobuf::RepeatedPtrField<Ydb::Scheme::Permissions> *permissions); struct TACLAttrs { ui32 AccessMask; @@ -35,9 +35,9 @@ struct TACLAttrs { TACLAttrs ConvertYdbPermissionNameToACLAttrs(const TString& name); -TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32); - +TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32); + void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme::Entry* to, bool processAcl); void ConvertDirectoryEntry(const NKikimrSchemeOp::TPathDescription& from, Ydb::Scheme::Entry* to, bool processAcl); -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/ydb_convert/ydb_convert_ut.cpp b/ydb/core/ydb_convert/ydb_convert_ut.cpp index d64201ff62..cb80206926 100644 --- a/ydb/core/ydb_convert/ydb_convert_ut.cpp +++ b/ydb/core/ydb_convert/ydb_convert_ut.cpp @@ -1,1022 +1,1022 @@ -#include "ydb_convert.h" - +#include "ydb_convert.h" + #include <google/protobuf/text_format.h> - + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - -static void TestConvertTypeToYdb(const TString& input, const TString& expected) { - NKikimrMiniKQL::TType typeFrom; - google::protobuf::TextFormat::ParseFromString(input, &typeFrom); + +namespace NKikimr { + +static void TestConvertTypeToYdb(const TString& input, const TString& expected) { + NKikimrMiniKQL::TType typeFrom; + google::protobuf::TextFormat::ParseFromString(input, &typeFrom); Ydb::Type typeTo; - ConvertMiniKQLTypeToYdbType(typeFrom, typeTo); - TString result; - google::protobuf::TextFormat::PrintToString(typeTo, &result); - UNIT_ASSERT_NO_DIFF(result, expected); -} - -static void TestConvertTypeFromYdb(const TString& input, const TString& expected) { + ConvertMiniKQLTypeToYdbType(typeFrom, typeTo); + TString result; + google::protobuf::TextFormat::PrintToString(typeTo, &result); + UNIT_ASSERT_NO_DIFF(result, expected); +} + +static void TestConvertTypeFromYdb(const TString& input, const TString& expected) { Ydb::Type typeFrom; - google::protobuf::TextFormat::ParseFromString(input, &typeFrom); - NKikimrMiniKQL::TType typeTo; - ConvertYdbTypeToMiniKQLType(typeFrom, typeTo); - TString result; - google::protobuf::TextFormat::PrintToString(typeTo, &result); - UNIT_ASSERT_NO_DIFF(result, expected); -} - -static void TestConvertValueToYdb(const TString& inputType, const TString& input, const TString& expected) { - NKikimrMiniKQL::TValue valueFrom; - google::protobuf::TextFormat::ParseFromString(input, &valueFrom); - NKikimrMiniKQL::TType typeFrom; - google::protobuf::TextFormat::ParseFromString(inputType, &typeFrom); + google::protobuf::TextFormat::ParseFromString(input, &typeFrom); + NKikimrMiniKQL::TType typeTo; + ConvertYdbTypeToMiniKQLType(typeFrom, typeTo); + TString result; + google::protobuf::TextFormat::PrintToString(typeTo, &result); + UNIT_ASSERT_NO_DIFF(result, expected); +} + +static void TestConvertValueToYdb(const TString& inputType, const TString& input, const TString& expected) { + NKikimrMiniKQL::TValue valueFrom; + google::protobuf::TextFormat::ParseFromString(input, &valueFrom); + NKikimrMiniKQL::TType typeFrom; + google::protobuf::TextFormat::ParseFromString(inputType, &typeFrom); Ydb::Value valueTo; - ConvertMiniKQLValueToYdbValue(typeFrom, valueFrom, valueTo); - TString result; - google::protobuf::TextFormat::PrintToString(valueTo, &result); - UNIT_ASSERT_NO_DIFF(result, expected); -} - -static void TestConvertValueFromYdb(const TString& inputType, const TString& input, const TString& expected) { + ConvertMiniKQLValueToYdbValue(typeFrom, valueFrom, valueTo); + TString result; + google::protobuf::TextFormat::PrintToString(valueTo, &result); + UNIT_ASSERT_NO_DIFF(result, expected); +} + +static void TestConvertValueFromYdb(const TString& inputType, const TString& input, const TString& expected) { Ydb::Value valueFrom; - google::protobuf::TextFormat::ParseFromString(input, &valueFrom); + google::protobuf::TextFormat::ParseFromString(input, &valueFrom); Ydb::Type typeFrom; - google::protobuf::TextFormat::ParseFromString(inputType, &typeFrom); - NKikimrMiniKQL::TValue valueTo; - ConvertYdbValueToMiniKQLValue(typeFrom, valueFrom, valueTo); - TString result; - google::protobuf::TextFormat::PrintToString(valueTo, &result); - UNIT_ASSERT_NO_DIFF(result, expected); -} - + google::protobuf::TextFormat::ParseFromString(inputType, &typeFrom); + NKikimrMiniKQL::TValue valueTo; + ConvertYdbValueToMiniKQLValue(typeFrom, valueFrom, valueTo); + TString result; + google::protobuf::TextFormat::PrintToString(valueTo, &result); + UNIT_ASSERT_NO_DIFF(result, expected); +} + Y_UNIT_TEST_SUITE(ConvertMiniKQLTypeToYdbTypeTest) { Y_UNIT_TEST(SimpleType) { - const TString input = - "Kind: Data\n" - "Data {\n" - " Scheme: 1\n" - "}\n"; - const TString expected = - "type_id: INT32\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(TTzDate) { - const TString input = - "Kind: Data\n" - "Data {\n" - " Scheme: 52\n" - "}\n"; - const TString expected = - "type_id: TZ_DATE\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(TTzDateTime) { - const TString input = - "Kind: Data\n" - "Data {\n" - " Scheme: 53\n" - "}\n"; - const TString expected = - "type_id: TZ_DATETIME\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(TTzTimeStamp) { - const TString input = - "Kind: Data\n" - "Data {\n" - " Scheme: 54\n" - "}\n"; - const TString expected = - "type_id: TZ_TIMESTAMP\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(DecimalType) { - const TString input = - "Kind: Data\n" - "Data {\n" + const TString input = + "Kind: Data\n" + "Data {\n" + " Scheme: 1\n" + "}\n"; + const TString expected = + "type_id: INT32\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(TTzDate) { + const TString input = + "Kind: Data\n" + "Data {\n" + " Scheme: 52\n" + "}\n"; + const TString expected = + "type_id: TZ_DATE\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(TTzDateTime) { + const TString input = + "Kind: Data\n" + "Data {\n" + " Scheme: 53\n" + "}\n"; + const TString expected = + "type_id: TZ_DATETIME\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(TTzTimeStamp) { + const TString input = + "Kind: Data\n" + "Data {\n" + " Scheme: 54\n" + "}\n"; + const TString expected = + "type_id: TZ_TIMESTAMP\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(DecimalType) { + const TString input = + "Kind: Data\n" + "Data {\n" " Scheme: 4865\n" - " DecimalParams {\n" - " Precision: 21\n" - " Scale: 8\n" - " }\n" - "}\n"; - const TString expected = - "decimal_type {\n" - " precision: 21\n" - " scale: 8\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(UuidType) { - const TString input = - "Kind: Data\n" - "Data {\n" - " Scheme: 4611\n" - "}\n"; - const TString expected = - "type_id: UUID\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - - Y_UNIT_TEST(VariantTuple) { - const TString a = R"___(Kind: Variant -Variant { - TupleItems { - Element { - Kind: Data - Data { - Scheme: 4 - } - } - Element { - Kind: Data - Data { - Scheme: 2 - } - } - } -} -)___"; - - const TString b = R"___(variant_type { - tuple_items { - elements { - type_id: UINT64 - } - elements { - type_id: UINT32 - } - } -} -)___"; - TestConvertTypeToYdb(a, b); - TestConvertTypeFromYdb(b, a); - } - - Y_UNIT_TEST(VariantStruct) { - const TString a = R"___(Kind: Variant -Variant { - StructItems { - Member { - Name: "a" - Type { - Kind: Data - Data { - Scheme: 4 - } - } - } - Member { - Name: "b" - Type { - Kind: Data - Data { - Scheme: 2 - } - } - } - } -} -)___"; - - const TString b = R"___(variant_type { - struct_items { - members { - name: "a" - type { - type_id: UINT64 - } - } - members { - name: "b" - type { - type_id: UINT32 - } - } - } -} -)___"; - TestConvertTypeToYdb(a, b); - TestConvertTypeFromYdb(b, a); - } - - + " DecimalParams {\n" + " Precision: 21\n" + " Scale: 8\n" + " }\n" + "}\n"; + const TString expected = + "decimal_type {\n" + " precision: 21\n" + " scale: 8\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(UuidType) { + const TString input = + "Kind: Data\n" + "Data {\n" + " Scheme: 4611\n" + "}\n"; + const TString expected = + "type_id: UUID\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + + Y_UNIT_TEST(VariantTuple) { + const TString a = R"___(Kind: Variant +Variant { + TupleItems { + Element { + Kind: Data + Data { + Scheme: 4 + } + } + Element { + Kind: Data + Data { + Scheme: 2 + } + } + } +} +)___"; + + const TString b = R"___(variant_type { + tuple_items { + elements { + type_id: UINT64 + } + elements { + type_id: UINT32 + } + } +} +)___"; + TestConvertTypeToYdb(a, b); + TestConvertTypeFromYdb(b, a); + } + + Y_UNIT_TEST(VariantStruct) { + const TString a = R"___(Kind: Variant +Variant { + StructItems { + Member { + Name: "a" + Type { + Kind: Data + Data { + Scheme: 4 + } + } + } + Member { + Name: "b" + Type { + Kind: Data + Data { + Scheme: 2 + } + } + } + } +} +)___"; + + const TString b = R"___(variant_type { + struct_items { + members { + name: "a" + type { + type_id: UINT64 + } + } + members { + name: "b" + type { + type_id: UINT32 + } + } + } +} +)___"; + TestConvertTypeToYdb(a, b); + TestConvertTypeFromYdb(b, a); + } + + Y_UNIT_TEST(Void) { - const TString input = "Kind: Void\n"; - const TString expected = "void_type: NULL_VALUE\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - + const TString input = "Kind: Void\n"; + const TString expected = "void_type: NULL_VALUE\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + Y_UNIT_TEST(Optional) { - const TString input = - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - "}\n"; - const TString expected = - "optional_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - + const TString input = + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + "}\n"; + const TString expected = + "optional_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + Y_UNIT_TEST(List) { - const TString input = - "Kind: List\n" - "List {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - "}\n"; - const TString expected = "list_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - + const TString input = + "Kind: List\n" + "List {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + "}\n"; + const TString expected = "list_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + Y_UNIT_TEST(Tuple) { - const TString input = - "Kind: Tuple\n" - "Tuple {\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString expected = - "tuple_type {\n" - " elements {\n" - " type_id: INT32\n" - " }\n" - " elements {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - + const TString input = + "Kind: Tuple\n" + "Tuple {\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString expected = + "tuple_type {\n" + " elements {\n" + " type_id: INT32\n" + " }\n" + " elements {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + Y_UNIT_TEST(Struct) { - const TString input = - "Kind: Struct\n" - "Struct {\n" - " Member {\n" - " Name: \"x\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " }\n" - " Member {\n" - " Name: \"y\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString expected = - "struct_type {\n" - " members {\n" - " name: \"x\"\n" - " type {\n" - " type_id: INT32\n" - " }\n" - " }\n" - " members {\n" - " name: \"y\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } - + const TString input = + "Kind: Struct\n" + "Struct {\n" + " Member {\n" + " Name: \"x\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " }\n" + " Member {\n" + " Name: \"y\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString expected = + "struct_type {\n" + " members {\n" + " name: \"x\"\n" + " type {\n" + " type_id: INT32\n" + " }\n" + " }\n" + " members {\n" + " name: \"y\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } + Y_UNIT_TEST(Dict) { - const TString input = - "Kind: Dict\n" - "Dict {\n" - " Key {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Payload {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString expected = - "dict_type {\n" - " key {\n" - " type_id: INT32\n" - " }\n" - " payload {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - TestConvertTypeToYdb(input, expected); - TestConvertTypeFromYdb(expected, input); - } -} - + const TString input = + "Kind: Dict\n" + "Dict {\n" + " Key {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Payload {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString expected = + "dict_type {\n" + " key {\n" + " type_id: INT32\n" + " }\n" + " payload {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + TestConvertTypeToYdb(input, expected); + TestConvertTypeFromYdb(expected, input); + } +} + Y_UNIT_TEST_SUITE(ConvertMiniKQLValueToYdbValueTest) { Y_UNIT_TEST(Void) { - TestConvertValueToYdb("Kind: Void\n", "", ""); - } - + TestConvertValueToYdb("Kind: Void\n", "", ""); + } + Y_UNIT_TEST(SimpleBool) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 6\n" - "}\n"; - TestConvertValueToYdb(inputType, "Bool: true\n", "bool_value: true\n"); - } - + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 6\n" + "}\n"; + TestConvertValueToYdb(inputType, "Bool: true\n", "bool_value: true\n"); + } + Y_UNIT_TEST(SimpleInt32) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 1\n" - "}\n"; - TestConvertValueToYdb(inputType, "Int32: -42\n", "int32_value: -42\n"); - } - + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 1\n" + "}\n"; + TestConvertValueToYdb(inputType, "Int32: -42\n", "int32_value: -42\n"); + } + Y_UNIT_TEST(SimpleInt64) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 3\n" - "}\n"; - TestConvertValueToYdb(inputType, "Int64: -42000000000\n", "int64_value: -42000000000\n"); - } - - Y_UNIT_TEST(SimpleTzDate) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 52\n" - "}\n"; - TestConvertValueToYdb(inputType, "Text: \"2020-09-22,Europe/Moscow\"\n", "text_value: \"2020-09-22,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleTzDateTime) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 53\n" - "}\n"; - - TestConvertValueToYdb(inputType, "Text: \"2018-02-03T15:00:00,Europe/Moscow\"\n", "text_value: \"2018-02-03T15:00:00,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleTzTimeStamp) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 54\n" - "}\n"; - - TestConvertValueToYdb(inputType, "Text: \"2018-02-03T15:00:00,Europe/Moscow\"\n", "text_value: \"2018-02-03T15:00:00,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleDecimal) { - const TString inputType = - "Kind: Data\n" - "Data {\n" + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 3\n" + "}\n"; + TestConvertValueToYdb(inputType, "Int64: -42000000000\n", "int64_value: -42000000000\n"); + } + + Y_UNIT_TEST(SimpleTzDate) { + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 52\n" + "}\n"; + TestConvertValueToYdb(inputType, "Text: \"2020-09-22,Europe/Moscow\"\n", "text_value: \"2020-09-22,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleTzDateTime) { + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 53\n" + "}\n"; + + TestConvertValueToYdb(inputType, "Text: \"2018-02-03T15:00:00,Europe/Moscow\"\n", "text_value: \"2018-02-03T15:00:00,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleTzTimeStamp) { + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 54\n" + "}\n"; + + TestConvertValueToYdb(inputType, "Text: \"2018-02-03T15:00:00,Europe/Moscow\"\n", "text_value: \"2018-02-03T15:00:00,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleDecimal) { + const TString inputType = + "Kind: Data\n" + "Data {\n" " Scheme: 4865\n" - " DecimalParams {\n" - " Precision: 21\n" - " Scale: 8\n" - " }\n" - "}\n"; - TestConvertValueToYdb(inputType, "Low128: 123\nHi128: 456\n", "low_128: 123\nhigh_128: 456\n"); - } - - Y_UNIT_TEST(SimpleUuid) { - const TString inputType = - "Kind: Data\n" - "Data {\n" - " Scheme: 4611\n" - "}\n"; - const TString uuidStr = R"__(Low128: 1 -Hi128: 2 -)__"; - TestConvertValueToYdb(inputType, uuidStr, "low_128: 1\nhigh_128: 2\n"); - } - + " DecimalParams {\n" + " Precision: 21\n" + " Scale: 8\n" + " }\n" + "}\n"; + TestConvertValueToYdb(inputType, "Low128: 123\nHi128: 456\n", "low_128: 123\nhigh_128: 456\n"); + } + + Y_UNIT_TEST(SimpleUuid) { + const TString inputType = + "Kind: Data\n" + "Data {\n" + " Scheme: 4611\n" + "}\n"; + const TString uuidStr = R"__(Low128: 1 +Hi128: 2 +)__"; + TestConvertValueToYdb(inputType, uuidStr, "low_128: 1\nhigh_128: 2\n"); + } + Y_UNIT_TEST(OptionalString) { - const TString inputType = - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "Optional {\n" - " Bytes: \"abc\"\n" - "}\n"; - TestConvertValueToYdb(inputType, inputValue, "bytes_value: \"abc\"\n"); - } - + const TString inputType = + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "Optional {\n" + " Bytes: \"abc\"\n" + "}\n"; + TestConvertValueToYdb(inputType, inputValue, "bytes_value: \"abc\"\n"); + } + Y_UNIT_TEST(OptionalEmpty) { - const TString inputType = - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = ""; - TestConvertValueToYdb(inputType, inputValue, "null_flag_value: NULL_VALUE\n"); - } - + const TString inputType = + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = ""; + TestConvertValueToYdb(inputType, inputValue, "null_flag_value: NULL_VALUE\n"); + } + Y_UNIT_TEST(OptionalOptionalEmpty) { - const TString inputType = - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Optional\n" - " Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "Optional {\n" - "}\n"; - TestConvertValueToYdb(inputType, inputValue, - "nested_value {\n" - " null_flag_value: NULL_VALUE\n" - "}\n"); - } - + const TString inputType = + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Optional\n" + " Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "Optional {\n" + "}\n"; + TestConvertValueToYdb(inputType, inputValue, + "nested_value {\n" + " null_flag_value: NULL_VALUE\n" + "}\n"); + } + Y_UNIT_TEST(OptionalOptionalEmpty2) { - const TString inputType = - "Kind: Optional\n" - "Optional {\n" - " Item {\n" - " Kind: Optional\n" - " Optional {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = ""; - TestConvertValueToYdb(inputType, inputValue, - "null_flag_value: NULL_VALUE\n"); - } - + const TString inputType = + "Kind: Optional\n" + "Optional {\n" + " Item {\n" + " Kind: Optional\n" + " Optional {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = ""; + TestConvertValueToYdb(inputType, inputValue, + "null_flag_value: NULL_VALUE\n"); + } + Y_UNIT_TEST(List) { - const TString inputType = - "Kind: List\n" - "List {\n" - " Item {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "List {\n" - " Bytes: \"abc\"\n" - "}\n" - "List {\n" - " Bytes: \"zxc\"\n" - "}\n"; - TestConvertValueToYdb(inputType, inputValue, - "items {\n" - " bytes_value: \"abc\"\n" - "}\n" - "items {\n" - " bytes_value: \"zxc\"\n" - "}\n"); - } - + const TString inputType = + "Kind: List\n" + "List {\n" + " Item {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "List {\n" + " Bytes: \"abc\"\n" + "}\n" + "List {\n" + " Bytes: \"zxc\"\n" + "}\n"; + TestConvertValueToYdb(inputType, inputValue, + "items {\n" + " bytes_value: \"abc\"\n" + "}\n" + "items {\n" + " bytes_value: \"zxc\"\n" + "}\n"); + } + Y_UNIT_TEST(Struct) { - const TString inputType = - "Kind: Struct\n" - "Struct {\n" - " Member {\n" - " Name: \"x\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 32\n" - " }\n" - " }\n" - " }\n" - " Member {\n" - " Name: \"y\"\n" - " Type {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "Struct {\n" - " Double: 42.33\n" - "}\n" - "Struct {\n" - " Bytes: \"abc\"\n" - "}\n"; - TestConvertValueToYdb(inputType, inputValue, - "items {\n" - " double_value: 42.33\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"); - } - + const TString inputType = + "Kind: Struct\n" + "Struct {\n" + " Member {\n" + " Name: \"x\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 32\n" + " }\n" + " }\n" + " }\n" + " Member {\n" + " Name: \"y\"\n" + " Type {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "Struct {\n" + " Double: 42.33\n" + "}\n" + "Struct {\n" + " Bytes: \"abc\"\n" + "}\n"; + TestConvertValueToYdb(inputType, inputValue, + "items {\n" + " double_value: 42.33\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(Dict) { - const TString inputType = - "Kind: Dict\n" - "Dict {\n" - " Key {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Payload {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "Dict {\n" - " Key {\n" - " Int32: 42\n" - " }\n" - " Payload {\n" - " Bytes: \"abc\"\n" - " }\n" - "}\n"; - TestConvertValueToYdb(inputType, inputValue, - "pairs {\n" - " key {\n" - " int32_value: 42\n" - " }\n" - " payload {\n" - " bytes_value: \"abc\"\n" - " }\n" - "}\n"); - } - + const TString inputType = + "Kind: Dict\n" + "Dict {\n" + " Key {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Payload {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "Dict {\n" + " Key {\n" + " Int32: 42\n" + " }\n" + " Payload {\n" + " Bytes: \"abc\"\n" + " }\n" + "}\n"; + TestConvertValueToYdb(inputType, inputValue, + "pairs {\n" + " key {\n" + " int32_value: 42\n" + " }\n" + " payload {\n" + " bytes_value: \"abc\"\n" + " }\n" + "}\n"); + } + Y_UNIT_TEST(Tuple) { - const TString inputType = - "Kind: Tuple\n" - "Tuple {\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 1\n" - " }\n" - " }\n" - " Element {\n" - " Kind: Data\n" - " Data {\n" - " Scheme: 4097\n" - " }\n" - " }\n" - "}\n"; - - const TString inputValue = - "Tuple {\n" - " Int32: 42\n" - "}\n" - "Tuple {\n" - " Bytes: \"abc\"\n" - "}\n"; - - TestConvertValueToYdb(inputType, inputValue, - "items {\n" - " int32_value: 42\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"); - } - - Y_UNIT_TEST(Variant) { - const TString inputType = R"___(Kind: Variant -Variant { - TupleItems { - Element { - Kind: Data - Data { - Scheme: 4 - } - } - Element { - Kind: Data - Data { - Scheme: 2 - } - } - } -} -)___"; - - const TString inputValue = R"___(Optional { - Uint32: 66 -} -VariantIndex: 1 -)___"; - - const TString expected = R"___(nested_value { - uint32_value: 66 -} -variant_index: 1 -)___"; - - TestConvertValueToYdb(inputType, inputValue, expected); - } - -} - + const TString inputType = + "Kind: Tuple\n" + "Tuple {\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 1\n" + " }\n" + " }\n" + " Element {\n" + " Kind: Data\n" + " Data {\n" + " Scheme: 4097\n" + " }\n" + " }\n" + "}\n"; + + const TString inputValue = + "Tuple {\n" + " Int32: 42\n" + "}\n" + "Tuple {\n" + " Bytes: \"abc\"\n" + "}\n"; + + TestConvertValueToYdb(inputType, inputValue, + "items {\n" + " int32_value: 42\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"); + } + + Y_UNIT_TEST(Variant) { + const TString inputType = R"___(Kind: Variant +Variant { + TupleItems { + Element { + Kind: Data + Data { + Scheme: 4 + } + } + Element { + Kind: Data + Data { + Scheme: 2 + } + } + } +} +)___"; + + const TString inputValue = R"___(Optional { + Uint32: 66 +} +VariantIndex: 1 +)___"; + + const TString expected = R"___(nested_value { + uint32_value: 66 +} +variant_index: 1 +)___"; + + TestConvertValueToYdb(inputType, inputValue, expected); + } + +} + Y_UNIT_TEST_SUITE(ConvertYdbValueToMiniKQLValueTest) { Y_UNIT_TEST(Void) { - TestConvertValueFromYdb("void_type: NULL_VALUE\n", "", ""); - } - + TestConvertValueFromYdb("void_type: NULL_VALUE\n", "", ""); + } + Y_UNIT_TEST(SimpleBool) { - const TString inputType = - "type_id: BOOL\n"; - TestConvertValueFromYdb(inputType, "bool_value: true\n", "Bool: true\n"); - } - - Y_UNIT_TEST(SimpleBoolTypeMissmatch) { - const TString inputType = - "type_id: BOOL\n"; - UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "int32_value: -42\n", "Bool: true\n"), yexception); - } - + const TString inputType = + "type_id: BOOL\n"; + TestConvertValueFromYdb(inputType, "bool_value: true\n", "Bool: true\n"); + } + + Y_UNIT_TEST(SimpleBoolTypeMissmatch) { + const TString inputType = + "type_id: BOOL\n"; + UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "int32_value: -42\n", "Bool: true\n"), yexception); + } + Y_UNIT_TEST(SimpleInt32) { - const TString inputType = - "type_id: INT32\n"; - TestConvertValueFromYdb(inputType, "int32_value: -42\n", "Int32: -42\n"); - } - - Y_UNIT_TEST(SimpleTzDate) { - const TString inputType = - "type_id: TZ_DATE\n"; - TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22,Europe/Moscow\"", "Text: \"2020-09-22,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleTzDateTime) { - const TString inputType = - "type_id: TZ_DATETIME\n"; - TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22T15:00:00,Europe/Moscow\"", "Text: \"2020-09-22T15:00:00,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleTzTimeStamp) { - const TString inputType = - "type_id: TZ_TIMESTAMP\n"; - TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22T15:00:00,Europe/Moscow\"", "Text: \"2020-09-22T15:00:00,Europe/Moscow\"\n"); - } - - Y_UNIT_TEST(SimpleInt32TypeMissmatch) { - const TString inputType = - "type_id: INT32\n"; - const TString inputValue = "bytes_value: \"abc\"\n"; - UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, inputValue, "Int32: -42\n"), yexception); - } - - Y_UNIT_TEST(SimpleUuid) { - const TString inputType = - "type_id: UUID\n"; - const TString uuidStr = R"__(Low128: 1 -Hi128: 2 -)__"; - TestConvertValueFromYdb(inputType, "low_128: 1\nhigh_128: 2\n", uuidStr); - } - - Y_UNIT_TEST(SimpleUuidTypeMissmatch) { - const TString inputType = - "type_id: UUID\n"; - const TString uuidStr = R"__(Low128: 1 -Hi128: 2 -)__"; - UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "bytes_value: \"abc\"\n", uuidStr), yexception); - } - - Y_UNIT_TEST(SimpleDecimal) { - const TString inputType = - "decimal_type {\n" - " precision: 21\n" - " scale: 8\n" - "}\n"; - TestConvertValueFromYdb(inputType, "low_128: 123\nhigh_128: 456\n", "Low128: 123\nHi128: 456\n"); - } - - Y_UNIT_TEST(SimpleDecimalTypeMissmatch) { - const TString inputType = - "decimal_type {\n" - " precision: 21\n" - " scale: 8\n" - "}\n"; - UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "bytes_value: \"abc\"\n", "Low128: 123\nHi128: 456\n"), yexception); - } - + const TString inputType = + "type_id: INT32\n"; + TestConvertValueFromYdb(inputType, "int32_value: -42\n", "Int32: -42\n"); + } + + Y_UNIT_TEST(SimpleTzDate) { + const TString inputType = + "type_id: TZ_DATE\n"; + TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22,Europe/Moscow\"", "Text: \"2020-09-22,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleTzDateTime) { + const TString inputType = + "type_id: TZ_DATETIME\n"; + TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22T15:00:00,Europe/Moscow\"", "Text: \"2020-09-22T15:00:00,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleTzTimeStamp) { + const TString inputType = + "type_id: TZ_TIMESTAMP\n"; + TestConvertValueFromYdb(inputType, "text_value: \"2020-09-22T15:00:00,Europe/Moscow\"", "Text: \"2020-09-22T15:00:00,Europe/Moscow\"\n"); + } + + Y_UNIT_TEST(SimpleInt32TypeMissmatch) { + const TString inputType = + "type_id: INT32\n"; + const TString inputValue = "bytes_value: \"abc\"\n"; + UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, inputValue, "Int32: -42\n"), yexception); + } + + Y_UNIT_TEST(SimpleUuid) { + const TString inputType = + "type_id: UUID\n"; + const TString uuidStr = R"__(Low128: 1 +Hi128: 2 +)__"; + TestConvertValueFromYdb(inputType, "low_128: 1\nhigh_128: 2\n", uuidStr); + } + + Y_UNIT_TEST(SimpleUuidTypeMissmatch) { + const TString inputType = + "type_id: UUID\n"; + const TString uuidStr = R"__(Low128: 1 +Hi128: 2 +)__"; + UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "bytes_value: \"abc\"\n", uuidStr), yexception); + } + + Y_UNIT_TEST(SimpleDecimal) { + const TString inputType = + "decimal_type {\n" + " precision: 21\n" + " scale: 8\n" + "}\n"; + TestConvertValueFromYdb(inputType, "low_128: 123\nhigh_128: 456\n", "Low128: 123\nHi128: 456\n"); + } + + Y_UNIT_TEST(SimpleDecimalTypeMissmatch) { + const TString inputType = + "decimal_type {\n" + " precision: 21\n" + " scale: 8\n" + "}\n"; + UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, "bytes_value: \"abc\"\n", "Low128: 123\nHi128: 456\n"), yexception); + } + Y_UNIT_TEST(OptionalString) { - const TString inputType = - "optional_type {\n" - " item {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - const TString inputValue = "bytes_value: \"abc\"\n"; - TestConvertValueFromYdb(inputType, inputValue, - "Optional {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - + const TString inputType = + "optional_type {\n" + " item {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + const TString inputValue = "bytes_value: \"abc\"\n"; + TestConvertValueFromYdb(inputType, inputValue, + "Optional {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(OptionalEmpty) { - const TString inputType = - "optional_type {\n" - " item {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - const TString inputValue = "null_flag_value: NULL_VALUE\n"; - TestConvertValueFromYdb(inputType, inputValue, ""); - } - + const TString inputType = + "optional_type {\n" + " item {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + const TString inputValue = "null_flag_value: NULL_VALUE\n"; + TestConvertValueFromYdb(inputType, inputValue, ""); + } + Y_UNIT_TEST(OptionalOptionalEmpty) { - const TString inputType = - "optional_type {\n" - " item {\n" - " optional_type {\n" - " item {\n" - " type_id: STRING\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "nested_value {\n" - " null_flag_value: NULL_VALUE\n" - "}\n"; - TestConvertValueFromYdb(inputType, inputValue, - "Optional {\n" - "}\n"); - } - + const TString inputType = + "optional_type {\n" + " item {\n" + " optional_type {\n" + " item {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "nested_value {\n" + " null_flag_value: NULL_VALUE\n" + "}\n"; + TestConvertValueFromYdb(inputType, inputValue, + "Optional {\n" + "}\n"); + } + Y_UNIT_TEST(OptionalOptionalEmpty2) { - const TString inputType = - "optional_type {\n" - " item {\n" - " optional_type {\n" - " item {\n" - " type_id: STRING\n" - " }\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = "null_flag_value: NULL_VALUE\n"; - TestConvertValueFromYdb(inputType, inputValue, ""); - } - + const TString inputType = + "optional_type {\n" + " item {\n" + " optional_type {\n" + " item {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = "null_flag_value: NULL_VALUE\n"; + TestConvertValueFromYdb(inputType, inputValue, ""); + } + Y_UNIT_TEST(List) { - const TString inputType = - "list_type {\n" - " item {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - const TString inputValue = - "items {\n" - " bytes_value: \"abc\"\n" - "}\n" - "items {\n" - " bytes_value: \"zxc\"\n" - "}\n"; - TestConvertValueFromYdb(inputType, inputValue, - "List {\n" - " Bytes: \"abc\"\n" - "}\n" - "List {\n" - " Bytes: \"zxc\"\n" - "}\n"); - } - + const TString inputType = + "list_type {\n" + " item {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + const TString inputValue = + "items {\n" + " bytes_value: \"abc\"\n" + "}\n" + "items {\n" + " bytes_value: \"zxc\"\n" + "}\n"; + TestConvertValueFromYdb(inputType, inputValue, + "List {\n" + " Bytes: \"abc\"\n" + "}\n" + "List {\n" + " Bytes: \"zxc\"\n" + "}\n"); + } + Y_UNIT_TEST(Struct) { - const TString inputType = - "struct_type {\n" - " members {\n" - " name: \"x\"\n" - " type {\n" - " type_id: DOUBLE\n" - " }\n" - " }\n" - " members {\n" - " name: \"y\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" - "}\n"; - const TString inputValue = - "items {\n" - " double_value: 42.33\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"; - TestConvertValueFromYdb(inputType, inputValue, - "Struct {\n" - " Double: 42.33\n" - "}\n" - "Struct {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - + const TString inputType = + "struct_type {\n" + " members {\n" + " name: \"x\"\n" + " type {\n" + " type_id: DOUBLE\n" + " }\n" + " }\n" + " members {\n" + " name: \"y\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" + "}\n"; + const TString inputValue = + "items {\n" + " double_value: 42.33\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"; + TestConvertValueFromYdb(inputType, inputValue, + "Struct {\n" + " Double: 42.33\n" + "}\n" + "Struct {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + Y_UNIT_TEST(Dict) { - const TString inputType = - "dict_type {\n" - " key {\n" - " type_id: INT32\n" - " }\n" - " payload {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - const TString inputValue = - "pairs {\n" - " key {\n" - " int32_value: 42\n" - " }\n" - " payload {\n" - " bytes_value: \"abc\"\n" - " }\n" - "}\n"; - - TestConvertValueFromYdb(inputType, inputValue, - "Dict {\n" - " Key {\n" - " Int32: 42\n" - " }\n" - " Payload {\n" - " Bytes: \"abc\"\n" - " }\n" - "}\n"); - - } - + const TString inputType = + "dict_type {\n" + " key {\n" + " type_id: INT32\n" + " }\n" + " payload {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + const TString inputValue = + "pairs {\n" + " key {\n" + " int32_value: 42\n" + " }\n" + " payload {\n" + " bytes_value: \"abc\"\n" + " }\n" + "}\n"; + + TestConvertValueFromYdb(inputType, inputValue, + "Dict {\n" + " Key {\n" + " Int32: 42\n" + " }\n" + " Payload {\n" + " Bytes: \"abc\"\n" + " }\n" + "}\n"); + + } + Y_UNIT_TEST(Tuple) { - const TString inputType = - "tuple_type {\n" - " elements {\n" - " type_id: INT32\n" - " }\n" - " elements {\n" - " type_id: STRING\n" - " }\n" - "}\n"; - - const TString inputValue = - "items {\n" - " int32_value: 42\n" - "}\n" - "items {\n" - " bytes_value: \"abc\"\n" - "}\n"; - - TestConvertValueFromYdb(inputType, inputValue, - "Tuple {\n" - " Int32: 42\n" - "}\n" - "Tuple {\n" - " Bytes: \"abc\"\n" - "}\n"); - } - - Y_UNIT_TEST(Variant) { - const TString inputType = R"___(variant_type { - tuple_items { - elements { - type_id: UINT64 - } - elements { - type_id: UINT32 - } - } -} -)___"; - - const TString inputValue = R"___(nested_value { - uint32_value: 66 -} -variant_index: 1 -)___"; - - const TString expected = R"___(Optional { - Uint32: 66 -} -VariantIndex: 1 -)___"; - - TestConvertValueFromYdb(inputType, inputValue, expected); - } - - Y_UNIT_TEST(VariantIndexUnderflow) { - const TString inputType = R"___(variant_type { - tuple_items { - elements { - type_id: UINT64 - } - elements { - type_id: UINT32 - } - } -} -)___"; - - const TString inputValue = R"___(nested_value { - uint32_value: 66 -} -variant_index: 3435973836 -)___"; - - const TString expected = ""; - - UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, inputValue, expected), yexception); - } - - -} - -} // namespace NKikimr + const TString inputType = + "tuple_type {\n" + " elements {\n" + " type_id: INT32\n" + " }\n" + " elements {\n" + " type_id: STRING\n" + " }\n" + "}\n"; + + const TString inputValue = + "items {\n" + " int32_value: 42\n" + "}\n" + "items {\n" + " bytes_value: \"abc\"\n" + "}\n"; + + TestConvertValueFromYdb(inputType, inputValue, + "Tuple {\n" + " Int32: 42\n" + "}\n" + "Tuple {\n" + " Bytes: \"abc\"\n" + "}\n"); + } + + Y_UNIT_TEST(Variant) { + const TString inputType = R"___(variant_type { + tuple_items { + elements { + type_id: UINT64 + } + elements { + type_id: UINT32 + } + } +} +)___"; + + const TString inputValue = R"___(nested_value { + uint32_value: 66 +} +variant_index: 1 +)___"; + + const TString expected = R"___(Optional { + Uint32: 66 +} +VariantIndex: 1 +)___"; + + TestConvertValueFromYdb(inputType, inputValue, expected); + } + + Y_UNIT_TEST(VariantIndexUnderflow) { + const TString inputType = R"___(variant_type { + tuple_items { + elements { + type_id: UINT64 + } + elements { + type_id: UINT32 + } + } +} +)___"; + + const TString inputValue = R"___(nested_value { + uint32_value: 66 +} +variant_index: 3435973836 +)___"; + + const TString expected = ""; + + UNIT_ASSERT_EXCEPTION(TestConvertValueFromYdb(inputType, inputValue, expected), yexception); + } + + +} + +} // namespace NKikimr diff --git a/ydb/core/ymq/actor/executor.cpp b/ydb/core/ymq/actor/executor.cpp index b70fc7f92f..e17dc68fe9 100644 --- a/ydb/core/ymq/actor/executor.cpp +++ b/ydb/core/ymq/actor/executor.cpp @@ -444,7 +444,7 @@ void TMiniKqlExecutionActor::WaitForCompletion(bool retry) { const ui64 schemeShardId = ResponseEvent_->Get()->Record.GetSchemeShardTabletId(); NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = {.RetryLimitCount = 5, .MinRetryTime = TDuration::MilliSeconds(100), .DoFirstRetryInstantly = !retry}; - TabletPipeClient_ = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), schemeShardId, clientConfig)); + TabletPipeClient_ = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), schemeShardId, clientConfig)); TAutoPtr<NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion> request(new NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion()); request->Record.SetTxId(ResponseEvent_->Get()->Record.GetTxId()); diff --git a/ydb/core/ymq/actor/garbage_collector.cpp b/ydb/core/ymq/actor/garbage_collector.cpp index 2f00c8f8ba..e82284c020 100644 --- a/ydb/core/ymq/actor/garbage_collector.cpp +++ b/ydb/core/ymq/actor/garbage_collector.cpp @@ -142,7 +142,7 @@ private: if (entry.ListNodeEntry && CurrentDepth <= MaxDepth) { for (const auto& child : entry.ListNodeEntry->Children) { auto yetAnotherPathToTraverse = entry.Path; - yetAnotherPathToTraverse.push_back(child.Name); + yetAnotherPathToTraverse.push_back(child.Name); newPathsToTraverse.push_back(std::move(yetAnotherPathToTraverse)); } diff --git a/ydb/core/ymq/client/cpp/client.cpp b/ydb/core/ymq/client/cpp/client.cpp index b38bbf1ca5..c96f2dcaab 100644 --- a/ydb/core/ymq/client/cpp/client.cpp +++ b/ydb/core/ymq/client/cpp/client.cpp @@ -26,7 +26,7 @@ public: Y_CAT(Y_CAT(T, name), Response) resp; \ request.Y_CAT(Mutable, name)()->CopyFrom(req); \ TAutoEvent e; \ - Client_.SqsRequest(request, [e, &resp] (const NGRpcProxy::TGrpcError* error, const NKikimrClient::TSqsResponse& result) mutable \ + Client_.SqsRequest(request, [e, &resp] (const NGRpcProxy::TGrpcError* error, const NKikimrClient::TSqsResponse& result) mutable \ { \ if (error) { \ resp.MutableError()->SetStatus(502); \ diff --git a/ydb/core/yq/libs/ydb/ydb.cpp b/ydb/core/yq/libs/ydb/ydb.cpp index d6e1f82e3d..d6096abce3 100644 --- a/ydb/core/yq/libs/ydb/ydb.cpp +++ b/ydb/core/yq/libs/ydb/ydb.cpp @@ -241,7 +241,7 @@ TStatus MakeErrorStatus( auto& issue = issues.back(); issue.SetCode((ui32)code, severity); - return TStatus(code, std::move(issues)); + return TStatus(code, std::move(issues)); } NYql::TIssues StatusToIssues(const NYdb::TStatus& status) { diff --git a/ydb/library/aclib/aclib.cpp b/ydb/library/aclib/aclib.cpp index 2931ab736a..946497ea30 100644 --- a/ydb/library/aclib/aclib.cpp +++ b/ydb/library/aclib/aclib.cpp @@ -11,8 +11,8 @@ std::pair<ui32, ui32>& operator |=(std::pair<ui32, ui32>& a, const std::pair<ui3 a.first |= b.first; a.second |= b.second; return a; -} - +} + void TUserToken::SetGroupSIDs(const TVector<TString>& groupSIDs) { auto& hashTable = *MutableGroupSIDs(); auto& hashBuckets = *hashTable.MutableBuckets(); @@ -249,12 +249,12 @@ void TSecurityObject::RemoveAccess(NACLib::EAccessType type, ui32 access, const void TSecurityObject::ClearAccess() Y_NO_SANITIZE("undefined") { static_cast<TACL*>(MutableACL())->ClearAccess(); -} - +} + NACLibProto::TACL TSecurityObject::GetImmediateACL() const Y_NO_SANITIZE("undefined") { return static_cast<const TACL&>(GetACL()).GetImmediateACL(); -} - +} + TSecurityObject::TSecurityObject() : IsContainer(false) {} @@ -688,17 +688,17 @@ void TDiffACL::RemoveAccess(const NACLibProto::TACE& access) { ace->CopyFrom(access); } -void TDiffACL::ClearAccess() { - NACLibProto::TDiffACE* diffACE(AddDiffACE()); +void TDiffACL::ClearAccess() { + NACLibProto::TDiffACE* diffACE(AddDiffACE()); diffACE->SetDiffType(static_cast<ui32>(EDiffType::Remove)); -} - -void TDiffACL::ClearAccessForSid(const NACLib::TSID& sid) { - NACLibProto::TDiffACE* diffACE(AddDiffACE()); +} + +void TDiffACL::ClearAccessForSid(const NACLib::TSID& sid) { + NACLibProto::TDiffACE* diffACE(AddDiffACE()); diffACE->SetDiffType(static_cast<ui32>(EDiffType::Remove)); - diffACE->MutableACE()->SetSID(sid); -} - + diffACE->MutableACE()->SetSID(sid); +} + TString AccessRightsToString(ui32 accessRights) { switch (accessRights) { case EAccessRights::GenericFull: return "Full"; diff --git a/ydb/library/aclib/aclib.h b/ydb/library/aclib/aclib.h index 4885b57c98..655ca4b669 100644 --- a/ydb/library/aclib/aclib.h +++ b/ydb/library/aclib/aclib.h @@ -8,7 +8,7 @@ namespace NACLib { #define BUILTIN_ACL_DOMAIN "builtin" #define BUILTIN_ACL_ROOT "root@" BUILTIN_ACL_DOMAIN -#define BUILTIN_ERROR_DOMAIN "error" +#define BUILTIN_ERROR_DOMAIN "error" enum EAccessRights : ui32 { // bitmask NoAccess = 0x00000000, @@ -121,8 +121,8 @@ public: void RemoveAccess(NACLib::EAccessType type, ui32 access, const NACLib::TSID& sid, ui32 inheritance = InheritObject | InheritContainer); void AddAccess(const NACLibProto::TACE& access); void RemoveAccess(const NACLibProto::TACE& access); - void ClearAccess(); - void ClearAccessForSid(const NACLib::TSID& sid); + void ClearAccess(); + void ClearAccessForSid(const NACLib::TSID& sid); }; class TSecurityObject : public NACLibProto::TSecurityObject { @@ -140,7 +140,7 @@ public: void AddAccess(EAccessType type, ui32 access, const TSID& sid, ui32 inheritance = InheritObject | InheritContainer); void RemoveAccess(NACLib::EAccessType type, ui32 access, const NACLib::TSID& sid, ui32 inheritance = InheritObject | InheritContainer); void ApplyDiff(const NACLibProto::TDiffACL& diffACL); - void ClearAccess(); + void ClearAccess(); TInstant GetExpireTime() const; TString ToString() const; // simple text format void FromString(const TString& string); // simple text format diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 9e39c9ba99..676cc831ee 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -181,7 +181,7 @@ void PrintValue(IOutputStream& out, TValueParser& parser) { break; default: - ThrowFatalError(TStringBuilder() + ThrowFatalError(TStringBuilder() << "Unexpected type kind: " << parser.GetKind()); } } @@ -244,7 +244,7 @@ TMaybe<TValue> TryReadTable(TDriver driver, const NTable::TTableDescription& des if (lastWrittenPK) { settings.From(NTable::TKeyBound::Exclusive(*lastWrittenPK)); } - auto result = session.ReadTable(fullTablePath, settings).ExtractValueSync(); + auto result = session.ReadTable(fullTablePath, settings).ExtractValueSync(); if (result.IsSuccess()) { iter = result; } @@ -284,7 +284,7 @@ TMaybe<TValue> TryReadTable(TDriver driver, const NTable::TTableDescription& des // Next if (resultSetCurrent.Truncated()) { - auto resultSetStreamPart = nextResult.ExtractValueSync(); + auto resultSetStreamPart = nextResult.ExtractValueSync(); if (!resultSetStreamPart.IsSuccess()) { LOG_DEBUG("resultSetStreamPart is not successful, EOS# " << (resultSetStreamPart.EOS() ? "true" : "false")); diff --git a/ydb/library/backup/db_iterator.h b/ydb/library/backup/db_iterator.h index b3b01c67f6..3abc8ecb99 100644 --- a/ydb/library/backup/db_iterator.h +++ b/ydb/library/backup/db_iterator.h @@ -4,7 +4,7 @@ #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <util/folder/path.h> -#include <util/generic/deque.h> +#include <util/generic/deque.h> namespace NYdb { diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index 4f381a14de..53d7722dee 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -15,80 +15,80 @@ namespace { void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res); -Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res) { - auto dataType = static_cast<const TDataType*>(type); - switch (dataType->GetSchemeType()) { - case NUdf::TDataType<bool>::Id: - res.set_bool_value(value.Get<bool>()); - break; - case NUdf::TDataType<ui8>::Id: - res.set_uint32_value(value.Get<ui8>()); - break; - case NUdf::TDataType<i8>::Id: - res.set_int32_value(value.Get<i8>()); - break; - case NUdf::TDataType<ui16>::Id: - res.set_uint32_value(value.Get<ui16>()); - break; - case NUdf::TDataType<i16>::Id: - res.set_int32_value(value.Get<i16>()); - break; - case NUdf::TDataType<i32>::Id: - res.set_int32_value(value.Get<i32>()); - break; - case NUdf::TDataType<ui32>::Id: - res.set_uint32_value(value.Get<ui32>()); - break; - case NUdf::TDataType<i64>::Id: - res.set_int64_value(value.Get<i64>()); - break; - case NUdf::TDataType<ui64>::Id: - res.set_uint64_value(value.Get<ui64>()); - break; - case NUdf::TDataType<float>::Id: - res.set_float_value(value.Get<float>()); - break; - case NUdf::TDataType<double>::Id: - res.set_double_value(value.Get<double>()); - break; - case NUdf::TDataType<NUdf::TJson>::Id: - case NUdf::TDataType<NUdf::TUtf8>::Id: { - const auto& stringRef = value.AsStringRef(); - res.set_text_value(stringRef.Data(), stringRef.Size()); - break; - } - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { - const NUdf::TUnboxedValue out(ValueToString(NUdf::GetDataSlot(dataType->GetSchemeType()), value)); - const auto& stringRef = out.AsStringRef(); - res.set_text_value(stringRef.Data(), stringRef.Size()); - break; - } - case NUdf::TDataType<NUdf::TDecimal>::Id: { - auto decimal = value.GetInt128(); - const auto p = reinterpret_cast<ui8*>(&decimal); - res.set_low_128(*reinterpret_cast<ui64*>(p)); - res.set_high_128(*reinterpret_cast<ui64*>(p+8)); - break; - } - case NUdf::TDataType<NUdf::TDate>::Id: - res.set_uint32_value(value.Get<ui16>()); - break; - case NUdf::TDataType<NUdf::TDatetime>::Id: - res.set_uint32_value(value.Get<ui32>()); - break; - case NUdf::TDataType<NUdf::TTimestamp>::Id: - res.set_uint64_value(value.Get<ui64>()); - break; - case NUdf::TDataType<NUdf::TInterval>::Id: - res.set_int64_value(value.Get<i64>()); - break; - case NUdf::TDataType<NUdf::TUuid>::Id: { - const auto& stringRef = value.AsStringRef(); - UuidToYdbProto(stringRef.Data(), stringRef.Size(), res); - break; - } +Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res) { + auto dataType = static_cast<const TDataType*>(type); + switch (dataType->GetSchemeType()) { + case NUdf::TDataType<bool>::Id: + res.set_bool_value(value.Get<bool>()); + break; + case NUdf::TDataType<ui8>::Id: + res.set_uint32_value(value.Get<ui8>()); + break; + case NUdf::TDataType<i8>::Id: + res.set_int32_value(value.Get<i8>()); + break; + case NUdf::TDataType<ui16>::Id: + res.set_uint32_value(value.Get<ui16>()); + break; + case NUdf::TDataType<i16>::Id: + res.set_int32_value(value.Get<i16>()); + break; + case NUdf::TDataType<i32>::Id: + res.set_int32_value(value.Get<i32>()); + break; + case NUdf::TDataType<ui32>::Id: + res.set_uint32_value(value.Get<ui32>()); + break; + case NUdf::TDataType<i64>::Id: + res.set_int64_value(value.Get<i64>()); + break; + case NUdf::TDataType<ui64>::Id: + res.set_uint64_value(value.Get<ui64>()); + break; + case NUdf::TDataType<float>::Id: + res.set_float_value(value.Get<float>()); + break; + case NUdf::TDataType<double>::Id: + res.set_double_value(value.Get<double>()); + break; + case NUdf::TDataType<NUdf::TJson>::Id: + case NUdf::TDataType<NUdf::TUtf8>::Id: { + const auto& stringRef = value.AsStringRef(); + res.set_text_value(stringRef.Data(), stringRef.Size()); + break; + } + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { + const NUdf::TUnboxedValue out(ValueToString(NUdf::GetDataSlot(dataType->GetSchemeType()), value)); + const auto& stringRef = out.AsStringRef(); + res.set_text_value(stringRef.Data(), stringRef.Size()); + break; + } + case NUdf::TDataType<NUdf::TDecimal>::Id: { + auto decimal = value.GetInt128(); + const auto p = reinterpret_cast<ui8*>(&decimal); + res.set_low_128(*reinterpret_cast<ui64*>(p)); + res.set_high_128(*reinterpret_cast<ui64*>(p+8)); + break; + } + case NUdf::TDataType<NUdf::TDate>::Id: + res.set_uint32_value(value.Get<ui16>()); + break; + case NUdf::TDataType<NUdf::TDatetime>::Id: + res.set_uint32_value(value.Get<ui32>()); + break; + case NUdf::TDataType<NUdf::TTimestamp>::Id: + res.set_uint64_value(value.Get<ui64>()); + break; + case NUdf::TDataType<NUdf::TInterval>::Id: + res.set_int64_value(value.Get<i64>()); + break; + case NUdf::TDataType<NUdf::TUuid>::Id: { + const auto& stringRef = value.AsStringRef(); + UuidToYdbProto(stringRef.Data(), stringRef.Size(), res); + break; + } case NUdf::TDataType<NUdf::TJsonDocument>::Id: { NUdf::TUnboxedValue json = ValueToString(NUdf::EDataSlot::JsonDocument, value); const auto stringRef = json.AsStringRef(); @@ -101,12 +101,12 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed res.set_text_value(stringRef.Data(), stringRef.Size()); break; } - default: - const auto& stringRef = value.AsStringRef(); - res.set_bytes_value(stringRef.Data(), stringRef.Size()); - } -} - + default: + const auto& stringRef = value.AsStringRef(); + res.set_bytes_value(stringRef.Data(), stringRef.Size()); + } +} + template<typename TOut> Y_FORCE_INLINE void ExportStructTypeToProto(TStructType* structType, TOut& res) { for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { @@ -216,12 +216,12 @@ void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res) { } } -void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { - switch (type->GetKind()) { - case TType::EKind::Void: - res.set_void_type(::google::protobuf::NULL_VALUE); - break; - +void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { + switch (type->GetKind()) { + case TType::EKind::Void: + res.set_void_type(::google::protobuf::NULL_VALUE); + break; + case TType::EKind::Null: res.set_null_type(::google::protobuf::NULL_VALUE); break; @@ -234,89 +234,89 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { res.set_empty_dict_type(::google::protobuf::NULL_VALUE); break; - case TType::EKind::Data: { - auto dataType = static_cast<TDataType*>(type); - auto schemeType = dataType->GetSchemeType(); - - switch (schemeType) { - case NYql::NProto::TypeIds::Decimal: { - auto decimalType = static_cast<TDataDecimalType*>(dataType); - auto params = decimalType->GetParams(); - auto decimalParams = res.mutable_decimal_type(); - decimalParams->set_precision(params.first); - decimalParams->set_scale(params.second); - break; - } - default: - ExportPrimitiveTypeToProto(schemeType, res); - break; - } - - break; - } - - case TType::EKind::Optional: { - auto optionalType = static_cast<TOptionalType*>(type); - ExportTypeToProtoImpl(optionalType->GetItemType(), *res.mutable_optional_type()->mutable_item()); - break; - } - - case TType::EKind::List: { - auto listType = static_cast<TListType*>(type); - ExportTypeToProtoImpl(listType->GetItemType(), *res.mutable_list_type()->mutable_item()); - break; - } - - case TType::EKind::Struct: { - auto structType = static_cast<TStructType*>(type); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto newMember = res.mutable_struct_type()->add_members(); - newMember->set_name(TString(structType->GetMemberName(index))); - ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->mutable_type()); - } - - break; - } - - case TType::EKind::Tuple: { - auto tupleType = static_cast<TTupleType*>(type); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - ExportTypeToProtoImpl(tupleType->GetElementType(index), *res.mutable_tuple_type()->add_elements()); - } - - break; - } - - case TType::EKind::Dict: { - auto dictType = static_cast<TDictType*>(type); - ExportTypeToProtoImpl(dictType->GetKeyType(), *res.mutable_dict_type()->mutable_key()); - ExportTypeToProtoImpl(dictType->GetPayloadType(), *res.mutable_dict_type()->mutable_payload()); - break; - } - - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType*>(type); - TType* innerType = variantType->GetUnderlyingType(); - if (innerType->IsStruct()) { - TStructType* structType = static_cast<TStructType*>(innerType); - auto resItems = res.mutable_variant_type()->mutable_struct_items(); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto newMember = resItems->add_members(); - newMember->set_name(TString(structType->GetMemberName(index))); - ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->mutable_type()); - } - } else if (innerType->IsTuple()) { - auto resItems = res.mutable_variant_type()->mutable_tuple_items(); - TTupleType* tupleType = static_cast<TTupleType*>(innerType); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - ExportTypeToProtoImpl(tupleType->GetElementType(index), *resItems->add_elements()); - } - } else { - MKQL_ENSURE(false, TStringBuilder() << "Unknown underlying variant type: " << innerType->GetKindAsStr()); - } - break; - } - + case TType::EKind::Data: { + auto dataType = static_cast<TDataType*>(type); + auto schemeType = dataType->GetSchemeType(); + + switch (schemeType) { + case NYql::NProto::TypeIds::Decimal: { + auto decimalType = static_cast<TDataDecimalType*>(dataType); + auto params = decimalType->GetParams(); + auto decimalParams = res.mutable_decimal_type(); + decimalParams->set_precision(params.first); + decimalParams->set_scale(params.second); + break; + } + default: + ExportPrimitiveTypeToProto(schemeType, res); + break; + } + + break; + } + + case TType::EKind::Optional: { + auto optionalType = static_cast<TOptionalType*>(type); + ExportTypeToProtoImpl(optionalType->GetItemType(), *res.mutable_optional_type()->mutable_item()); + break; + } + + case TType::EKind::List: { + auto listType = static_cast<TListType*>(type); + ExportTypeToProtoImpl(listType->GetItemType(), *res.mutable_list_type()->mutable_item()); + break; + } + + case TType::EKind::Struct: { + auto structType = static_cast<TStructType*>(type); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + auto newMember = res.mutable_struct_type()->add_members(); + newMember->set_name(TString(structType->GetMemberName(index))); + ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->mutable_type()); + } + + break; + } + + case TType::EKind::Tuple: { + auto tupleType = static_cast<TTupleType*>(type); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + ExportTypeToProtoImpl(tupleType->GetElementType(index), *res.mutable_tuple_type()->add_elements()); + } + + break; + } + + case TType::EKind::Dict: { + auto dictType = static_cast<TDictType*>(type); + ExportTypeToProtoImpl(dictType->GetKeyType(), *res.mutable_dict_type()->mutable_key()); + ExportTypeToProtoImpl(dictType->GetPayloadType(), *res.mutable_dict_type()->mutable_payload()); + break; + } + + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType*>(type); + TType* innerType = variantType->GetUnderlyingType(); + if (innerType->IsStruct()) { + TStructType* structType = static_cast<TStructType*>(innerType); + auto resItems = res.mutable_variant_type()->mutable_struct_items(); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + auto newMember = resItems->add_members(); + newMember->set_name(TString(structType->GetMemberName(index))); + ExportTypeToProtoImpl(structType->GetMemberType(index), *newMember->mutable_type()); + } + } else if (innerType->IsTuple()) { + auto resItems = res.mutable_variant_type()->mutable_tuple_items(); + TTupleType* tupleType = static_cast<TTupleType*>(innerType); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + ExportTypeToProtoImpl(tupleType->GetElementType(index), *resItems->add_elements()); + } + } else { + MKQL_ENSURE(false, TStringBuilder() << "Unknown underlying variant type: " << innerType->GetKindAsStr()); + } + break; + } + case TType::EKind::Tagged: { auto taggedType = static_cast<TTaggedType*>(type); TType* innerType = taggedType->GetBaseType(); @@ -326,12 +326,12 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { break; } - default: - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } -} - - + default: + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } +} + + Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res) { auto dataType = static_cast<const TDataType*>(type); switch (dataType->GetSchemeType()) { @@ -519,180 +519,180 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK } void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) { - switch (type->GetKind()) { - case TType::EKind::Void: + switch (type->GetKind()) { + case TType::EKind::Void: case TType::EKind::EmptyList: case TType::EKind::EmptyDict: break; - + case TType::EKind::Null: { res.set_null_flag_value(::google::protobuf::NULL_VALUE); break; } - case TType::EKind::Data: { - HandleKindDataExport(type, value, res); - break; - } - - case TType::EKind::Optional: { - auto optionalType = static_cast<TOptionalType*>(type); - if (value.HasValue()) { - const auto nextType = optionalType->GetItemType(); - if (nextType->GetKind() == TType::EKind::Data) { - HandleKindDataExport(nextType, value.GetOptionalValue(), res); - } else { - ExportValueToProtoImpl(nextType, value.GetOptionalValue(), res); - } - } else { - if (value) { - ExportValueToProtoImpl(optionalType->GetItemType(), value.GetOptionalValue(), *res.mutable_nested_value()); - } else { - res.set_null_flag_value(::google::protobuf::NULL_VALUE); - } - } - - break; - } - - case TType::EKind::List: { - auto listType = static_cast<TListType*>(type); - auto itemType = listType->GetItemType(); - if (value.HasFastListLength()) { - res.mutable_items()->Reserve(value.GetListLength()); - } - const auto iter = value.GetListIterator(); - for (NUdf::TUnboxedValue item; iter.Next(item);) { - ExportValueToProtoImpl(itemType, item, *res.mutable_items()->Add()); - } - - break; - } - - case TType::EKind::Struct: { - auto structType = static_cast<TStructType*>(type); - res.mutable_items()->Reserve(structType->GetMembersCount()); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + case TType::EKind::Data: { + HandleKindDataExport(type, value, res); + break; + } + + case TType::EKind::Optional: { + auto optionalType = static_cast<TOptionalType*>(type); + if (value.HasValue()) { + const auto nextType = optionalType->GetItemType(); + if (nextType->GetKind() == TType::EKind::Data) { + HandleKindDataExport(nextType, value.GetOptionalValue(), res); + } else { + ExportValueToProtoImpl(nextType, value.GetOptionalValue(), res); + } + } else { + if (value) { + ExportValueToProtoImpl(optionalType->GetItemType(), value.GetOptionalValue(), *res.mutable_nested_value()); + } else { + res.set_null_flag_value(::google::protobuf::NULL_VALUE); + } + } + + break; + } + + case TType::EKind::List: { + auto listType = static_cast<TListType*>(type); + auto itemType = listType->GetItemType(); + if (value.HasFastListLength()) { + res.mutable_items()->Reserve(value.GetListLength()); + } + const auto iter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item);) { + ExportValueToProtoImpl(itemType, item, *res.mutable_items()->Add()); + } + + break; + } + + case TType::EKind::Struct: { + auto structType = static_cast<TStructType*>(type); + res.mutable_items()->Reserve(structType->GetMembersCount()); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; auto memberType = structType->GetMemberType(memberIndex); ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.mutable_items()->Add()); - } - - break; - } - - case TType::EKind::Tuple: { - auto tupleType = static_cast<TTupleType*>(type); - res.mutable_items()->Reserve(tupleType->GetElementsCount()); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - auto elementType = tupleType->GetElementType(index); - ExportValueToProtoImpl(elementType, value.GetElement(index), *res.mutable_items()->Add()); - } - - break; - } - - case TType::EKind::Dict: { - auto dictType = static_cast<TDictType*>(type); - auto keyType = dictType->GetKeyType(); - auto payloadType = dictType->GetPayloadType(); - const auto iter = value.GetDictIterator(); - for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) { - auto dictItem = res.mutable_pairs()->Add(); - ExportValueToProtoImpl(keyType, key, *dictItem->mutable_key()); - ExportValueToProtoImpl(payloadType, payload, *dictItem->mutable_payload()); - } - - break; - } - - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType*>(type); - auto variantIndex = value.GetVariantIndex(); - res.set_variant_index(variantIndex); - ExportValueToProtoImpl(variantType->GetAlternativeType(variantIndex), value.GetVariantItem(), *res.mutable_nested_value()); - - break; - } - + } + + break; + } + + case TType::EKind::Tuple: { + auto tupleType = static_cast<TTupleType*>(type); + res.mutable_items()->Reserve(tupleType->GetElementsCount()); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + auto elementType = tupleType->GetElementType(index); + ExportValueToProtoImpl(elementType, value.GetElement(index), *res.mutable_items()->Add()); + } + + break; + } + + case TType::EKind::Dict: { + auto dictType = static_cast<TDictType*>(type); + auto keyType = dictType->GetKeyType(); + auto payloadType = dictType->GetPayloadType(); + const auto iter = value.GetDictIterator(); + for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) { + auto dictItem = res.mutable_pairs()->Add(); + ExportValueToProtoImpl(keyType, key, *dictItem->mutable_key()); + ExportValueToProtoImpl(payloadType, payload, *dictItem->mutable_payload()); + } + + break; + } + + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType*>(type); + auto variantIndex = value.GetVariantIndex(); + res.set_variant_index(variantIndex); + ExportValueToProtoImpl(variantType->GetAlternativeType(variantIndex), value.GetVariantItem(), *res.mutable_nested_value()); + + break; + } + case TType::EKind::Tagged: { auto taggedType = static_cast<TTaggedType*>(type); ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); break; } - default: { - MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); - } - } -} - -Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const NKikimrMiniKQL::TValue& value) { - auto dataType = static_cast<const TDataType*>(type); - auto oneOfCase = value.value_value_case(); + default: { + MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); + } + } +} + +Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const NKikimrMiniKQL::TValue& value) { + auto dataType = static_cast<const TDataType*>(type); + auto oneOfCase = value.value_value_case(); switch (dataType->GetSchemeType()) { case NUdf::TDataType<bool>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBool); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBool); return NUdf::TUnboxedValuePod(value.GetBool()); case NUdf::TDataType<ui8>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); return NUdf::TUnboxedValuePod(ui8(value.GetUint32())); case NUdf::TDataType<i8>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); return NUdf::TUnboxedValuePod(i8(value.GetInt32())); case NUdf::TDataType<ui16>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); case NUdf::TDataType<i16>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); return NUdf::TUnboxedValuePod(i16(value.GetInt32())); case NUdf::TDataType<i32>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt32); return NUdf::TUnboxedValuePod(value.GetInt32()); case NUdf::TDataType<ui32>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); return NUdf::TUnboxedValuePod(value.GetUint32()); case NUdf::TDataType<i64>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); return NUdf::TUnboxedValuePod(value.GetInt64()); case NUdf::TDataType<ui64>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); return NUdf::TUnboxedValuePod(value.GetUint64()); case NUdf::TDataType<float>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kFloat); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kFloat); return NUdf::TUnboxedValuePod(value.GetFloat()); case NUdf::TDataType<double>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kDouble); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kDouble); return NUdf::TUnboxedValuePod(value.GetDouble()); case NUdf::TDataType<NUdf::TJson>::Id: case NUdf::TDataType<NUdf::TUtf8>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); return MakeString(value.GetText()); - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); - return NUdf::TUnboxedValuePod(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText())); - } + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kText); + return NUdf::TUnboxedValuePod(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText())); + } case NUdf::TDataType<NUdf::TDate>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); return NUdf::TUnboxedValuePod(ui16(value.GetUint32())); case NUdf::TDataType<NUdf::TDatetime>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint32); return NUdf::TUnboxedValuePod(value.GetUint32()); case NUdf::TDataType<NUdf::TTimestamp>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kUint64); return NUdf::TUnboxedValuePod(value.GetUint64()); case NUdf::TDataType<NUdf::TInterval>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kInt64); return NUdf::TUnboxedValuePod(value.GetInt64()); case NUdf::TDataType<NUdf::TJsonDocument>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes); return MakeString(value.GetBytes()); case NUdf::TDataType<NUdf::TDecimal>::Id: return NUdf::TUnboxedValuePod(NYql::NDecimal::FromHalfs(value.GetLow128(), value.GetHi128())); case NUdf::TDataType<NUdf::TDyNumber>::Id: - MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes); + MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes); return MakeString(value.GetBytes()); case NUdf::TDataType<NUdf::TUuid>::Id: { MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kLow128); @@ -713,49 +713,49 @@ Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const } -void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output) { - switch (schemeType) { - case NYql::NProto::TypeIds::Bool: - case NYql::NProto::TypeIds::Int8: - case NYql::NProto::TypeIds::Uint8: - case NYql::NProto::TypeIds::Int16: - case NYql::NProto::TypeIds::Uint16: - case NYql::NProto::TypeIds::Int32: - case NYql::NProto::TypeIds::Uint32: - case NYql::NProto::TypeIds::Int64: - case NYql::NProto::TypeIds::Uint64: - case NYql::NProto::TypeIds::Float: - case NYql::NProto::TypeIds::Double: - case NYql::NProto::TypeIds::Date: - case NYql::NProto::TypeIds::Datetime: - case NYql::NProto::TypeIds::Timestamp: - case NYql::NProto::TypeIds::Interval: - case NYql::NProto::TypeIds::TzDate: - case NYql::NProto::TypeIds::TzDatetime: - case NYql::NProto::TypeIds::TzTimestamp: - case NYql::NProto::TypeIds::String: - case NYql::NProto::TypeIds::Utf8: - case NYql::NProto::TypeIds::Yson: - case NYql::NProto::TypeIds::Json: - case NYql::NProto::TypeIds::Uuid: +void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output) { + switch (schemeType) { + case NYql::NProto::TypeIds::Bool: + case NYql::NProto::TypeIds::Int8: + case NYql::NProto::TypeIds::Uint8: + case NYql::NProto::TypeIds::Int16: + case NYql::NProto::TypeIds::Uint16: + case NYql::NProto::TypeIds::Int32: + case NYql::NProto::TypeIds::Uint32: + case NYql::NProto::TypeIds::Int64: + case NYql::NProto::TypeIds::Uint64: + case NYql::NProto::TypeIds::Float: + case NYql::NProto::TypeIds::Double: + case NYql::NProto::TypeIds::Date: + case NYql::NProto::TypeIds::Datetime: + case NYql::NProto::TypeIds::Timestamp: + case NYql::NProto::TypeIds::Interval: + case NYql::NProto::TypeIds::TzDate: + case NYql::NProto::TypeIds::TzDatetime: + case NYql::NProto::TypeIds::TzTimestamp: + case NYql::NProto::TypeIds::String: + case NYql::NProto::TypeIds::Utf8: + case NYql::NProto::TypeIds::Yson: + case NYql::NProto::TypeIds::Json: + case NYql::NProto::TypeIds::Uuid: case NYql::NProto::TypeIds::JsonDocument: case NYql::NProto::TypeIds::DyNumber: - output.set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(schemeType)); - break; - - default: - MKQL_ENSURE(false, TStringBuilder() << "Unsupported primitive scheme type: " << schemeType); - } -} - -void ExportTypeToProto(TType* type, Ydb::Type& res) { - ExportTypeToProtoImpl(type, res); -} - + output.set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(schemeType)); + break; + + default: + MKQL_ENSURE(false, TStringBuilder() << "Unsupported primitive scheme type: " << schemeType); + } +} + +void ExportTypeToProto(TType* type, Ydb::Type& res) { + ExportTypeToProtoImpl(type, res); +} + void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder) { ExportValueToProtoImpl(type, value, res, columnOrder); -} - +} + template <typename ValueType> class TBufferArray { protected: using size_type = ui32; @@ -872,9 +872,9 @@ public: ~TProtoImporter(); TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type); TNode* ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TValue& value); - NUdf::TUnboxedValue ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, - const THolderFactory& factory); -private: + NUdf::TUnboxedValue ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, + const THolderFactory& factory); +private: TTupleType* ImportTupleTypeFromProto(const NKikimrMiniKQL::TTupleType& protoTupleType); TStructType* ImportStructTypeFromProto(const NKikimrMiniKQL::TStructType& protoStructType); const TTypeEnvironment& env; @@ -1054,11 +1054,11 @@ TNode* TProtoImporter::ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TV case NUdf::TDataType<NUdf::TUtf8>::Id: dataNode = TDataLiteral::Create(env.NewStringValue(value.GetText()), dataType, env); break; - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: - dataNode = TDataLiteral::Create(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText()), dataType, env); - break; + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: + dataNode = TDataLiteral::Create(ValueFromString(NUdf::GetDataSlot(dataType->GetSchemeType()), value.GetText()), dataType, env); + break; case NUdf::TDataType<NUdf::TDate>::Id: dataNode = TDataLiteral::Create(NUdf::TUnboxedValuePod(ui16(value.GetUint32())), dataType, env); break; @@ -1184,7 +1184,7 @@ void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikim } -NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, const THolderFactory& factory) { +NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, const NKikimrMiniKQL::TValue& value, const THolderFactory& factory) { switch (type->GetKind()) { case TType::EKind::Void: return NUdf::TUnboxedValuePod::Void(); @@ -1196,10 +1196,10 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons return HandleKindDataImport(type, value); case TType::EKind::Optional: { - auto optionalType = static_cast<const TOptionalType*>(type); + auto optionalType = static_cast<const TOptionalType*>(type); if (value.HasOptional()) { - const TType* itemType = optionalType->GetItemType(); - return ImportValueFromProto(itemType, value.GetOptional(), factory).MakeOptional(); + const TType* itemType = optionalType->GetItemType(); + return ImportValueFromProto(itemType, value.GetOptional(), factory).MakeOptional(); } else { return NUdf::TUnboxedValue(); @@ -1207,8 +1207,8 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons } case TType::EKind::List: { - auto listType = static_cast<const TListType*>(type); - const TType* itemType = listType->GetItemType(); + auto listType = static_cast<const TListType*>(type); + const TType* itemType = listType->GetItemType(); const auto& list = value.GetList(); NUdf::TUnboxedValue *items = nullptr; auto array = factory.CreateDirectArrayHolder(list.size(), items); @@ -1220,25 +1220,25 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons } case TType::EKind::Struct: { - auto structType = static_cast<const TStructType*>(type); + auto structType = static_cast<const TStructType*>(type); NUdf::TUnboxedValue* itemsPtr = nullptr; auto res = factory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr); - TRemapArray remap = TRemapArray::FromCookie(structType->GetCookie()); + TRemapArray remap = TRemapArray::FromCookie(structType->GetCookie()); for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - ui32 remapped = remap.empty() ? index : remap[index]; - const TType* memberType = structType->GetMemberType(index); - itemsPtr[index] = ImportValueFromProto(memberType, value.GetStruct(remapped), factory); + ui32 remapped = remap.empty() ? index : remap[index]; + const TType* memberType = structType->GetMemberType(index); + itemsPtr[index] = ImportValueFromProto(memberType, value.GetStruct(remapped), factory); } return std::move(res); } case TType::EKind::Tuple: { - auto tupleType = static_cast<const TTupleType*>(type); + auto tupleType = static_cast<const TTupleType*>(type); NUdf::TUnboxedValue* itemsPtr = nullptr; auto res = factory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr); for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - const TType* elementType = tupleType->GetElementType(index); + const TType* elementType = tupleType->GetElementType(index); itemsPtr[index] = ImportValueFromProto(elementType, value.GetTuple(index), factory); } @@ -1246,9 +1246,9 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons } case TType::EKind::Dict: { - auto dictType = static_cast<const TDictType*>(type); - const TType* keyType = dictType->GetKeyType(); - const TType* payloadType = dictType->GetPayloadType(); + auto dictType = static_cast<const TDictType*>(type); + const TType* keyType = dictType->GetKeyType(); + const TType* payloadType = dictType->GetPayloadType(); auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed); for (const auto& x : value.GetDict()) { @@ -1266,20 +1266,20 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons } } -std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, - const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env, const THolderFactory& factory) -{ - TProtoImporter importer(env); - TType* nodeType = importer.ImportTypeFromProto(type); - auto unboxedValue = importer.ImportValueFromProto(nodeType, value, factory); - return {nodeType, unboxedValue}; -} - -TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env) { - TProtoImporter importer(env); - return importer.ImportTypeFromProto(type); -} - +std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, + const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env, const THolderFactory& factory) +{ + TProtoImporter importer(env); + TType* nodeType = importer.ImportTypeFromProto(type); + auto unboxedValue = importer.ImportValueFromProto(nodeType, value, factory); + return {nodeType, unboxedValue}; +} + +TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env) { + TProtoImporter importer(env); + return importer.ImportTypeFromProto(type); +} + TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env) { diff --git a/ydb/library/mkql_proto/mkql_proto.h b/ydb/library/mkql_proto/mkql_proto.h index ab43386a76..716f8530d3 100644 --- a/ydb/library/mkql_proto/mkql_proto.h +++ b/ydb/library/mkql_proto/mkql_proto.h @@ -12,16 +12,16 @@ class THolderFactory; void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res); void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr); -void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output); - -void ExportTypeToProto(TType* type, Ydb::Type& res); +void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output); + +void ExportTypeToProto(TType* type, Ydb::Type& res); void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr); - - + + TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env); -std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, - const TTypeEnvironment& env, const THolderFactory& factory); +std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, + const TTypeEnvironment& env, const THolderFactory& factory); TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, const TTypeEnvironment& env); TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env); @@ -37,15 +37,15 @@ inline void UuidToMkqlProto(const char* str, size_t sz, NKikimrMiniKQL::TValue& res.SetHi128(buf.half[1]); } -inline void UuidToYdbProto(const char* str, size_t sz, Ydb::Value& res) { - union { - ui64 half[2]; - char bytes[sizeof(ui64) * 2]; - } buf; - Y_VERIFY(sizeof(buf) == sz); - memcpy(buf.bytes, str, sizeof(buf)); - res.set_low_128(buf.half[0]); - res.set_high_128(buf.half[1]); +inline void UuidToYdbProto(const char* str, size_t sz, Ydb::Value& res) { + union { + ui64 half[2]; + char bytes[sizeof(ui64) * 2]; + } buf; + Y_VERIFY(sizeof(buf) == sz); + memcpy(buf.bytes, str, sizeof(buf)); + res.set_low_128(buf.half[0]); + res.set_high_128(buf.half[1]); +} + } - -} diff --git a/ydb/library/mkql_proto/protos/minikql.proto b/ydb/library/mkql_proto/protos/minikql.proto index a6e4533ca2..2a4243855c 100644 --- a/ydb/library/mkql_proto/protos/minikql.proto +++ b/ydb/library/mkql_proto/protos/minikql.proto @@ -1,6 +1,6 @@ package NKikimrMiniKQL; option java_package = "ru.yandex.kikimr.proto"; -option cc_enable_arenas = true; +option cc_enable_arenas = true; import "google/protobuf/struct.proto"; @@ -13,7 +13,7 @@ enum ETypeKind { Tuple = 5; Struct = 6; Dict = 7; - Variant = 8; + Variant = 8; Null = 9; Reserved_10 = 10; Reserved_11 = 11; @@ -22,14 +22,14 @@ enum ETypeKind { Reserved_14 = 14; } -message TDecimalParams { - required uint32 Precision = 1; - required uint32 Scale = 2; -} - +message TDecimalParams { + required uint32 Precision = 1; + required uint32 Scale = 2; +} + message TDataType { required uint32 Scheme = 1; - optional TDecimalParams DecimalParams = 2; + optional TDecimalParams DecimalParams = 2; } message TOptionalType { @@ -53,13 +53,13 @@ message TStructType { repeated TMember Member = 1; } -message TVariantType { - oneof type { - TTupleType TupleItems = 1; - TStructType StructItems = 2; - } -} - +message TVariantType { + oneof type { + TTupleType TupleItems = 1; + TStructType StructItems = 2; + } +} + message TDictType { required TType Key = 1; required TType Payload = 2; @@ -74,7 +74,7 @@ message TType { TTupleType Tuple = 5; TStructType Struct = 6; TDictType Dict = 7; - TVariantType Variant = 8; + TVariantType Variant = 8; } } @@ -95,7 +95,7 @@ message TValue { bytes Bytes = 8; string Text = 9; TValue Optional = 10; - fixed64 Low128 = 15; + fixed64 Low128 = 15; google.protobuf.NullValue NullFlagValue = 18; // Set if current TValue is terminal Null } // Logically part of oneof, @@ -105,8 +105,8 @@ message TValue { repeated TValue Tuple = 12; repeated TValue Struct = 13; repeated TValuePair Dict = 14; - optional fixed64 Hi128 = 16; - optional uint32 VariantIndex = 17; + optional fixed64 Hi128 = 16; + optional uint32 VariantIndex = 17; } message TResult { diff --git a/ydb/library/persqueue/tests/counters.cpp b/ydb/library/persqueue/tests/counters.cpp index c26460afb0..caddba0e04 100644 --- a/ydb/library/persqueue/tests/counters.cpp +++ b/ydb/library/persqueue/tests/counters.cpp @@ -1 +1 @@ -#include "counters.h" +#include "counters.h" diff --git a/ydb/library/persqueue/tests/counters.h b/ydb/library/persqueue/tests/counters.h index 22845bbad9..6e346c4ac8 100644 --- a/ydb/library/persqueue/tests/counters.h +++ b/ydb/library/persqueue/tests/counters.h @@ -1,71 +1,71 @@ -#pragma once - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/http/io/stream.h> -#include <library/cpp/json/json_reader.h> -#include <library/cpp/json/json_value.h> - -#include <util/string/builder.h> -#include <util/string/vector.h> -#include <util/string/join.h> -#include <util/network/socket.h> - - - -namespace NKikimr::NPersQueueTests { - -NJson::TJsonValue GetCounters(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath, const TString& clientDc = "", const TString& originalDc = "") { - TString dcFilter = ""; - { - auto prepareDC = [](TString dc) { - dc.front() = std::toupper(dc.front()); - return dc; - }; - if (originalDc) { - dcFilter = TStringBuilder() << "/OriginDC=" << prepareDC(originalDc); - } else if (clientDc) { - dcFilter = TStringBuilder() << "/ClientDC=" << prepareDC(clientDc); - } - } - TVector<TString> pathItems = SplitString(topicPath, "/"); - UNIT_ASSERT(pathItems.size() >= 2); - TString account = pathItems.front(); - TString producer = JoinRange("@", pathItems.begin(), pathItems.end() - 1); - TString query = TStringBuilder() << "/counters/counters=" << counters - << "/subsystem=" << subsystem << "/Account=" << account << "/Producer=" << producer - << "/Topic=" << Join("--", producer, pathItems.back()) << "/TopicPath=" << JoinRange("%2F", pathItems.begin(), pathItems.end()) - << dcFilter << "/json"; - - - TNetworkAddress addr("localhost", port); - TString q2 = TStringBuilder() << "/counters/counters=" << counters - << "/subsystem=" << subsystem;// << "/Account=" << account << "/Producer=" << producer; - //q2 += "/json"; - - /*Cerr << "q2: " << q2 << Endl; - TSocket s2(addr); - SendMinimalHttpRequest(s2, "localhost", q2); - TSocketInput si2(s2); - THttpInput input2(&si2); - Cerr << "===Counters response: " << input2.ReadAll() << Endl;*/ - - - - Cerr << "===Request counters with query: " << query << Endl; - TSocket s(addr); - SendMinimalHttpRequest(s, "localhost", query); - TSocketInput si(s); - THttpInput input(&si); - TString firstLine = input.FirstLine(); - //Cerr << "Counters2: '" << firstLine << "' content: " << input.ReadAll() << Endl; - - unsigned httpCode = ParseHttpRetCode(firstLine); - UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); - NJson::TJsonValue value; - UNIT_ASSERT(NJson::ReadJsonTree(&input, &value)); - - Cerr << "counters: " << value.GetStringRobust() << "\n"; - return value; -} - -} // NKikimr::NPersQueueTests +#pragma once + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/http/io/stream.h> +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_value.h> + +#include <util/string/builder.h> +#include <util/string/vector.h> +#include <util/string/join.h> +#include <util/network/socket.h> + + + +namespace NKikimr::NPersQueueTests { + +NJson::TJsonValue GetCounters(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath, const TString& clientDc = "", const TString& originalDc = "") { + TString dcFilter = ""; + { + auto prepareDC = [](TString dc) { + dc.front() = std::toupper(dc.front()); + return dc; + }; + if (originalDc) { + dcFilter = TStringBuilder() << "/OriginDC=" << prepareDC(originalDc); + } else if (clientDc) { + dcFilter = TStringBuilder() << "/ClientDC=" << prepareDC(clientDc); + } + } + TVector<TString> pathItems = SplitString(topicPath, "/"); + UNIT_ASSERT(pathItems.size() >= 2); + TString account = pathItems.front(); + TString producer = JoinRange("@", pathItems.begin(), pathItems.end() - 1); + TString query = TStringBuilder() << "/counters/counters=" << counters + << "/subsystem=" << subsystem << "/Account=" << account << "/Producer=" << producer + << "/Topic=" << Join("--", producer, pathItems.back()) << "/TopicPath=" << JoinRange("%2F", pathItems.begin(), pathItems.end()) + << dcFilter << "/json"; + + + TNetworkAddress addr("localhost", port); + TString q2 = TStringBuilder() << "/counters/counters=" << counters + << "/subsystem=" << subsystem;// << "/Account=" << account << "/Producer=" << producer; + //q2 += "/json"; + + /*Cerr << "q2: " << q2 << Endl; + TSocket s2(addr); + SendMinimalHttpRequest(s2, "localhost", q2); + TSocketInput si2(s2); + THttpInput input2(&si2); + Cerr << "===Counters response: " << input2.ReadAll() << Endl;*/ + + + + Cerr << "===Request counters with query: " << query << Endl; + TSocket s(addr); + SendMinimalHttpRequest(s, "localhost", query); + TSocketInput si(s); + THttpInput input(&si); + TString firstLine = input.FirstLine(); + //Cerr << "Counters2: '" << firstLine << "' content: " << input.ReadAll() << Endl; + + unsigned httpCode = ParseHttpRetCode(firstLine); + UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); + NJson::TJsonValue value; + UNIT_ASSERT(NJson::ReadJsonTree(&input, &value)); + + Cerr << "counters: " << value.GetStringRobust() << "\n"; + return value; +} + +} // NKikimr::NPersQueueTests diff --git a/ydb/library/persqueue/tests/ya.make b/ydb/library/persqueue/tests/ya.make index 0bf5ee3e95..71e5672b84 100644 --- a/ydb/library/persqueue/tests/ya.make +++ b/ydb/library/persqueue/tests/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - -OWNER(g:logbroker) - -SRCS(counters.cpp) - - -PEERDIR( - library/cpp/testing/unittest - library/cpp/http/io - library/cpp/json -) - -END() +LIBRARY() + +OWNER(g:logbroker) + +SRCS(counters.cpp) + + +PEERDIR( + library/cpp/testing/unittest + library/cpp/http/io + library/cpp/json +) + +END() diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index d7f9d872cf..82f6c5812a 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -112,7 +112,7 @@ namespace { } void AddError(const TAstNode& node, const TString& message) { - Expr.AddError(TIssue(node.GetPosition(), message)); + Expr.AddError(TIssue(node.GetPosition(), message)); } TExprNode::TPtr&& ProcessNode(const TAstNode& node, TExprNode::TPtr&& exprNode) { @@ -2639,7 +2639,7 @@ bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprCo } if (!IsUtf8(name)) { - ctx.AddError(TIssue(position, TStringBuilder() << + ctx.AddError(TIssue(position, TStringBuilder() << TString(descr).to_title() << " name must be a valid utf-8 byte sequence: " << TString{name}.Quote())); return false; } @@ -2731,7 +2731,7 @@ bool TStructExprType::Validate(TPosition position, TExprContext& ctx) const { } if (item->GetName() == lastName) { - ctx.AddError(TIssue(position, TStringBuilder() << "Duplicated member: " << lastName)); + ctx.AddError(TIssue(position, TStringBuilder() << "Duplicated member: " << lastName)); return false; } @@ -2748,18 +2748,18 @@ bool TStructExprType::Validate(TPositionHandle position, TExprContext& ctx) cons bool TVariantExprType::Validate(TPosition position, TExprContext& ctx) const { if (UnderlyingType->GetKind() == ETypeAnnotationKind::Tuple) { if (!UnderlyingType->Cast<TTupleExprType>()->GetSize()) { - ctx.AddError(TIssue(position, TStringBuilder() << "Empty tuple is not allowed as underlying type")); + ctx.AddError(TIssue(position, TStringBuilder() << "Empty tuple is not allowed as underlying type")); return false; } } else if (UnderlyingType->GetKind() == ETypeAnnotationKind::Struct) { if (!UnderlyingType->Cast<TStructExprType>()->GetSize()) { - ctx.AddError(TIssue(position, TStringBuilder() << "Empty struct is not allowed as underlying type")); + ctx.AddError(TIssue(position, TStringBuilder() << "Empty struct is not allowed as underlying type")); return false; } } else { - ctx.AddError(TIssue(position, TStringBuilder() << "Expected tuple or struct, but got:" << *UnderlyingType)); + ctx.AddError(TIssue(position, TStringBuilder() << "Expected tuple or struct, but got:" << *UnderlyingType)); return false; } @@ -2810,7 +2810,7 @@ bool TDictExprType::Validate(TPositionHandle position, TExprContext& ctx) const bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { if (OptionalArgumentsCount > Arguments.size()) { - ctx.AddError(TIssue(position, TStringBuilder() << "Too many optional arguments: " << OptionalArgumentsCount + ctx.AddError(TIssue(position, TStringBuilder() << "Too many optional arguments: " << OptionalArgumentsCount << ", function has only " << Arguments.size() << " arguments")); return false; } @@ -2818,7 +2818,7 @@ bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { for (ui32 index = Arguments.size() - OptionalArgumentsCount; index < Arguments.size(); ++index) { auto type = Arguments[index].Type; if (type->GetKind() != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(position, TStringBuilder() << "Expected optional type for argument: " << (index + 1) + ctx.AddError(TIssue(position, TStringBuilder() << "Expected optional type for argument: " << (index + 1) << " because it's an optional argument, but got: " << *type)); return false; } @@ -2830,7 +2830,7 @@ bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { bool hasName = !Arguments[index].Name.empty(); if (startedNames) { if (!hasName) { - ctx.AddError(TIssue(position, TStringBuilder() << "Unexpected positional argument at position " + ctx.AddError(TIssue(position, TStringBuilder() << "Unexpected positional argument at position " << (index + 1) << " just after named arguments")); return false; } @@ -2840,7 +2840,7 @@ bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { if (hasName) { if (!usedNames.insert(Arguments[index].Name).second) { - ctx.AddError(TIssue(position, TStringBuilder() << "Duplication of named argument: " << Arguments[index].Name)); + ctx.AddError(TIssue(position, TStringBuilder() << "Duplication of named argument: " << Arguments[index].Name)); return false; } } @@ -2870,14 +2870,14 @@ TExprContext::TExprContext(ui64 nextUniqueId) [this](TPositionHandle p) { return GetHash(p); }, [this](TPositionHandle a, TPositionHandle b) { return IsEqual(a, b); } ) -{ +{ auto handle = AppendPosition(TPosition()); YQL_ENSURE(handle.Handle == 0); - IssueManager.SetWarningToErrorTreatMessage( - "Treat warning as error mode enabled. " - "To disable it use \"pragma warning(\"default\", <code>);\""); + IssueManager.SetWarningToErrorTreatMessage( + "Treat warning as error mode enabled. " + "To disable it use \"pragma warning(\"default\", <code>);\""); IssueManager.SetIssueCountLimit(100); -} +} TPositionHandle TExprContext::AppendPosition(const TPosition& pos) { YQL_ENSURE(Positions.size() <= Max<ui32>(), "Too many positions"); @@ -3017,7 +3017,7 @@ const TVariantExprType* TMakeTypeImpl<TVariantExprType>::Make(TExprContext& ctx, return AddType<TVariantExprType>(ctx, hash, underlyingType); } -const TErrorExprType* TMakeTypeImpl<TErrorExprType>::Make(TExprContext& ctx, const TIssue& error) { +const TErrorExprType* TMakeTypeImpl<TErrorExprType>::Make(TExprContext& ctx, const TIssue& error) { const auto hash = TErrorExprType::MakeHash(error); TErrorExprType sample(hash, error); if (const auto found = FindType(sample, ctx)) diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index 856ef3ee8a..d6eb544f9a 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -1100,16 +1100,16 @@ class TErrorExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Error; - TErrorExprType(ui64 hash, const TIssue& error) + TErrorExprType(ui64 hash, const TIssue& error) : TTypeAnnotationNode(KindValue, 0, hash) , Error(error) {} - static ui64 MakeHash(const TIssue& error) { + static ui64 MakeHash(const TIssue& error) { return error.Hash(); } - const TIssue& GetError() const { + const TIssue& GetError() const { return Error; } @@ -1118,7 +1118,7 @@ public: } private: - const TIssue Error; + const TIssue Error; }; class TEmptyListExprType : public TTypeAnnotationNode { @@ -2134,7 +2134,7 @@ struct TMakeTypeImpl<TVariantExprType> { template <> struct TMakeTypeImpl<TErrorExprType> { - static const TErrorExprType* Make(TExprContext& ctx, const TIssue& error); + static const TErrorExprType* Make(TExprContext& ctx, const TIssue& error); }; template <> @@ -2233,7 +2233,7 @@ struct TExprContext : private TNonCopyable { TExprContext& Ctx; }; - TIssueManager IssueManager; + TIssueManager IssueManager; TNodeMap<TIssues> AssociativeIssues; TMemoryPool StringPool; @@ -2435,16 +2435,16 @@ struct TExprContext : private TNonCopyable { template <typename T, typename... Args> const T* MakeConstraint(Args&&... args); - void AddError(const TIssue& error) { + void AddError(const TIssue& error) { ENSURE_NOT_FROZEN_CTX - IssueManager.RaiseIssue(error); + IssueManager.RaiseIssue(error); + } + + bool AddWarning(const TIssue& warning) { + ENSURE_NOT_FROZEN_CTX + return IssueManager.RaiseWarning(warning); } - bool AddWarning(const TIssue& warning) { - ENSURE_NOT_FROZEN_CTX - return IssueManager.RaiseWarning(warning); - } - void Freeze(); void UnFreeze(); diff --git a/ydb/library/yql/ast/yql_expr_ut.cpp b/ydb/library/yql/ast/yql_expr_ut.cpp index 3992b64fce..06550f2507 100644 --- a/ydb/library/yql/ast/yql_expr_ut.cpp +++ b/ydb/library/yql/ast/yql_expr_ut.cpp @@ -16,7 +16,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, ui32 typeAnnotationIndex = Max<ui32>()) { const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex); - exprCtx.IssueManager.GetIssues().PrintTo(Cout); + exprCtx.IssueManager.GetIssues().PrintTo(Cout); UNIT_ASSERT(success); UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), typeAnnotationIndex != Max<ui32>() ? TExprNode::EState::TypeComplete : TExprNode::EState::Initial); @@ -24,7 +24,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { static void CompileExprWithCheck(TAstNode& root, TLibraryCohesion& cohesion, TExprContext& exprCtx) { const bool success = CompileExpr(root, cohesion, exprCtx); - exprCtx.IssueManager.GetIssues().PrintTo(Cout); + exprCtx.IssueManager.GetIssues().PrintTo(Cout); UNIT_ASSERT(success); } @@ -34,7 +34,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { TExprContext exprCtx; TExprNode::TPtr exprRoot; bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr); - exprCtx.IssueManager.GetIssues().PrintTo(Cout); + exprCtx.IssueManager.GetIssues().PrintTo(Cout); return result; } @@ -902,7 +902,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) { TExprContext exprCtx2; TExprNode::TPtr exprRoot2; auto compileOk = CompileExpr(*convRes.Root, exprRoot2, exprCtx2, nullptr); - exprCtx2.IssueManager.GetIssues().PrintTo(Cout); + exprCtx2.IssueManager.GetIssues().PrintTo(Cout); UNIT_ASSERT(compileOk); UNIT_ASSERT(exprRoot2); const TExprNode* node = exprRoot.Get(); diff --git a/ydb/library/yql/ast/yql_type_string.cpp b/ydb/library/yql/ast/yql_type_string.cpp index 6cb5b2bfd4..af9af93259 100644 --- a/ydb/library/yql/ast/yql_type_string.cpp +++ b/ydb/library/yql/ast/yql_type_string.cpp @@ -159,10 +159,10 @@ class TTypeParser { public: TTypeParser( - TStringBuf str, TIssues& issues, + TStringBuf str, TIssues& issues, TPosition position, TMemoryPool& pool) : Str(str) - , Issues(issues) + , Issues(issues) , Position(position) , Index(0) , Pool(pool) @@ -1085,7 +1085,7 @@ private: private: TStringBuf Str; - TIssues& Issues; + TIssues& Issues; TPosition TokenBegin, Position; size_t Index; int Token; @@ -1352,10 +1352,10 @@ private: } // namespace -TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, +TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, TPosition position /* = TPosition(1, 1) */) { - TTypeParser parser(str, issues, position, pool); + TTypeParser parser(str, issues, position, pool); return parser.ParseTopLevelType(); } diff --git a/ydb/library/yql/ast/yql_type_string.h b/ydb/library/yql/ast/yql_type_string.h index 546b710cf7..95c1908b9f 100644 --- a/ydb/library/yql/ast/yql_type_string.h +++ b/ydb/library/yql/ast/yql_type_string.h @@ -7,7 +7,7 @@ namespace NYql { class TTypeAnnotationNode; -TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, +TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, TPosition position = {1, 1}); TString FormatType(const TTypeAnnotationNode* typeNode); diff --git a/ydb/library/yql/ast/yql_type_string_ut.cpp b/ydb/library/yql/ast/yql_type_string_ut.cpp index 6eedbd2a76..2f22c42258 100644 --- a/ydb/library/yql/ast/yql_type_string_ut.cpp +++ b/ydb/library/yql/ast/yql_type_string_ut.cpp @@ -10,7 +10,7 @@ Y_UNIT_TEST_SUITE(TTypeString) { void TestFail(const TStringBuf& prog, ui32 column, const TStringBuf& expectedError) { TMemoryPool pool(4096); - TIssues errors; + TIssues errors; auto res = ParseType(prog, pool, errors); UNIT_ASSERT(res == nullptr); UNIT_ASSERT(!errors.Empty()); @@ -21,7 +21,7 @@ Y_UNIT_TEST_SUITE(TTypeString) void TestOk(const TStringBuf& prog, const TStringBuf& expectedType) { TMemoryPool pool(4096); - TIssues errors; + TIssues errors; auto res = ParseType(prog, pool, errors); if (!res) { errors.PrintWithProgramTo(Cerr, "-memory-", TString(prog)); @@ -520,8 +520,8 @@ Y_UNIT_TEST_SUITE(TTypeString) TExprContext ctx; const TTypeAnnotationNode* type = CompileTypeAnnotation(*astRes.Root->GetChild(0), ctx); - if (!type) { - ctx.IssueManager.GetIssues().PrintWithProgramTo(Cerr, "-memory-", yql); + if (!type) { + ctx.IssueManager.GetIssues().PrintWithProgramTo(Cerr, "-memory-", yql); UNIT_FAIL("Can't compile types"); } diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index a5dc2a6812..edfcaa1530 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -31,21 +31,21 @@ ] }, { - "Name": "TCoIndex", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Name", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Type", "Type": "TCoAtom"}, - {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}, - {"Index": 3, "Name": "DataColumns", "Type": "TCoAtomList"} - ] - }, - { - "Name": "TCoIndexList", - "ListBase": "TCoIndex" - }, - { + "Name": "TCoIndex", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Name", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Type", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}, + {"Index": 3, "Name": "DataColumns", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TCoIndexList", + "ListBase": "TCoIndex" + }, + { "Name": "TCoNameValueTupleList", "ListBase": "TCoNameValueTuple" }, @@ -343,16 +343,16 @@ "Match": {"Type": "Callable", "Name": "Collect"} }, { - "Name": "TCoHead", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "Head"} - }, - { + "Name": "TCoHead", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "Head"} + }, + { "Name": "TCoLast", - "Base": "TCoInputBase", + "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "Last"} - }, - { + }, + { "Name": "TCoUnorderedBase", "Base": "TCoInputBase", "Match": {"Type": "CallableBase"} @@ -1888,14 +1888,14 @@ "Children": [ {"Index": 0, "Name": "Input", "Type": "TExprBase"} ] - }, - { - "Name": "TCoSecureParam", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SecureParam"}, - "Children": [ - {"Index": 0, "Name": "Name", "Type": "TCoAtom"} - ] + }, + { + "Name": "TCoSecureParam", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SecureParam"}, + "Children": [ + {"Index": 0, "Name": "Name", "Type": "TCoAtom"} + ] }, { "Name": "TCoEnsure", diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp index 825eaf4b3c..b74abc1c34 100644 --- a/ydb/library/yql/core/facade/yql_facade.cpp +++ b/ydb/library/yql/core/facade/yql_facade.cpp @@ -1363,12 +1363,12 @@ TFuture<void> TProgram::OpenSession(const TString& username) void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan) { TVector<TTransformStage> printTransformers; - const auto issueCode = TIssuesIds::DEFAULT_ERROR; + const auto issueCode = TIssuesIds::DEFAULT_ERROR; if (exprOut) { printTransformers.push_back(TTransformStage( TExprOutputTransformer::Sync(ExprRoot_, exprOut), "ExprOutput", - issueCode)); + issueCode)); } if (planOut) { if (cleanPlan) { @@ -1377,10 +1377,10 @@ void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanP printTransformers.push_back(TTransformStage( TPlanOutputTransformer::Sync(planOut, GetPlanBuilder(), OutputFormat_), "PlanOutput", - issueCode)); + issueCode)); } - auto compositeTransformer = CreateCompositeGraphTransformer(printTransformers, false); + auto compositeTransformer = CreateCompositeGraphTransformer(printTransformers, false); InstantTransform(*compositeTransformer, ExprRoot_, *ExprCtx_); } diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h index 4b80175b18..1f8f47681d 100644 --- a/ydb/library/yql/core/facade/yql_facade.h +++ b/ydb/library/yql/core/facade/yql_facade.h @@ -162,7 +162,7 @@ public: inline TIssues Issues() { if (ExprCtx_) { - return ExprCtx_->IssueManager.GetIssues(); + return ExprCtx_->IssueManager.GetIssues(); } else { return {}; } @@ -180,7 +180,7 @@ public: inline void PrintErrorsTo(IOutputStream& out) const { if (ExprCtx_) { - ExprCtx_->IssueManager.GetIssues().PrintWithProgramTo(out, Filename_, SourceCode_); + ExprCtx_->IssueManager.GetIssues().PrintWithProgramTo(out, Filename_, SourceCode_); } } diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index ed2d457c54..a2ed91d640 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -1,25 +1,25 @@ -package NYql; -option java_package = "ru.yandex.yql.proto"; - +package NYql; +option java_package = "ru.yandex.yql.proto"; + import "ydb/library/yql/public/issue/protos/issue_severity.proto"; - -message TIssuesIds { - - enum EIssueCode { - DEFAULT_ERROR = 0; - UNEXPECTED = 1; + +message TIssuesIds { + + enum EIssueCode { + DEFAULT_ERROR = 0; + UNEXPECTED = 1; INFO = 2; WARNING = 3; SUCCESS = 4; // core codes - CORE = 1000; - CORE_GC = 1010; - CORE_PRE_TYPE_ANN = 1020; - CORE_TYPE_ANN = 1030; - CORE_INTENT = 1040; - CORE_TABLE_METADATA_LOADER = 1050; - CORE_EXEC = 1060; - CORE_OPTIMIZATION = 1070; + CORE = 1000; + CORE_GC = 1010; + CORE_PRE_TYPE_ANN = 1020; + CORE_TYPE_ANN = 1030; + CORE_INTENT = 1040; + CORE_TABLE_METADATA_LOADER = 1050; + CORE_EXEC = 1060; + CORE_OPTIMIZATION = 1070; CORE_PARAM_EVALUATION = 1080; CORE_EXPR_EVALUATION = 1081; CORE_POST_TYPE_ANN = 1090; @@ -43,7 +43,7 @@ message TIssuesIds { CORE_GC_STRINGS_LIMIT_EXCEEDED = 1501; // kikimr - KIKIMR_LOCKS_INVALIDATED = 2001; + KIKIMR_LOCKS_INVALIDATED = 2001; KIKIMR_LOCKS_ACQUIRE_FAILURE = 2002; KIKIMR_SCHEME_ERROR = 2003; KIKIMR_COMPILE_ERROR = 2004; @@ -65,10 +65,10 @@ message TIssuesIds { KIKIMR_OPERATION_CANCELLED = 2021; KIKIMR_UNAUTHENTICATED = 2022; KIKIMR_UNIMPLEMENTED = 2023; - KIKIMR_INDEX_METADATA_LOAD_FAILED = 2024; - KIKIMR_GRPC_CONTEXT_ERROR = 2025; + KIKIMR_INDEX_METADATA_LOAD_FAILED = 2024; + KIKIMR_GRPC_CONTEXT_ERROR = 2025; KIKIMR_OPERATION_STATE_UNKNOWN = 2026; - KIKIMR_INDEX_IS_NOT_READY = 2027; + KIKIMR_INDEX_IS_NOT_READY = 2027; KIKIMR_SCHEME_MISMATCH = 2028; KIKIMR_PRECONDITION_FAILED = 2029; KIKIMR_UNSUPPORTED = 2030; @@ -79,11 +79,11 @@ message TIssuesIds { KIKIMR_READ_MODIFIED_TABLE = 2500; KIKIMR_OPERATION_REVERTED = 2501; KIKIMR_UPDATE_TABLE_WITH_DELETES = 2502; - KIKIMR_WRONG_INDEX_USAGE = 2503; + KIKIMR_WRONG_INDEX_USAGE = 2503; // yt YT_TABLE_PATH_RECORD_FOR_TMP = 3001; - YT_INFER_SCHEMA = 3002; + YT_INFER_SCHEMA = 3002; YT_ACCESS_DENIED = 3003; YT_TABLE_NOT_FOUND = 3004; YT_NATIVE_PRAGMA = 3005; @@ -173,13 +173,13 @@ message TIssuesIds { // range [200000, 399999) reserved for KiKiMR issue codes, do not use! - } - - message TIssueId { - optional EIssueCode code = 2; - optional TSeverityIds.ESeverityId severity = 1; + } + + message TIssueId { + optional EIssueCode code = 2; + optional TSeverityIds.ESeverityId severity = 1; optional string format = 3; - } - - repeated TIssueId ids = 1; -} + } + + repeated TIssueId ids = 1; +} diff --git a/ydb/library/yql/core/issue/protos/ya.make b/ydb/library/yql/core/issue/protos/ya.make index 5c0f2e25bc..1daa53202e 100644 --- a/ydb/library/yql/core/issue/protos/ya.make +++ b/ydb/library/yql/core/issue/protos/ya.make @@ -1,15 +1,15 @@ -PROTO_LIBRARY() - +PROTO_LIBRARY() + OWNER(g:yql g:yql_ydb_core) - -SRCS( - issue_id.proto -) - -PEERDIR( + +SRCS( + issue_id.proto +) + +PEERDIR( ydb/library/yql/public/issue/protos -) - +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/library/yql/core/issue/ut/ya.make b/ydb/library/yql/core/issue/ut/ya.make index cecf94a0e2..46b21a1702 100644 --- a/ydb/library/yql/core/issue/ut/ya.make +++ b/ydb/library/yql/core/issue/ut/ya.make @@ -1,17 +1,17 @@ UNITTEST_FOR(ydb/library/yql/core/issue) - + OWNER(g:yql) - -FORK_SUBTESTS() - -SRCS( - yql_issue_ut.cpp -) - -PEERDIR( -) - + +FORK_SUBTESTS() + +SRCS( + yql_issue_ut.cpp +) + +PEERDIR( +) + TIMEOUT(300) SIZE(MEDIUM) -END() +END() diff --git a/ydb/library/yql/core/issue/ya.make b/ydb/library/yql/core/issue/ya.make index 5013751fc8..1e9bc1ce81 100644 --- a/ydb/library/yql/core/issue/ya.make +++ b/ydb/library/yql/core/issue/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER(g:yql g:yql_ydb_core) - -SRCS( - yql_issue.cpp -) - -PEERDIR( + +SRCS( + yql_issue.cpp +) + +PEERDIR( library/cpp/resource - contrib/libs/protobuf + contrib/libs/protobuf ydb/library/yql/public/issue ydb/library/yql/core/issue/protos -) - -RESOURCE( +) + +RESOURCE( ydb/library/yql/core/issue/yql_issue.txt yql_issue.txt -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/core/issue/yql_issue.cpp b/ydb/library/yql/core/issue/yql_issue.cpp index ddc12a8bc8..3d1ea9b9fe 100644 --- a/ydb/library/yql/core/issue/yql_issue.cpp +++ b/ydb/library/yql/core/issue/yql_issue.cpp @@ -1,11 +1,11 @@ -#include "yql_issue.h" - -namespace NYql { - -const char IssueMapResource[] = "yql_issue.txt"; - -static_assert(DEFAULT_ERROR == TIssuesIds::DEFAULT_ERROR, - "value of particular and common error mismatched for \"DEFAULT_ERROR\""); -static_assert(UNEXPECTED_ERROR == TIssuesIds::UNEXPECTED, - "value of particular and common error mismatched for \"UNEXPECTED_ERROR\""); -} +#include "yql_issue.h" + +namespace NYql { + +const char IssueMapResource[] = "yql_issue.txt"; + +static_assert(DEFAULT_ERROR == TIssuesIds::DEFAULT_ERROR, + "value of particular and common error mismatched for \"DEFAULT_ERROR\""); +static_assert(UNEXPECTED_ERROR == TIssuesIds::UNEXPECTED, + "value of particular and common error mismatched for \"UNEXPECTED_ERROR\""); +} diff --git a/ydb/library/yql/core/issue/yql_issue.h b/ydb/library/yql/core/issue/yql_issue.h index 7ce224f8a5..9ec7ab14bb 100644 --- a/ydb/library/yql/core/issue/yql_issue.h +++ b/ydb/library/yql/core/issue/yql_issue.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/issue/protos/issue_id.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_id.h> - -namespace NYql { - + +namespace NYql { + extern const char IssueMapResource[14]; -using EYqlIssueCode = TIssuesIds::EIssueCode; - -inline ESeverity GetSeverity(EYqlIssueCode id) { - return GetSeverity<TIssuesIds, IssueMapResource>(id); -} - +using EYqlIssueCode = TIssuesIds::EIssueCode; + +inline ESeverity GetSeverity(EYqlIssueCode id) { + return GetSeverity<TIssuesIds, IssueMapResource>(id); +} + inline TString GetMessage(EYqlIssueCode id) { return GetMessage<TIssuesIds, IssueMapResource>(id); } -inline TIssue& SetIssueCode(EYqlIssueCode id, TIssue& issue) { - issue.SetCode(id, GetSeverity(id)); - return issue; -} - -inline TString IssueCodeToString(EYqlIssueCode id) { +inline TIssue& SetIssueCode(EYqlIssueCode id, TIssue& issue) { + issue.SetCode(id, GetSeverity(id)); + return issue; +} + +inline TString IssueCodeToString(EYqlIssueCode id) { const TString& message = GetMessage(id); if (message) { return message; } else { return IssueCodeToString<TIssuesIds>(id); } -} - +} + inline TIssue YqlIssue(const TPosition& position, EYqlIssueCode id, const TString& message) { TIssue issue(position, message); SetIssueCode(id, issue); return issue; -} +} inline TIssue YqlIssue(const TPosition& position, EYqlIssueCode id) { return YqlIssue(position, id, IssueCodeToString(id)); diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index beb8ceb926..897d5d6f9a 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -1,12 +1,12 @@ -ids { - code: UNEXPECTED - severity: S_FATAL -} -ids { - code: DEFAULT_ERROR - severity: S_ERROR -} -ids { +ids { + code: UNEXPECTED + severity: S_FATAL +} +ids { + code: DEFAULT_ERROR + severity: S_ERROR +} +ids { code: INFO severity: S_INFO } @@ -19,49 +19,49 @@ ids { severity: S_WARNING } ids { - code: CORE + code: CORE severity: S_INFO -} -ids { - code: CORE_GC +} +ids { + code: CORE_GC severity: S_INFO -} -ids { - code: CORE_PRE_TYPE_ANN +} +ids { + code: CORE_PRE_TYPE_ANN severity: S_INFO format: "Pre type annotation" -} -ids { - code: CORE_TYPE_ANN +} +ids { + code: CORE_TYPE_ANN severity: S_INFO format: "Type annotation" -} -ids { +} +ids { code: CORE_POST_TYPE_ANN severity: S_INFO format: "Post type annotation" } ids { - code: CORE_INTENT + code: CORE_INTENT severity: S_INFO format: "Table intent determination" -} -ids { - code: CORE_TABLE_METADATA_LOADER +} +ids { + code: CORE_TABLE_METADATA_LOADER severity: S_INFO format: "Table metadata loading" -} -ids { - code: CORE_EXEC +} +ids { + code: CORE_EXEC severity: S_INFO format: "Execution" -} -ids { - code: CORE_OPTIMIZATION +} +ids { + code: CORE_OPTIMIZATION severity: S_INFO format: "Optimization" -} -ids { +} +ids { code: CORE_PARAM_EVALUATION severity: S_INFO format: "Parameters evaluation" @@ -128,10 +128,10 @@ ids { severity: S_ERROR } ids { - code: KIKIMR_LOCKS_INVALIDATED - severity: S_ERROR + code: KIKIMR_LOCKS_INVALIDATED + severity: S_ERROR format: "Transaction locks invalidated." -} +} ids { code: KIKIMR_LOCKS_ACQUIRE_FAILURE severity: S_ERROR @@ -250,28 +250,28 @@ ids { severity: S_WARNING } ids { - code: KIKIMR_WRONG_INDEX_USAGE - severity: S_WARNING -} -ids { - code: KIKIMR_GRPC_CONTEXT_ERROR - severity: S_ERROR - format: "Operation with grpc context failed" -} -ids { + code: KIKIMR_WRONG_INDEX_USAGE + severity: S_WARNING +} +ids { + code: KIKIMR_GRPC_CONTEXT_ERROR + severity: S_ERROR + format: "Operation with grpc context failed" +} +ids { code: KIKIMR_OPERATION_STATE_UNKNOWN severity: S_ERROR format: "State of operation is unknown." } ids { - code: KIKIMR_INDEX_IS_NOT_READY - severity: S_ERROR -} -ids { - code: KIKIMR_INDEX_METADATA_LOAD_FAILED - severity: S_ERROR -} -ids { + code: KIKIMR_INDEX_IS_NOT_READY + severity: S_ERROR +} +ids { + code: KIKIMR_INDEX_METADATA_LOAD_FAILED + severity: S_ERROR +} +ids { code: KIKIMR_SCHEME_MISMATCH severity: S_ERROR format: "Scheme mismatch found while executing query." @@ -443,7 +443,7 @@ ids { ids { code: YQL_NOT_ALLOWED_IN_DISCOVERY severity: S_ERROR -} +} ids { code: JSONPATH_PARSE_ERROR severity: S_ERROR diff --git a/ydb/library/yql/core/issue/yql_issue_ut.cpp b/ydb/library/yql/core/issue/yql_issue_ut.cpp index 62a86e85a1..fa8f97e63a 100644 --- a/ydb/library/yql/core/issue/yql_issue_ut.cpp +++ b/ydb/library/yql/core/issue/yql_issue_ut.cpp @@ -1,30 +1,30 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "yql_issue.h" - -using namespace NYql; - + +#include "yql_issue.h" + +using namespace NYql; + Y_UNIT_TEST_SUITE(TIssuesDataTest) { Y_UNIT_TEST(SeverityMapTest) { - auto severity = GetSeverity(TIssuesIds::UNEXPECTED); - auto severityName = SeverityToString(severity); - UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(severity), - static_cast<ui32>(TSeverityIds::S_FATAL)); + auto severity = GetSeverity(TIssuesIds::UNEXPECTED); + auto severityName = SeverityToString(severity); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(severity), + static_cast<ui32>(TSeverityIds::S_FATAL)); UNIT_ASSERT_VALUES_EQUAL(severityName, "Fatal"); - } - + } + Y_UNIT_TEST(UnknownSeverityNameTest) { - auto severityName = SeverityToString(static_cast<ESeverity>(999)); + auto severityName = SeverityToString(static_cast<ESeverity>(999)); UNIT_ASSERT_VALUES_EQUAL(severityName, "Unknown"); - } - + } + Y_UNIT_TEST(IssueNameTest) { - auto issueName = IssueCodeToString(TIssuesIds::DEFAULT_ERROR); + auto issueName = IssueCodeToString(TIssuesIds::DEFAULT_ERROR); UNIT_ASSERT_VALUES_EQUAL(issueName, "Default error"); - } - + } + Y_UNIT_TEST(IssueDefaultErrorNumber) { - UNIT_ASSERT_VALUES_EQUAL((ui32)TIssuesIds::DEFAULT_ERROR, 0); - } - -} + UNIT_ASSERT_VALUES_EQUAL((ui32)TIssuesIds::DEFAULT_ERROR, 0); + } + +} diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.cpp b/ydb/library/yql/core/services/yql_transform_pipeline.cpp index 031b8c76bf..ae425f34c8 100644 --- a/ydb/library/yql/core/services/yql_transform_pipeline.cpp +++ b/ydb/library/yql/core/services/yql_transform_pipeline.cpp @@ -68,7 +68,7 @@ TTransformationPipeline& TTransformationPipeline::AddExpressionEvaluation(const TTransformationPipeline& TTransformationPipeline::AddPreTypeAnnotation(EYqlIssueCode issueCode) { auto& typeCtx = *TypeAnnotationContext_; Transformers_.push_back(TTransformStage(CreateFunctorTransformer(&ExpandApply), "ExpandApply", - issueCode)); + issueCode)); Transformers_.push_back(TTransformStage(CreateFunctorTransformer( [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { return ValidateProviders(input, output, ctx, typeCtx); @@ -142,7 +142,7 @@ TTransformationPipeline& TTransformationPipeline::AddCommonOptimization(EYqlIssu Transformers_.push_back(TTransformStage( CreateCommonOptTransformer(TypeAnnotationContext_.Get()), "CommonOptimization", - issueCode)); + issueCode)); return *this; } @@ -161,20 +161,20 @@ TTransformationPipeline& TTransformationPipeline::AddOptimization(bool checkWorl "RecaptureDataProposals", issueCode)); Transformers_.push_back(TTransformStage( - CreateLogicalDataProposalsInspector(*TypeAnnotationContext_), + CreateLogicalDataProposalsInspector(*TypeAnnotationContext_), "LogicalDataProposals", - issueCode)); + issueCode)); Transformers_.push_back(TTransformStage( - CreatePhysicalDataProposalsInspector(*TypeAnnotationContext_), + CreatePhysicalDataProposalsInspector(*TypeAnnotationContext_), "PhysicalDataProposals", - issueCode)); + issueCode)); Transformers_.push_back(TTransformStage( - CreatePhysicalFinalizers(*TypeAnnotationContext_), + CreatePhysicalFinalizers(*TypeAnnotationContext_), "PhysicalFinalizers", - issueCode)); - if (withFinalOptimization) { + issueCode)); + if (withFinalOptimization) { AddFinalCommonOptimization(issueCode); - } + } AddCheckExecution(checkWorld, issueCode); return *this; } @@ -183,7 +183,7 @@ TTransformationPipeline& TTransformationPipeline::AddCheckExecution(bool checkWo Transformers_.push_back(TTransformStage( CreateCheckExecutionTransformer(*TypeAnnotationContext_, checkWorld), "CheckExecution", - issueCode)); + issueCode)); return *this; } @@ -191,23 +191,23 @@ TTransformationPipeline& TTransformationPipeline::AddRun(TOperationProgressWrite Transformers_.push_back(TTransformStage( CreateExecutionTransformer(*TypeAnnotationContext_, writer), "Execution", - issueCode)); + issueCode)); return *this; } TTransformationPipeline& TTransformationPipeline::AddIntentDeterminationTransformer(EYqlIssueCode issueCode) { Transformers_.push_back(TTransformStage( - CreateIntentDeterminationTransformer(*TypeAnnotationContext_), + CreateIntentDeterminationTransformer(*TypeAnnotationContext_), "IntentDetermination", - issueCode)); + issueCode)); return *this; } TTransformationPipeline& TTransformationPipeline::AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode) { Transformers_.push_back(TTransformStage( - CreateTableMetadataLoader(*TypeAnnotationContext_), + CreateTableMetadataLoader(*TypeAnnotationContext_), "TableMetadataLoader", - issueCode)); + issueCode)); return *this; } @@ -215,9 +215,9 @@ TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer( TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode) { Transformers_.push_back(TTransformStage( - CreateTypeAnnotationTransformer(callableTransformer, *TypeAnnotationContext_), + CreateTypeAnnotationTransformer(callableTransformer, *TypeAnnotationContext_), "TypeAnnotation", - issueCode)); + issueCode)); return *this; } @@ -227,8 +227,8 @@ TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(E return AddTypeAnnotationTransformer(callableTransformer, issueCode); } -TAutoPtr<IGraphTransformer> TTransformationPipeline::Build(bool useIssueScopes) { - return CreateCompositeGraphTransformer(Transformers_, useIssueScopes); +TAutoPtr<IGraphTransformer> TTransformationPipeline::Build(bool useIssueScopes) { + return CreateCompositeGraphTransformer(Transformers_, useIssueScopes); } TAutoPtr<IGraphTransformer> TTransformationPipeline::BuildWithNoArgChecks(bool useIssueScopes) { diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.h b/ydb/library/yql/core/services/yql_transform_pipeline.h index 1da39b3c33..7e2f5c911e 100644 --- a/ydb/library/yql/core/services/yql_transform_pipeline.h +++ b/ydb/library/yql/core/services/yql_transform_pipeline.h @@ -47,7 +47,7 @@ public: TTransformationPipeline& Add(IGraphTransformer& transformer, const TString& stageName, EYqlIssueCode issueCode = TIssuesIds::DEFAULT_ERROR, const TString& issueMessage = {}); - TAutoPtr<IGraphTransformer> Build(bool useIssueScopes = true); + TAutoPtr<IGraphTransformer> Build(bool useIssueScopes = true); TAutoPtr<IGraphTransformer> BuildWithNoArgChecks(bool useIssueScopes = true); TIntrusivePtr<TTypeAnnotationContext> GetTypeAnnotationContext() const; diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 216d1c9d38..5846e6cb10 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -593,10 +593,10 @@ namespace NTypeAnnImpl { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - + if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } TMaybe<TString> textValue; if (input->Content() == "Bool") { @@ -11496,41 +11496,41 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus SecureParamWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TExtContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus SecureParamWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TExtContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + auto tokenName = input->Head().Content(); - auto separator = tokenName.find(":"); - - if (separator == TString::npos || separator == tokenName.size()) { + auto separator = tokenName.find(":"); + + if (separator == TString::npos || separator == tokenName.size()) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "malformed secure param: " << tokenName)); - return IGraphTransformer::TStatus::Error; - } - - const auto p0 = tokenName.substr(0, separator); - if (p0 == "api") { - const auto p1 = tokenName.substr(separator + 1); - if (p1 != "oauth" && p1 != "cookie") { + return IGraphTransformer::TStatus::Error; + } + + const auto p0 = tokenName.substr(0, separator); + if (p0 == "api") { + const auto p1 = tokenName.substr(separator + 1); + if (p1 != "oauth" && p1 != "cookie") { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "unknown token: " << p1 << ", prefix: " << p0)); - return IGraphTransformer::TStatus::Error; - } - if (p1 == "oauth" && ctx.Types.UserCredentials.OauthToken.empty()) { + return IGraphTransformer::TStatus::Error; + } + if (p1 == "oauth" && ctx.Types.UserCredentials.OauthToken.empty()) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "got empty Oauth token string")); - return IGraphTransformer::TStatus::Error; - } - if (p1 == "cookie" && ctx.Types.UserCredentials.BlackboxSessionIdCookie.empty()) { + return IGraphTransformer::TStatus::Error; + } + if (p1 == "cookie" && ctx.Types.UserCredentials.BlackboxSessionIdCookie.empty()) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "got empty session cookie")); - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } } else if (p0 == "token" || p0 == "cluster") { - const auto p1 = tokenName.substr(separator + 1); - auto cred = ctx.Types.FindCredential(p1); + const auto p1 = tokenName.substr(separator + 1); + auto cred = ctx.Types.FindCredential(p1); TMaybe<TCredential> clusterCred; if (cred == nullptr && p0 == "cluster") { if (p1.StartsWith("default_")) { @@ -11558,24 +11558,24 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } } - if (cred == nullptr) { + if (cred == nullptr) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "unknown token id: " << p1 << ", prefix: " << p0)); - return IGraphTransformer::TStatus::Error; - } - if (cred->Content.empty()) { + return IGraphTransformer::TStatus::Error; + } + if (cred->Content.empty()) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "got empty credential content for id: " << p1)); - return IGraphTransformer::TStatus::Error; - } - } else { + return IGraphTransformer::TStatus::Error; + } + } else { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "unknown token prefix: " << p0)); - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)); - - return IGraphTransformer::TStatus::Ok; - } - + + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus MuxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -13232,7 +13232,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> ExtFunctions["CurrentOperationId"] = &CurrentOperationIdWrapper; ExtFunctions["CurrentOperationSharedId"] = &CurrentOperationSharedIdWrapper; ExtFunctions["CurrentAuthenticatedUser"] = &CurrentAuthenticatedUserWrapper; - ExtFunctions["SecureParam"] = &SecureParamWrapper; + ExtFunctions["SecureParam"] = &SecureParamWrapper; ExtFunctions["UnsafeTimestampCast"] = &UnsafeTimestampCastWrapper; ExtFunctions["JsonValue"] = &JsonValueWrapper; ExtFunctions["JsonExists"] = &JsonExistsWrapper; diff --git a/ydb/library/yql/core/type_ann/type_ann_expr.cpp b/ydb/library/yql/core/type_ann/type_ann_expr.cpp index 00c57251c9..f67d3b4120 100644 --- a/ydb/library/yql/core/type_ann/type_ann_expr.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_expr.cpp @@ -579,23 +579,23 @@ TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer( transformers.push_back(TTransformStage( CreateFunctorTransformer(&ExpandApply), "ExpandApply", - issueCode)); + issueCode)); transformers.push_back(TTransformStage( CreateFunctorTransformer( - [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { return ValidateProviders(input, output, ctx, typeAnnotationContext); - }), + }), "ValidateProviders", - issueCode)); + issueCode)); transformers.push_back(TTransformStage( - CreateConfigureTransformer(typeAnnotationContext), + CreateConfigureTransformer(typeAnnotationContext), "Configure", - issueCode)); + issueCode)); transformers.push_back(TTransformStage( - CreateIODiscoveryTransformer(typeAnnotationContext), + CreateIODiscoveryTransformer(typeAnnotationContext), "IODiscovery", - issueCode)); + issueCode)); transformers.push_back(TTransformStage( CreateEpochsTransformer(typeAnnotationContext), "Epochs", @@ -604,35 +604,35 @@ TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer( transformers.push_back(TTransformStage( CreateIntentDeterminationTransformer(typeAnnotationContext), "IntentDetermination", - issueCode)); + issueCode)); transformers.push_back(TTransformStage( - CreateTableMetadataLoader(typeAnnotationContext), + CreateTableMetadataLoader(typeAnnotationContext), "TableMetadataLoader", - issueCode)); + issueCode)); auto& typeCtx = typeAnnotationContext; transformers.push_back(TTransformStage( CreateFunctorTransformer( - [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - return RewriteIO(input, output, typeCtx, ctx); - }), + [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + return RewriteIO(input, output, typeCtx, ctx); + }), "RewriteIO", - issueCode)); + issueCode)); issueCode = TIssuesIds::CORE_TYPE_ANN; auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(typeAnnotationContext); auto typeTransformer = CreateTypeAnnotationTransformer(callableTransformer, typeAnnotationContext); transformers.push_back(TTransformStage( - typeTransformer, + typeTransformer, "TypeAnnotation", - issueCode)); + issueCode)); if (wholeProgram) { transformers.push_back(TTransformStage( CreateFunctorTransformer(&CheckWholeProgramType), "CheckWholeProgramType", - issueCode)); + issueCode)); } - return CreateCompositeGraphTransformer(transformers, true); + return CreateCompositeGraphTransformer(transformers, true); } bool SyncAnnotateTypes( diff --git a/ydb/library/yql/core/type_ann/type_ann_join.cpp b/ydb/library/yql/core/type_ann/type_ann_join.cpp index c71ebbb857..18516a8ab8 100644 --- a/ydb/library/yql/core/type_ann/type_ann_join.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_join.cpp @@ -262,7 +262,7 @@ namespace NTypeAnnImpl { const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType(); if (itemType->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue( + ctx.Expr.AddError(TIssue( ctx.Expr.GetPosition(list.Pos()), TStringBuilder() << "Expected list of struct" )); @@ -288,7 +288,7 @@ namespace NTypeAnnImpl { } if (auto err = labels.Add(ctx.Expr, *listPair.Child(1), structType)) { ctx.Expr.AddError(*err); - ctx.Expr.AddError(TIssue( + ctx.Expr.AddError(TIssue( ctx.Expr.GetPosition(input->Child(idx)->Pos()), TStringBuilder() << "Failed to parse labels of struct as second element of " << idx << " argument" )); diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp index 3eed46784c..b145cde389 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp @@ -2275,7 +2275,7 @@ namespace { continue; } - auto err = TIssue( + auto err = TIssue( ctx.Expr.GetPosition(pos), TStringBuilder() << "Uncompatible member " << item->GetName() << " types: " diff --git a/ydb/library/yql/core/type_ann/type_ann_types.cpp b/ydb/library/yql/core/type_ann/type_ann_types.cpp index b16f23b565..c27c0c433f 100644 --- a/ydb/library/yql/core/type_ann/type_ann_types.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_types.cpp @@ -882,10 +882,10 @@ namespace NTypeAnnImpl { } TMemoryPool pool(4096); - TIssues issues; + TIssues issues; auto parsedType = ParseType(input->Child(0)->Content(), pool, issues, ctx.Expr.GetPosition(input->Child(0)->Pos())); if (!parsedType) { - ctx.Expr.IssueManager.AddIssues(issues); + ctx.Expr.IssueManager.AddIssues(issues); return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/yql_execution.cpp b/ydb/library/yql/core/yql_execution.cpp index 1f0ef97440..eb138b1b4f 100644 --- a/ydb/library/yql/core/yql_execution.cpp +++ b/ydb/library/yql/core/yql_execution.cpp @@ -801,9 +801,9 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio } else if (node->IsCallable("TablePath") || node->IsCallable("TableRecord")) { auto issue = TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << node->Content() << " will be empty"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_FREE_TABLE_PATH_RECORD, issue); - if (!ctx.AddWarning(issue)) { - hasErrors = true; - } + if (!ctx.AddWarning(issue)) { + hasErrors = true; + } } else if (node->IsCallable("UnsafeTimestampCast")) { auto issue = TIssue(ctx.GetPosition(node->Pos()), "Unsafe conversion integral value to Timestamp, consider using date types"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_CAST_INTEGRAL_TO_TIMESTAMP_UNSAFE, issue); diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index 5b9fd0ec40..f2b793af8d 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -281,10 +281,10 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& if (node->IsCallable({"String", "Utf8"}) && to == EDataSlot::Json) { if (const auto atom = node->Head().Content(); NDom::IsValidJson(atom)) { node = ctx.RenameNode(*node, "Json"); - return IGraphTransformer::TStatus::Repeat; - } - } - + return IGraphTransformer::TStatus::Repeat; + } + } + bool allow = false; bool isSafe = true; bool useCast = false; @@ -4465,10 +4465,10 @@ bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprC return true; } -TMaybe<TIssue> NormalizeName(TPosition position, TString& name) { +TMaybe<TIssue> NormalizeName(TPosition position, TString& name) { const ui32 inputLength = name.length(); - ui32 startCharPos = 0; - ui32 totalSkipped = 0; + ui32 startCharPos = 0; + ui32 totalSkipped = 0; bool atStart = true; bool justSkippedUnderscore = false; for (ui32 i = 0; i < inputLength; ++i) { @@ -4476,45 +4476,45 @@ TMaybe<TIssue> NormalizeName(TPosition position, TString& name) { if (c == '_') { if (!atStart) { if (justSkippedUnderscore) { - return TIssue(position, TStringBuilder() << "\"" << name << "\" looks weird, has multiple consecutive underscores"); + return TIssue(position, TStringBuilder() << "\"" << name << "\" looks weird, has multiple consecutive underscores"); } justSkippedUnderscore = true; ++totalSkipped; continue; - } else { - ++startCharPos; + } else { + ++startCharPos; } } else { atStart = false; justSkippedUnderscore = false; } - } + } if (totalSkipped >= 5) { - return TIssue(position, TStringBuilder() << "\"" << name << "\" looks weird, has multiple consecutive underscores"); - } - - ui32 outPos = startCharPos; - for (ui32 i = startCharPos; i < inputLength; i++) { - const char c = name.at(i); - if (c == '_') { - continue; - } else { - name[outPos] = AsciiToLower(c); - ++outPos; - } - } - - name.resize(outPos); - Y_VERIFY(inputLength - outPos == totalSkipped); - + return TIssue(position, TStringBuilder() << "\"" << name << "\" looks weird, has multiple consecutive underscores"); + } + + ui32 outPos = startCharPos; + for (ui32 i = startCharPos; i < inputLength; i++) { + const char c = name.at(i); + if (c == '_') { + continue; + } else { + name[outPos] = AsciiToLower(c); + ++outPos; + } + } + + name.resize(outPos); + Y_VERIFY(inputLength - outPos == totalSkipped); + return Nothing(); } TString NormalizeName(const TStringBuf& name) { TString result(name); - TMaybe<TIssue> error = NormalizeName(TPosition(), result); + TMaybe<TIssue> error = NormalizeName(TPosition(), result); YQL_ENSURE(error.Empty(), "" << error->Message); return result; } diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h index d1be880eb6..a865aa27cd 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.h +++ b/ydb/library/yql/core/yql_expr_type_annotation.h @@ -253,7 +253,7 @@ bool IsDataTypeString(EDataSlot dataSlot); bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); -TMaybe<TIssue> NormalizeName(TPosition position, TString& name); +TMaybe<TIssue> NormalizeName(TPosition position, TString& name); TString NormalizeName(const TStringBuf& name); bool HasError(const TTypeAnnotationNode* type, TExprContext& ctx); diff --git a/ydb/library/yql/core/yql_graph_transformer.cpp b/ydb/library/yql/core/yql_graph_transformer.cpp index ab76d505da..26193c4c40 100644 --- a/ydb/library/yql/core/yql_graph_transformer.cpp +++ b/ydb/library/yql/core/yql_graph_transformer.cpp @@ -10,16 +10,16 @@ namespace { class TCompositeGraphTransformer : public TGraphTransformerBase { public: TCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes, bool doCheckArguments) - : Stages(stages) - , UseIssueScopes(useIssueScopes) + : Stages(stages) + , UseIssueScopes(useIssueScopes) , DoCheckArguments(doCheckArguments) - { - if (UseIssueScopes) { - for (const auto& stage : Stages) { - YQL_ENSURE(!stage.Name.empty()); - } - } - } + { + if (UseIssueScopes) { + for (const auto& stage : Stages) { + YQL_ENSURE(!stage.Name.empty()); + } + } + } void Rewind() override { for (auto& stage : Stages) { @@ -42,16 +42,16 @@ public: #endif if (Index >= Stages.size()) { - if (LastIssueScope) { - ctx.IssueManager.LeaveScope(); - LastIssueScope.Clear(); - } + if (LastIssueScope) { + ctx.IssueManager.LeaveScope(); + LastIssueScope.Clear(); + } return TStatus::Ok; } - if (UseIssueScopes) { - UpdateIssueScope(ctx.IssueManager); - } + if (UseIssueScopes) { + UpdateIssueScope(ctx.IssueManager); + } auto status = Stages[Index].GetTransformer().Transform(input, output, ctx); #ifndef NDEBUG if (DoCheckArguments && output && output != input) { @@ -116,29 +116,29 @@ private: return status; } - void UpdateIssueScope(TIssueManager& issueManager) { + void UpdateIssueScope(TIssueManager& issueManager) { YQL_ENSURE(Index < Stages.size()); const auto scopeIssueCode = Stages[Index].IssueCode; const auto scopeIssueMessage = Stages[Index].IssueMessage; - if (LastIssueScope != scopeIssueCode) { - if (!LastIssueScope.Empty()) { - issueManager.LeaveScope(); - } + if (LastIssueScope != scopeIssueCode) { + if (!LastIssueScope.Empty()) { + issueManager.LeaveScope(); + } issueManager.AddScope([scopeIssueCode, scopeIssueMessage]() { auto issue = new TIssue(TPosition(), scopeIssueMessage ? scopeIssueMessage : IssueCodeToString(scopeIssueCode)); - issue->SetCode(scopeIssueCode, GetSeverity(scopeIssueCode)); - return issue; - }); - LastIssueScope = scopeIssueCode; - } - } - + issue->SetCode(scopeIssueCode, GetSeverity(scopeIssueCode)); + return issue; + }); + LastIssueScope = scopeIssueCode; + } + } + protected: TVector<TTransformStage> Stages; - const bool UseIssueScopes; + const bool UseIssueScopes; const bool DoCheckArguments; size_t Index = 0; - TMaybe<EYqlIssueCode> LastIssueScope; + TMaybe<EYqlIssueCode> LastIssueScope; ui64 CheckArgumentsCount = 0; }; @@ -273,8 +273,8 @@ IGraphTransformer::TStatus SyncTransform(IGraphTransformer& transformer, TExprNo } AddTooManyTransformationsError(root->Pos(), "SyncTransform", ctx); } - catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e)); + catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e)); } return IGraphTransformer::TStatus::Error; } @@ -307,8 +307,8 @@ IGraphTransformer::TStatus InstantTransform(IGraphTransformer& transformer, TExp } AddTooManyTransformationsError(root->Pos(), "InstantTransform", ctx); } - catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e)); + catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e)); } return IGraphTransformer::TStatus::Error; } @@ -364,8 +364,8 @@ NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransform(IGraphTransformer return NThreading::MakeFuture(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error)); } } - catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e)); + catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e)); return NThreading::MakeFuture(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error)); } diff --git a/ydb/library/yql/core/yql_join.cpp b/ydb/library/yql/core/yql_join.cpp index 674146588f..9a9ded7a32 100644 --- a/ydb/library/yql/core/yql_join.cpp +++ b/ydb/library/yql/core/yql_join.cpp @@ -549,7 +549,7 @@ TMaybe<TIssue> TJoinLabels::Add(TExprContext& ctx, TExprNode& node, const TStruc for (auto& table : label.Tables) { if (!InputByTable.insert({ table, index }).second) { - return TIssue( + return TIssue( ctx.GetPosition(node.Pos()), TStringBuilder() << "Duplication of table name " << table); } diff --git a/ydb/library/yql/core/yql_library_compiler.cpp b/ydb/library/yql/core/yql_library_compiler.cpp index eaefa15fd5..79296792e6 100644 --- a/ydb/library/yql/core/yql_library_compiler.cpp +++ b/ydb/library/yql/core/yql_library_compiler.cpp @@ -92,11 +92,11 @@ bool CompileLibrary(const TString& alias, const TString& script, TExprContext& c const auto& res = ParseAst(script, nullptr, alias); if (!res.IsOk()) { for (const auto& originalError : res.Issues) { - TIssue error(originalError); + TIssue error(originalError); TStringBuilder message; message << error.Message << " (at " << alias << ")"; error.Message = message; - ctx.AddError(error); + ctx.AddError(error); } return false; } @@ -134,7 +134,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, if (import.second.first == lib.first) { ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), - TStringBuilder() << "Library '" << lib.first << "' tries to import itself.")); + TStringBuilder() << "Library '" << lib.first << "' tries to import itself.")); return false; } @@ -149,7 +149,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, if (!exportTable) { ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), - TStringBuilder() << "Library '" << lib.first << "' has unresolved dependency from '" << import.second.first << "'.")); + TStringBuilder() << "Library '" << lib.first << "' has unresolved dependency from '" << import.second.first << "'.")); return false; } @@ -157,7 +157,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, replaces[import.first] = externalModule ? ctxToClone.DeepCopy(*ex->second, exportTable->ExprCtx(), clones, true, false) : ex->second; } else { ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), - TStringBuilder() << "Library '" << lib.first << "' has unresolved symbol '" << import.second.second << "' from '" << import.second.first << "'.")); + TStringBuilder() << "Library '" << lib.first << "' has unresolved symbol '" << import.second.second << "' from '" << import.second.first << "'.")); return false; } } @@ -178,7 +178,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, for (const auto& expo : lib.second.Exports.Symbols()) { if (!ReplaceNodes(*expo.second, replaces, hasChanges)) { ctx.AddError(TIssue(ctxToClone.GetPosition(expo.second->Pos()), - TStringBuilder() << "Cross reference detected under '" << expo.first << "' in '" << lib.first << "'.")); + TStringBuilder() << "Cross reference detected under '" << expo.first << "' in '" << lib.first << "'.")); return false; } } diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h index 6bc6748e40..84eeda2edb 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h @@ -207,17 +207,17 @@ struct TDqExecutionSettings { const TDqExecutionSettings& GetDqExecutionSettings(); TDqExecutionSettings& GetDqExecutionSettingsForTests(); -struct TReportStatsSettings { - // Min interval between stats messages. - TDuration MinInterval; - // Max interval to sent stats in case of no activity. - TDuration MaxInterval; -}; - +struct TReportStatsSettings { + // Min interval between stats messages. + TDuration MinInterval; + // Max interval to sent stats in case of no activity. + TDuration MaxInterval; +}; + struct TComputeRuntimeSettings { TMaybe<TDuration> Timeout; NDqProto::EDqStatsMode StatsMode = NDqProto::DQ_STATS_MODE_NONE; - TMaybe<TReportStatsSettings> ReportStatsSettings; + TMaybe<TReportStatsSettings> ReportStatsSettings; // see kqp_rm.h // 0 - disable extra memory allocation @@ -230,7 +230,7 @@ struct TComputeRuntimeSettings { bool UseSpilling = false; std::function<void(bool success, const TString& reason)> TerminateHandler; - TMaybe<NDqProto::TRlPath> RlPath; + TMaybe<NDqProto::TRlPath> RlPath; }; using TAllocateMemoryCallback = std::function<bool(const TTxId& txId, ui64 taskId, ui64 memory)>; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index 677727f1ea..deaadefb42 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -63,12 +63,12 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived> , public IDqSourceActor::ICallbacks , public IDqSinkActor::ICallbacks { -protected: +protected: enum EEvWakeupTag : ui64 { TimeoutTag = 1, - PeriodicStatsTag = 2, - RlSendAllowedTag = 101, - RlNoResourceTag = 102, + PeriodicStatsTag = 2, + RlSendAllowedTag = 101, + RlNoResourceTag = 102, }; public: @@ -85,13 +85,13 @@ public: this->Schedule(*RuntimeSettings.Timeout, new NActors::TEvents::TEvWakeup(EEvWakeupTag::TimeoutTag)); } - if (auto reportStatsSettings = RuntimeSettings.ReportStatsSettings) { - if (reportStatsSettings->MaxInterval) { - CA_LOG_D("Set periodic stats " << reportStatsSettings->MaxInterval); + if (auto reportStatsSettings = RuntimeSettings.ReportStatsSettings) { + if (reportStatsSettings->MaxInterval) { + CA_LOG_D("Set periodic stats " << reportStatsSettings->MaxInterval); this->Schedule(reportStatsSettings->MaxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); - } - } - + } + } + if (SayHelloOnBootstrap()) { // say "Hello" to executer auto ev = MakeHolder<TEvDqCompute::TEvState>(); @@ -245,7 +245,7 @@ protected: } auto now = TInstant::Now(); - + if (Y_UNLIKELY(ProfileStats)) { ProfileStats->MkqlMaxUsedMemory = std::max(ProfileStats->MkqlMaxUsedMemory, alloc->GetPeakAllocated()); CA_LOG_D("Peak memory usage: " << ProfileStats->MkqlMaxUsedMemory); @@ -827,8 +827,8 @@ protected: } void HandleExecuteBase(NActors::TEvents::TEvWakeup::TPtr& ev) { - auto tag = (EEvWakeupTag) ev->Get()->Tag; - switch (tag) { + auto tag = (EEvWakeupTag) ev->Get()->Tag; + switch (tag) { case EEvWakeupTag::TimeoutTag: { auto abortEv = MakeHolder<TEvDq::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, TStringBuilder() << "Timeout event from compute actor " << this->SelfId() @@ -838,27 +838,27 @@ protected: TerminateSources("timeout exceeded", false); Terminate(false, "timeout exceeded"); - break; + break; } - case EEvWakeupTag::PeriodicStatsTag: { - const auto maxInterval = RuntimeSettings.ReportStatsSettings->MaxInterval; + case EEvWakeupTag::PeriodicStatsTag: { + const auto maxInterval = RuntimeSettings.ReportStatsSettings->MaxInterval; this->Schedule(maxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); - + auto now = NActors::TActivationContext::Now(); if (now - LastSendStatsTime >= maxInterval) { ReportStats(now); - } - break; - } - default: - static_cast<TDerived*>(this)->HandleEvWakeup(tag); + } + break; + } + default: + static_cast<TDerived*>(this)->HandleEvWakeup(tag); } } - void HandleEvWakeup(EEvWakeupTag tag) { - CA_LOG_E("Unhandled wakeup tag " << (ui64)tag); - } - + void HandleEvWakeup(EEvWakeupTag tag) { + CA_LOG_E("Unhandled wakeup tag " << (ui64)tag); + } + void HandleExecuteBase(NActors::TEvents::TEvUndelivered::TPtr& ev) { ui32 lostEventType = ev->Get()->SourceType; switch (lostEventType) { @@ -1099,10 +1099,10 @@ private: } protected: - const TMaybe<NDqProto::TRlPath>& GetRlPath() const { - return RuntimeSettings.RlPath; - } - + const TMaybe<NDqProto::TRlPath>& GetRlPath() const { + return RuntimeSettings.RlPath; + } + TTxId GetTxId() const { return TxId; } @@ -1424,37 +1424,37 @@ private: void ReportStats(TInstant now) { if (!RuntimeSettings.ReportStatsSettings) { - return; + return; } - + if (now - LastSendStatsTime < RuntimeSettings.ReportStatsSettings->MinInterval) { - return; + return; } - + auto evState = std::make_unique<TEvDqCompute::TEvState>(); - evState->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING); - evState->Record.SetTaskId(Task.GetId()); + evState->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING); + evState->Record.SetTaskId(Task.GetId()); FillStats(evState->Record.MutableStats(), /* last */ false); - - auto dbgPrintStats = [&]() { - NProtoBuf::TextFormat::Printer printer; - printer.SetUseShortRepeatedPrimitives(true); - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - - TString result; - printer.PrintToString(evState->Record.GetStats(), &result); - return result; - }; - - CA_LOG_D("Send stats to executor actor " << ExecuterId << " TaskId: " << Task.GetId() - << " Stats: " << dbgPrintStats()); - + + auto dbgPrintStats = [&]() { + NProtoBuf::TextFormat::Printer printer; + printer.SetUseShortRepeatedPrimitives(true); + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + + TString result; + printer.PrintToString(evState->Record.GetStats(), &result); + return result; + }; + + CA_LOG_D("Send stats to executor actor " << ExecuterId << " TaskId: " << Task.GetId() + << " Stats: " << dbgPrintStats()); + this->Send(ExecuterId, evState.release(), NActors::IEventHandle::FlagTrackDelivery); - - LastSendStatsTime = now; - } - + + LastSendStatsTime = now; + } + protected: const NActors::TActorId ExecuterId; const TTxId TxId; @@ -1497,9 +1497,9 @@ protected: }; TProcessOutputsState ProcessOutputsState; -private: +private: bool Running = true; - TInstant LastSendStatsTime; + TInstant LastSendStatsTime; }; } // namespace NYql diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto index da40416c34..0e38ab3bbc 100644 --- a/ydb/library/yql/dq/actors/protos/dq_events.proto +++ b/ydb/library/yql/dq/actors/protos/dq_events.proto @@ -1,5 +1,5 @@ package NYql.NDqProto; -option cc_enable_arenas = true; +option cc_enable_arenas = true; import "library/cpp/actors/protos/actors.proto"; import "ydb/library/yql/dq/actors/protos/dq_stats.proto"; @@ -78,13 +78,13 @@ message TEvAbortExecution { optional string Message = 2; } -message TRlPath { - optional string CoordinationNode = 1; - optional string ResourcePath = 2; - optional string Database = 3; - optional string Token = 4; -} - +message TRlPath { + optional string CoordinationNode = 1; + optional string ResourcePath = 2; + optional string Database = 3; + optional string Token = 4; +} + message TComputeRuntimeSettings { optional uint32 TimeoutMs = 1; @@ -100,7 +100,7 @@ message TComputeRuntimeSettings { optional bool UseLLVM = 4; optional bool UseSpilling = 6; optional uint32 TasksOnNodeCount = 5; // approx - optional TRlPath RlPath = 7; + optional TRlPath RlPath = 7; } message TEvNewCheckpointCoordinator { diff --git a/ydb/library/yql/dq/runtime/dq_transport.cpp b/ydb/library/yql/dq/runtime/dq_transport.cpp index 380555d772..93b85b0c6f 100644 --- a/ydb/library/yql/dq/runtime/dq_transport.cpp +++ b/ydb/library/yql/dq/runtime/dq_transport.cpp @@ -109,7 +109,7 @@ void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType NDqProto::TData SerializeParamV1(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv, const THolderFactory& holderFactory) { - auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); + auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory); return SerializeValuePickleV1(type, value); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp index d20d496f32..57599acb64 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp @@ -55,7 +55,7 @@ #include "mkql_multimap.h" #include "mkql_next_value.h" #include "mkql_now.h" -#include "mkql_null.h" +#include "mkql_null.h" #include "mkql_pickle.h" #include "mkql_prepend.h" #include "mkql_queue.h" diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp index 38a70ec8d1..7a0f69777e 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp @@ -318,7 +318,7 @@ IComputationNode* WrapNot(TCallable& callable, const TComputationNodeFactoryCont bool isOptional; const auto& dataType = UnpackOptionalData(callable.GetInput(0), isOptional); - const auto schemeType = dataType->GetSchemeType(); + const auto schemeType = dataType->GetSchemeType(); MKQL_ENSURE(schemeType == NUdf::TDataType<bool>::Id, "Expected bool"); const auto node = LocateNode(ctx.NodeLocator, callable, 0); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp index bb1e33b52b..a08f71206f 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp @@ -1,14 +1,14 @@ -#include "mkql_null.h" +#include "mkql_null.h" #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapNull(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 0, "Expected 0 arg"); + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapNull(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 0, "Expected 0 arg"); return ctx.NodeFactory.CreateImmutableNode(NUdf::TUnboxedValuePod()); -} - -} -} +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_null.h b/ydb/library/yql/minikql/comp_nodes/mkql_null.h index 3bdd38941e..97e1469b98 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_null.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_null.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapNull(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapNull(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index 0c955a849a..55838c4012 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -120,8 +120,8 @@ SRCS( mkql_next_value.h mkql_now.cpp mkql_now.h - mkql_null.cpp - mkql_null.h + mkql_null.cpp + mkql_null.h mkql_pickle.cpp mkql_pickle.h mkql_prepend.cpp diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node.h b/ydb/library/yql/minikql/computation/mkql_computation_node.h index bebf3c3d56..a5d11b180b 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node.h @@ -45,19 +45,19 @@ struct TComputationOpts { struct TComputationOptsFull: public TComputationOpts { TComputationOptsFull(IStatsRegistry* stats, TAllocState& allocState, IRandomProvider& randomProvider, - ITimeProvider& timeProvider, NUdf::EValidatePolicy validatePolicy, const NUdf::ISecureParamsProvider* secureParamsProvider) + ITimeProvider& timeProvider, NUdf::EValidatePolicy validatePolicy, const NUdf::ISecureParamsProvider* secureParamsProvider) : TComputationOpts(stats) , AllocState(allocState) , RandomProvider(randomProvider) , TimeProvider(timeProvider) , ValidatePolicy(validatePolicy) - , SecureParamsProvider(secureParamsProvider) + , SecureParamsProvider(secureParamsProvider) {} TAllocState& AllocState; IRandomProvider& RandomProvider; ITimeProvider& TimeProvider; NUdf::EValidatePolicy ValidatePolicy; - const NUdf::ISecureParamsProvider* SecureParamsProvider; + const NUdf::ISecureParamsProvider* SecureParamsProvider; }; struct TComputationMutables { @@ -293,7 +293,7 @@ struct TComputationPatternOpts { void SetOptions(TComputationNodeFactory factory, const IFunctionRegistry* functionRegistry, NUdf::EValidateMode validateMode, NUdf::EValidatePolicy validatePolicy, const TString& optLLVM, EGraphPerProcess graphPerProcess, IStatsRegistry* stats = nullptr, - NUdf::ICountersProvider* counters = nullptr, const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr) { + NUdf::ICountersProvider* counters = nullptr, const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr) { Factory = factory; FunctionRegistry = functionRegistry; ValidateMode = validateMode; @@ -302,7 +302,7 @@ struct TComputationPatternOpts { GraphPerProcess = graphPerProcess; Stats = stats; CountersProvider = counters; - SecureParamsProvider = secureParamsProvider; + SecureParamsProvider = secureParamsProvider; } mutable std::shared_ptr<TInjectedAlloc> CacheAlloc; @@ -311,18 +311,18 @@ struct TComputationPatternOpts { const TTypeEnvironment& Env; TComputationNodeFactory Factory; - const IFunctionRegistry* FunctionRegistry = nullptr; - NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::None; - NUdf::EValidatePolicy ValidatePolicy = NUdf::EValidatePolicy::Fail; + const IFunctionRegistry* FunctionRegistry = nullptr; + NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::None; + NUdf::EValidatePolicy ValidatePolicy = NUdf::EValidatePolicy::Fail; TString OptLLVM; - EGraphPerProcess GraphPerProcess = EGraphPerProcess::Multi; - IStatsRegistry* Stats = nullptr; - NUdf::ICountersProvider* CountersProvider = nullptr; - const NUdf::ISecureParamsProvider* SecureParamsProvider = nullptr; + EGraphPerProcess GraphPerProcess = EGraphPerProcess::Multi; + IStatsRegistry* Stats = nullptr; + NUdf::ICountersProvider* CountersProvider = nullptr; + const NUdf::ISecureParamsProvider* SecureParamsProvider = nullptr; /// \todo split and exclude TComputationOptsFull ToComputationOptions(IRandomProvider& randomProvider, ITimeProvider& timeProvider, TAllocState* allocStatePtr = nullptr) const { - return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, randomProvider, timeProvider, ValidatePolicy, SecureParamsProvider); + return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, randomProvider, timeProvider, ValidatePolicy, SecureParamsProvider); } }; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h index 8de36485b6..d9feede204 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h @@ -1001,7 +1001,7 @@ protected: return NextFunc(Ctx, static_cast<const NUdf::TUnboxedValuePod&>(Iterator), value); } - TComputationContext* const Ctx; + TComputationContext* const Ctx; const TNextPtr NextFunc; const NUdf::TUnboxedValue Iterator; }; @@ -1025,7 +1025,7 @@ protected: return NextFunc(Ctx, static_cast<const NUdf::TUnboxedValuePod&>(Iterator), State, value); } - TComputationContext* const Ctx; + TComputationContext* const Ctx; const TNextPtr NextFunc; const NUdf::TUnboxedValue Iterator; TStateType State; @@ -1048,7 +1048,7 @@ private: return NUdf::TUnboxedValuePod(new TIterator(GetMemInfo(), NextFunc, Ctx, List.GetListIterator())); } - TComputationContext* const Ctx; + TComputationContext* const Ctx; const TNextPtr NextFunc; const NUdf::TUnboxedValue List; }; @@ -1108,7 +1108,7 @@ private: return List.HasFastListLength(); } - TComputationContext* const Ctx; + TComputationContext* const Ctx; const TNextPtr NextFunc; const NUdf::TUnboxedValue List; }; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index e3ace52e8c..f2b188ae47 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -225,7 +225,7 @@ public: { PatternNodes->HolderFactory = MakeHolder<THolderFactory>(opts.AllocState, *PatternNodes->MemInfo, &FunctionRegistry); PatternNodes->ValueBuilder = MakeHolder<TDefaultValueBuilder>(*PatternNodes->HolderFactory, ValidatePolicy); - PatternNodes->ValueBuilder->SetSecureParamsProvider(opts.SecureParamsProvider); + PatternNodes->ValueBuilder->SetSecureParamsProvider(opts.SecureParamsProvider); NodeFactory = MakeHolder<TNodeFactory>(*PatternNodes->MemInfo, PatternNodes->Mutables); } @@ -437,7 +437,7 @@ private: if (!computationNode) { THROW yexception() - << "Computation graph builder, unsupported function: " << node.GetType()->GetName() << " type: " << Factory.target_type().name() ; + << "Computation graph builder, unsupported function: " << node.GetType()->GetName() << " type: " << Factory.target_type().name() ; } AddNode(node, computationNode); @@ -534,7 +534,7 @@ public: #endif HolderFactory = MakeHolder<THolderFactory>(CompOpts.AllocState, *MemInfo, patternNodes->HolderFactory->GetFunctionRegistry()); ValueBuilder = MakeHolder<TDefaultValueBuilder>(*HolderFactory.Get(), compOpts.ValidatePolicy); - ValueBuilder->SetSecureParamsProvider(CompOpts.SecureParamsProvider); + ValueBuilder->SetSecureParamsProvider(CompOpts.SecureParamsProvider); ArrowMemoryPool = MakeArrowMemoryPool(CompOpts.AllocState); } diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp index 6bb39620a5..ad47661715 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp @@ -5,8 +5,8 @@ #include <ydb/library/yql/minikql/mkql_string_util.h> #include <library/cpp/yson/node/node_io.h> -#include <util/system/env.h> - +#include <util/system/env.h> + namespace NKikimr { namespace NMiniKQL { @@ -229,11 +229,11 @@ bool TDefaultValueBuilder::FindTimezoneId(const NUdf::TStringRef& name, ui32& id return true; } -bool TDefaultValueBuilder::GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const { - if (SecureParamsProvider_) - return SecureParamsProvider_->GetSecureParam(key, value); - return false; -} +bool TDefaultValueBuilder::GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const { + if (SecureParamsProvider_) + return SecureParamsProvider_->GetSecureParam(key, value); + return false; +} } // namespace NMiniKQL } // namespace Nkikimr diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.h b/ydb/library/yql/minikql/computation/mkql_value_builder.h index c897b68011..b293b263a3 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.h +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.h @@ -26,7 +26,7 @@ public: void SetSecureParamsProvider(const NUdf::ISecureParamsProvider* provider); void RethrowAtTerminate(); void SetCalleePositionHolder(const NUdf::TSourcePosition*& position); - + void Terminate(const char* message) const final; NUdf::TUnboxedValue NewStringNotFilled(ui32 size) const final; @@ -54,13 +54,13 @@ public: return *this; } - bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef &value) const final; + bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef &value) const final; const NUdf::TSourcePosition* CalleePosition() const final; NUdf::TUnboxedValue Run(const NUdf::TSourcePosition& callee, const NUdf::IBoxedValue& value, const NUdf::TUnboxedValuePod* args) const; NUdf::TFlatDataBlockPtr NewFlatDataBlock(ui32 initialSize, ui32 initialCapacity) const; NUdf::TFlatArrayBlockPtr NewFlatArrayBlock(ui32 count) const; NUdf::TSingleBlockPtr NewSingleBlock(const NUdf::TUnboxedValue& value) const; - + bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final; bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final; @@ -89,7 +89,7 @@ public: private: const THolderFactory& HolderFactory_; NUdf::EValidatePolicy Policy_; - const NUdf::ISecureParamsProvider* SecureParamsProvider_ = nullptr; + const NUdf::ISecureParamsProvider* SecureParamsProvider_ = nullptr; const NUdf::TSourcePosition** CalleePositionPtr_ = nullptr; mutable bool Rethrow_ = false; }; diff --git a/ydb/library/yql/minikql/mkql_node_serialization.cpp b/ydb/library/yql/minikql/mkql_node_serialization.cpp index 5aa1bffb3f..e314f38cd6 100644 --- a/ydb/library/yql/minikql/mkql_node_serialization.cpp +++ b/ydb/library/yql/minikql/mkql_node_serialization.cpp @@ -1225,7 +1225,7 @@ namespace { } TNode* ReadDataType() { - const auto schemeType = ReadVar32(); + const auto schemeType = ReadVar32(); if (NUdf::TDataType<NUdf::TDecimal>::Id == schemeType) { const ui8 precision = Read(); const ui8 scale = Read(); diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 0aff49310c..47ae6e4c8f 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -1857,11 +1857,11 @@ TRuntimeNode TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Yson>(const NUdf:: return TRuntimeNode(BuildDataLiteral(data, NUdf::TDataType<NUdf::TYson>::Id, Env), true); } -template<> +template<> TRuntimeNode TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Json>(const NUdf::TStringRef& data) const { return TRuntimeNode(BuildDataLiteral(data, NUdf::TDataType<NUdf::TJson>::Id, Env), true); -} - +} + template<> TRuntimeNode TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::JsonDocument>(const NUdf::TStringRef& data) const { return TRuntimeNode(BuildDataLiteral(data, NUdf::TDataType<NUdf::TJsonDocument>::Id, Env), true); @@ -2104,7 +2104,7 @@ TRuntimeNode TProgramBuilder::NewVariant(TRuntimeNode item, ui32 index, TType* v TRuntimeNode TProgramBuilder::NewVariant(TRuntimeNode item, const std::string_view& member, TType* variantType) { const auto type = AS_TYPE(TVariantType, variantType); - MKQL_ENSURE(type->GetUnderlyingType()->IsStruct(), "Expected struct as underlying type"); + MKQL_ENSURE(type->GetUnderlyingType()->IsStruct(), "Expected struct as underlying type"); ui32 index = AS_TYPE(TStructType, type->GetUnderlyingType())->GetMemberIndex(member); return TRuntimeNode(TVariantLiteral::Create(item, index, type, Env), true); } @@ -3040,7 +3040,7 @@ TRuntimeNode TProgramBuilder::UnaryDataFunction(TRuntimeNode data, const std::st MKQL_ENSURE(!isOptional, "Optional data is not allowed"); } - auto schemeType = type->GetSchemeType(); + auto schemeType = type->GetSchemeType(); if (flags & TDataFunctionFlags::RequiresBooleanArgs) { MKQL_ENSURE(schemeType == NUdf::TDataType<bool>::Id, "Boolean data is required"); } else if (flags & TDataFunctionFlags::RequiresStringArgs) { @@ -3986,7 +3986,7 @@ TRuntimeNode TProgramBuilder::FromBytes(TRuntimeNode data, NUdf::TDataTypeId sch auto resultType = NewOptionalType(outDataType); TCallableBuilder callableBuilder(Env, __func__, resultType); callableBuilder.Add(data); - callableBuilder.Add(NewDataLiteral(static_cast<ui32>(schemeType))); + callableBuilder.Add(NewDataLiteral(static_cast<ui32>(schemeType))); return TRuntimeNode(callableBuilder.Build(), false); } @@ -5068,37 +5068,37 @@ bool CanExportType(TType* type, const TTypeEnvironment& env) { break; } - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType*>(node); - TType* innerType = variantType->GetUnderlyingType(); - - if (innerType->IsStruct()) { - auto structType = static_cast<TStructType*>(innerType); - for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - if (!structType->GetMemberType(index)->GetCookie()) { - canExport = false; - break; - } - } - } - - if (innerType->IsTuple()) { - auto tupleType = static_cast<TTupleType*>(innerType); - for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { - if (!tupleType->GetElementType(index)->GetCookie()) { - canExport = false; - break; - } - } - } - - if (canExport) { - node->SetCookie(1); - } - - break; - } - + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType*>(node); + TType* innerType = variantType->GetUnderlyingType(); + + if (innerType->IsStruct()) { + auto structType = static_cast<TStructType*>(innerType); + for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { + if (!structType->GetMemberType(index)->GetCookie()) { + canExport = false; + break; + } + } + } + + if (innerType->IsTuple()) { + auto tupleType = static_cast<TTupleType*>(innerType); + for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { + if (!tupleType->GetElementType(index)->GetCookie()) { + canExport = false; + break; + } + } + } + + if (canExport) { + node->SetCookie(1); + } + + break; + } + case TType::EKind::Type: break; diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index cfd5437173..10f1ad7ccf 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -127,7 +127,7 @@ public: //-- literal functions TRuntimeNode NewVoid(); - TRuntimeNode NewNull(); + TRuntimeNode NewNull(); TType* NewDataType(NUdf::TDataTypeId schemeType, bool optional = false); TType* NewDataType(NUdf::EDataSlot slot, bool optional = false) { diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp index 08b0efadf6..d559bb108c 100644 --- a/ydb/library/yql/minikql/mkql_type_ops.cpp +++ b/ydb/library/yql/minikql/mkql_type_ops.cpp @@ -334,33 +334,33 @@ static void WriteHex(ui16 bytes, IOutputStream& out, bool reverseBytes = false) } } -static void UuidToString(ui16 dw[8], IOutputStream& out) { - WriteHex(dw[1], out); - WriteHex(dw[0], out); - out << '-'; - WriteHex(dw[2], out); - out << '-'; - WriteHex(dw[3], out); - out << '-'; - WriteHex(dw[4], out, true); - out << '-'; - WriteHex(dw[5], out, true); - WriteHex(dw[6], out, true); - WriteHex(dw[7], out, true); -} - -} - -void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out) { - union { - char bytes[16]; - ui64 half[2]; - } buf; - buf.half[0] = low; - buf.half[1] = hi; - out.Write(buf.bytes, 16); -} - +static void UuidToString(ui16 dw[8], IOutputStream& out) { + WriteHex(dw[1], out); + WriteHex(dw[0], out); + out << '-'; + WriteHex(dw[2], out); + out << '-'; + WriteHex(dw[3], out); + out << '-'; + WriteHex(dw[4], out, true); + out << '-'; + WriteHex(dw[5], out, true); + WriteHex(dw[6], out, true); + WriteHex(dw[7], out, true); +} + +} + +void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out) { + union { + char bytes[16]; + ui64 half[2]; + } buf; + buf.half[0] = low; + buf.half[1] = hi; + out.Write(buf.bytes, 16); +} + NUdf::TUnboxedValuePod ValueToString(NUdf::EDataSlot type, NUdf::TUnboxedValuePod value) { TUnboxedValueStream out; switch (type) { @@ -417,7 +417,7 @@ NUdf::TUnboxedValuePod ValueToString(NUdf::EDataSlot type, NUdf::TUnboxedValuePo case NUdf::EDataSlot::Uuid: { ui16 dw[8]; std::memcpy(dw, value.AsStringRef().Data(), sizeof(dw)); - UuidToString(dw, out); + UuidToString(dw, out); break; } diff --git a/ydb/library/yql/minikql/mkql_type_ops.h b/ydb/library/yql/minikql/mkql_type_ops.h index cc1f817637..8ff9b258dd 100644 --- a/ydb/library/yql/minikql/mkql_type_ops.h +++ b/ydb/library/yql/minikql/mkql_type_ops.h @@ -19,7 +19,7 @@ bool IsLeapYear(ui32 year); ui32 GetMonthLength(ui32 month, bool isLeap); -void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out); +void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out); bool IsValidStringValue(NUdf::EDataSlot type, NUdf::TStringRef buf); diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 2e79389fad..7587e188fa 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -14,7 +14,7 @@ #include <ydb/library/yql/utils/swap_bytes.h> #include <library/cpp/yson/node/node_io.h> -#include <library/cpp/yson/writer.h> +#include <library/cpp/yson/writer.h> #include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/yson/parser.h> diff --git a/ydb/library/yql/providers/common/codec/yql_codec.h b/ydb/library/yql/providers/common/codec/yql_codec.h index 63745b1d4a..512e01224c 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.h +++ b/ydb/library/yql/providers/common/codec/yql_codec.h @@ -12,15 +12,15 @@ #include <util/generic/maybe.h> #include <util/stream/output.h> -#include <library/cpp/yson/public.h> - +#include <library/cpp/yson/public.h> + #include <list> #include <vector> -namespace NYT { - class TNode; -} - +namespace NYT { + class TNode; +} + namespace NYql { namespace NCommon { diff --git a/ydb/library/yql/providers/common/codec/yql_restricted_yson.h b/ydb/library/yql/providers/common/codec/yql_restricted_yson.h index cd96d5bdd9..0d0f7568d6 100644 --- a/ydb/library/yql/providers/common/codec/yql_restricted_yson.h +++ b/ydb/library/yql/providers/common/codec/yql_restricted_yson.h @@ -3,12 +3,12 @@ #include "yql_codec_results.h" #include <util/generic/strbuf.h> -#include <library/cpp/yson/public.h> +#include <library/cpp/yson/public.h> + +namespace NYT { + class TNode; +} -namespace NYT { - class TNode; -} - namespace NYql { namespace NCommon { diff --git a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp index 275206a085..3c92c84692 100644 --- a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp +++ b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp @@ -10,21 +10,21 @@ void TOperationResult::AddIssue(const TIssue& issue) { WalkThroughIssues(issue, false, [&](const TIssue& err, ui16 level) { Y_UNUSED(level); YQL_CLOG(NOTICE, ProviderCommon) << err; - }); + }); Issues_.AddIssue(issue); } void TOperationResult::AddIssues(const TIssues& issues) { - for (auto& topIssue: issues) { + for (auto& topIssue: issues) { WalkThroughIssues(topIssue, false, [&](const TIssue& err, ui16 level) { Y_UNUSED(level); YQL_CLOG(NOTICE, ProviderCommon) << err; - }); + }); } Issues_.AddIssues(issues); } -void TOperationResult::SetException(const std::exception& e, const TPosition& pos) { +void TOperationResult::SetException(const std::exception& e, const TPosition& pos) { YQL_CLOG(ERROR, ProviderCommon) << e.what(); auto issue = ExceptionToIssue(e, pos); diff --git a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h index 27928d8323..c4b498e9a1 100644 --- a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h +++ b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h @@ -20,7 +20,7 @@ public: void AddIssue(const TIssue& issue); void AddIssues(const TIssues& issues); - void SetException(const std::exception& e, const TPosition& pos = {}); + void SetException(const std::exception& e, const TPosition& pos = {}); void ReportIssues(TIssueManager& issueManager) const; private: @@ -71,7 +71,7 @@ TResult ResultFromError(const TString& error, TPosition pos = TPosition()) { } template<typename TResult> -TResult ResultFromErrors(const TIssues& errors) { +TResult ResultFromErrors(const TIssues& errors) { TResult result; result.AddIssues(errors); return result; diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index b241b363e1..dd1f162d78 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -888,8 +888,8 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("Json", [](const TExprNode& node, TMkqlBuildContext& ctx) { return ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::Json>(node.Head().Content()); - }); - + }); + AddCallable("JsonDocument", [](const TExprNode& node, TMkqlBuildContext& ctx) { // NOTE: ValueFromString returns TUnboxedValuePod. This type does not free string inside it during destruction. // To get smart pointer-like behaviour we convert TUnboxedValuePod to TUnboxedValue. Without this conversion there @@ -2052,8 +2052,8 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("Null", [](const TExprNode&, TMkqlBuildContext& ctx) { return ctx.ProgramBuilder.NewNull(); - }); - + }); + AddCallable({"AsTagged","Untag"}, [](const TExprNode& node, TMkqlBuildContext& ctx) { return MkqlBuildExpr(node.Head(), ctx); }); @@ -2282,10 +2282,10 @@ TMkqlCommonCallableCompiler::TShared::TShared() { const NNodes::TCoParameter parameter(&node); return ctx.ProgramBuilder.Member(ctx.Parameters, parameter.Name()); }); - + AddCallable("SecureParam", [](const TExprNode& node, TMkqlBuildContext& ctx) { return ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(node.Head().Content()); - }); + }); AddCallable(SkippableCallables, [](const TExprNode& node, TMkqlBuildContext& ctx) { return MkqlBuildExpr(node.Head(), ctx); diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 7a65e16b20..8561fe2536 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -103,7 +103,7 @@ message TKikimrGrpcData { optional uint64 MaxMessageSizeBytes = 3; optional uint32 MaxInFlight = 4; optional bool EnableEndpointDiscovery = 5; - optional bool UseLegacyApi = 6; + optional bool UseLegacyApi = 6; } message TKikimrClusterConfig { diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 72c4ce9f62..aa782ee93f 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -157,7 +157,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TMaybeNode<TCoLambda> filter; TMaybeNode<TCoLambda> update; TVector<TCoNameValueTuple> other; - TVector<TCoIndex> indexes; + TVector<TCoIndex> indexes; TVector<TCoChangefeed> changefeeds; TMaybeNode<TExprList> columnFamilies; TVector<TCoNameValueTuple> tableSettings; @@ -188,24 +188,24 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } else if (name == "update") { YQL_ENSURE(tuple.Value().Maybe<TCoLambda>()); update = tuple.Value().Cast<TCoLambda>(); - } else if (name == "index") { - YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); - auto index = Build<TCoIndex>(ctx, node.Pos()); - for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { - const auto& indexItemName = item.Name().Value(); - if (indexItemName == "indexName") { - index.Name(item.Value().Cast<TCoAtom>()); - } else if (indexItemName == "indexType") { - index.Type(item.Value().Cast<TCoAtom>()); - } else if (indexItemName == "indexColumns") { - index.Columns(item.Value().Cast<TCoAtomList>()); - } else if (indexItemName == "dataColumns") { - index.DataColumns(item.Value().Cast<TCoAtomList>()); - } else { - YQL_ENSURE(false, "unknown index item"); - } - } - indexes.push_back(index.Done()); + } else if (name == "index") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + auto index = Build<TCoIndex>(ctx, node.Pos()); + for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { + const auto& indexItemName = item.Name().Value(); + if (indexItemName == "indexName") { + index.Name(item.Value().Cast<TCoAtom>()); + } else if (indexItemName == "indexType") { + index.Type(item.Value().Cast<TCoAtom>()); + } else if (indexItemName == "indexColumns") { + index.Columns(item.Value().Cast<TCoAtomList>()); + } else if (indexItemName == "dataColumns") { + index.DataColumns(item.Value().Cast<TCoAtomList>()); + } else { + YQL_ENSURE(false, "unknown index item"); + } + } + indexes.push_back(index.Done()); } else if (name == "changefeed") { YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); auto cf = Build<TCoChangefeed>(ctx, node.Pos()); @@ -242,14 +242,14 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } } - const auto& otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) + const auto& otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) .Add(other) .Done(); - const auto& idx = Build<TCoIndexList>(ctx, node.Pos()) - .Add(indexes) - .Done(); - + const auto& idx = Build<TCoIndexList>(ctx, node.Pos()) + .Add(indexes) + .Done(); + const auto& cfs = Build<TCoChangefeedList>(ctx, node.Pos()) .Add(changefeeds) .Done(); @@ -266,7 +266,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { columnFamilies = Build<TExprList>(ctx, node.Pos()).Done(); } - TWriteTableSettings ret(otherSettings); + TWriteTableSettings ret(otherSettings); ret.Mode = mode; ret.Columns = columns; ret.PrimaryKey = primaryKey; @@ -274,7 +274,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.OrderBy = orderBy; ret.Filter = filter; ret.Update = update; - ret.Indexes = idx; + ret.Indexes = idx; ret.Changefeeds = cfs; ret.ColumnFamilies = columnFamilies; ret.TableSettings = tableProfileSettings; @@ -545,22 +545,22 @@ bool FillUsedFilesImpl( return childrenOk; } -static void GetToken(const TString& string, TString& out, const TTypeAnnotationContext& type) { - auto separator = string.find(":"); - const auto p0 = string.substr(0, separator); - if (p0 == "api") { - const auto p1 = string.substr(separator + 1); - if (p1 == "oauth") { - out = type.UserCredentials.OauthToken; - } else if (p1 == "cookie") { - out = type.UserCredentials.BlackboxSessionIdCookie; - } else { - YQL_ENSURE(false, "unexpected token id"); - } +static void GetToken(const TString& string, TString& out, const TTypeAnnotationContext& type) { + auto separator = string.find(":"); + const auto p0 = string.substr(0, separator); + if (p0 == "api") { + const auto p1 = string.substr(separator + 1); + if (p1 == "oauth") { + out = type.UserCredentials.OauthToken; + } else if (p1 == "cookie") { + out = type.UserCredentials.BlackboxSessionIdCookie; + } else { + YQL_ENSURE(false, "unexpected token id"); + } } else if (p0 == "token" || p0 == "cluster") { - const auto p1 = string.substr(separator + 1); - auto cred = type.FindCredential(p1); - if (cred == nullptr) { + const auto p1 = string.substr(separator + 1); + auto cred = type.FindCredential(p1); + if (cred == nullptr) { if (p0 == "cluster") { TStringBuf clusterName = p1; if (clusterName.SkipPrefix("default_")) { @@ -587,33 +587,33 @@ static void GetToken(const TString& string, TString& out, const TTypeAnnotationC } } - YQL_ENSURE(false, "unexpected token id"); - } - - out = cred->Content; - } else { - YQL_ENSURE(false, "unexpected token prefix"); - } -} - -void FillSecureParams( - const TExprNode::TPtr& root, - const TTypeAnnotationContext& types, - THashMap<TString, TString>& secureParams) { - - NYql::VisitExpr(root, [&secureParams](const TExprNode::TPtr& node) { - if (auto maybeSecureParam = TMaybeNode<TCoSecureParam>(node)) { - const auto& secureParamName = TString(maybeSecureParam.Cast().Name().Value()); - secureParams.insert({secureParamName, TString()}); - } - return true; - }); - - for (auto& it : secureParams) { - GetToken(it.first, it.second, types); - } -} - + YQL_ENSURE(false, "unexpected token id"); + } + + out = cred->Content; + } else { + YQL_ENSURE(false, "unexpected token prefix"); + } +} + +void FillSecureParams( + const TExprNode::TPtr& root, + const TTypeAnnotationContext& types, + THashMap<TString, TString>& secureParams) { + + NYql::VisitExpr(root, [&secureParams](const TExprNode::TPtr& node) { + if (auto maybeSecureParam = TMaybeNode<TCoSecureParam>(node)) { + const auto& secureParamName = TString(maybeSecureParam.Cast().Name().Value()); + secureParams.insert({secureParamName, TString()}); + } + return true; + }); + + for (auto& it : secureParams) { + GetToken(it.first, it.second, types); + } +} + bool FillUsedFiles( const TExprNode& node, TUserDataTable& files, diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index 36f7ace649..9e80cfd135 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -40,14 +40,14 @@ struct TWriteTableSettings { NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy; NNodes::TMaybeNode<NNodes::TCoLambda> Filter; NNodes::TMaybeNode<NNodes::TCoLambda> Update; - NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes; + NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes; NNodes::TMaybeNode<NNodes::TCoChangefeedList> Changefeeds; NNodes::TCoNameValueTupleList Other; NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies; NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings; NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> AlterActions; - TWriteTableSettings(const NNodes::TCoNameValueTupleList& other) + TWriteTableSettings(const NNodes::TCoNameValueTupleList& other) : Other(other) {} }; @@ -100,8 +100,8 @@ TVector<TString> GetStructFields(const TTypeAnnotationNode* type); void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStatistics& stats, NYson::TYsonWriter& writer); -void FillSecureParams(const TExprNode::TPtr& node, const TTypeAnnotationContext& types, THashMap<TString, TString>& secureParams); - +void FillSecureParams(const TExprNode::TPtr& node, const TTypeAnnotationContext& types, THashMap<TString, TString>& secureParams); + bool FillUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const TUserDataTable& crutches = {}); std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter, const TUserDataTable& crutches = {}); diff --git a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp index f4783073d2..0bbaf5c339 100644 --- a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp +++ b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp @@ -353,7 +353,7 @@ struct TExprTypeLoader { return ret; } void Error(const TString& info) { - Ctx.AddError(TIssue(Pos, info)); + Ctx.AddError(TIssue(Pos, info)); } }; @@ -361,7 +361,7 @@ const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext NYT::TNode node; TStringStream err; if (!ParseYson(node, yson, err)) { - ctx.AddError(TIssue(pos, err.Str())); + ctx.AddError(TIssue(pos, err.Str())); return nullptr; } diff --git a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp index 1dea71609a..0a34105714 100644 --- a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp +++ b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp @@ -13,7 +13,7 @@ #include <library/cpp/yson/node/node_io.h> #include <library/cpp/yson/node/node_builder.h> -#include <library/cpp/yson/writer.h> +#include <library/cpp/yson/writer.h> #include <util/stream/str.h> @@ -92,7 +92,7 @@ public: TBase::SaveEmptyDictType(); break; case TType::EKind::Data: { - const auto schemeType = static_cast<const TDataType*>(type)->GetSchemeType(); + const auto schemeType = static_cast<const TDataType*>(type)->GetSchemeType(); auto slot = NUdf::FindDataSlot(schemeType); if (!slot) { ythrow yexception() << "Unsupported data type: " << schemeType; @@ -101,7 +101,7 @@ public: auto dataType = NUdf::GetDataTypeInfo(*slot).Name; if (NKikimr::NUdf::TDataType<NUdf::TDecimal>::Id == schemeType) { const auto params = static_cast<const TDataDecimalType*>(type)->GetParams(); - TBase::SaveDataTypeParams(dataType, ToString(params.first), ToString(params.second)); + TBase::SaveDataTypeParams(dataType, ToString(params.first), ToString(params.second)); } else { TBase::SaveDataType(dataType); } diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index e2217c9e75..dc31e0ef30 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -145,7 +145,7 @@ public: for (auto udf : functions) { TStringBuf moduleName, funcName; if (!SplitUdfName(udf->Name, moduleName, funcName) || moduleName.empty() || funcName.empty()) { - ctx.AddError(TIssue(udf->Pos, TStringBuilder() << + ctx.AddError(TIssue(udf->Pos, TStringBuilder() << "Incorrect format of function name: " << udf->Name)); hasErrors = true; } else { @@ -191,8 +191,8 @@ public: try { LoadImport(holdingFileStorage, *filesBox, *import, request); - } catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e)); + } catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e)); hasErrors = true; } } @@ -327,7 +327,7 @@ private: import = *p; } if (importRes.HasError()) { - ctx.AddError(TIssue(import ? import->Pos : TPosition(), importRes.GetError())); + ctx.AddError(TIssue(import ? import->Pos : TPosition(), importRes.GetError())); hasErrors = true; } else if (import) { import->Modules.ConstructInPlace(); @@ -341,7 +341,7 @@ private: TFunction* udf = functions[i]; const TFunctionResult& udfRes = response.GetUdfs(i); if (udfRes.HasError()) { - ctx.AddError(TIssue(udf->Pos, udfRes.GetError())); + ctx.AddError(TIssue(udf->Pos, udfRes.GetError())); hasErrors = true; } else { udf->CallableType = ParseTypeFromYson(TStringBuf{udfRes.GetCallableType()}, ctx, udf->Pos); diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index 5bce71e060..3aac3487c0 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -48,7 +48,7 @@ public: auto& udf = *udfPtr; TStringBuf moduleName, funcName; if (!SplitUdfName(udf.Name, moduleName, funcName) || moduleName.empty() || funcName.empty()) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() << + ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Incorrect format of function name: " << udf.Name)); hasErrors = true; } else { @@ -84,7 +84,7 @@ public: path2import[path] = import; } else { if (import->Block->Type != EUserDataType::PATH) { - ctx.AddError(TIssue(import->Pos, TStringBuilder() << + ctx.AddError(TIssue(import->Pos, TStringBuilder() << "Only path file type is supported, cannot load file with alias: " << import->FileAlias)); hasErrors = true; continue; @@ -94,7 +94,7 @@ public: } } catch (yexception& e) { - ctx.AddError(TIssue(import->Pos, TStringBuilder() + ctx.AddError(TIssue(import->Pos, TStringBuilder() << "Internal error of loading udf module: " << import->FileAlias << ", reason: " << e.what())); hasErrors = true; diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index c073385af3..6952e96cf0 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -235,7 +235,7 @@ namespace { if (command == "PureDataSource") { if (Types.PureResultDataSource != node->Child(3)->Content()) { res = ctx.ChangeChild(*node, 3, ctx.RenameNode(*node->Child(3), Types.PureResultDataSource)); - } + } } } diff --git a/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp b/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp index b82f793116..9c263c2b0b 100644 --- a/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp +++ b/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp @@ -6,14 +6,14 @@ namespace NYql { namespace NDecimal { Y_UNIT_TEST_SUITE(TYqlDecimalTest) { - void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const TString& expected) { + void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const TString& expected) { TString result = ToString(v, precision, scale); UNIT_ASSERT_VALUES_EQUAL(result, expected); - TInt128 parsed = FromString(result, precision, scale); + TInt128 parsed = FromString(result, precision, scale); UNIT_ASSERT(parsed == v); } - void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { + void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { TString result = ToString(v, precision, scale); UNIT_ASSERT_VALUES_EQUAL(result, ""); } @@ -122,7 +122,7 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { } Y_UNIT_TEST(TestHugeNumberFormat) { - TInt128 max120 = Inf() - 1; + TInt128 max120 = Inf() - 1; const char max120String[] = "99999999999999999999999999999999999"; // 35 digits static_assert(sizeof(max120String) == 36, "sizeof(max120String) == 36"); SimplePositiveTest(max120, MaxPrecision, 0, max120String); diff --git a/ydb/library/yql/public/decimal/ya.make b/ydb/library/yql/public/decimal/ya.make index 6da29862a9..d3c8fbe4e6 100644 --- a/ydb/library/yql/public/decimal/ya.make +++ b/ydb/library/yql/public/decimal/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - -OWNER( - vvvv - dcherednik - g:yql - g:kikimr +LIBRARY() + +OWNER( + vvvv + dcherednik + g:yql + g:kikimr g:yql_ydb_core -) - -SRCS( - yql_decimal.h - yql_decimal.cpp +) + +SRCS( + yql_decimal.h + yql_decimal.cpp yql_decimal_serialize.h yql_decimal_serialize.cpp -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/public/decimal/yql_decimal.cpp b/ydb/library/yql/public/decimal/yql_decimal.cpp index 383d11896a..89fc92cd49 100644 --- a/ydb/library/yql/public/decimal/yql_decimal.cpp +++ b/ydb/library/yql/public/decimal/yql_decimal.cpp @@ -1,4 +1,4 @@ -#include "yql_decimal.h" +#include "yql_decimal.h" #include <cstring> #include <ostream> @@ -16,7 +16,7 @@ TUint128 GetDivider(ui8 scale) { return d; } -bool IsError(TInt128 v) { +bool IsError(TInt128 v) { return v > Nan() || v < -Nan(); } @@ -119,7 +119,7 @@ namespace { } -TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { +TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { if (scale > precision) return Err(); diff --git a/ydb/library/yql/public/decimal/yql_decimal.h b/ydb/library/yql/public/decimal/yql_decimal.h index 3f1a74350b..20695f0d84 100644 --- a/ydb/library/yql/public/decimal/yql_decimal.h +++ b/ydb/library/yql/public/decimal/yql_decimal.h @@ -14,15 +14,15 @@ namespace NDecimal { #define DONT_USE_NATIVE_INT128 #endif #endif - + #ifdef DONT_USE_NATIVE_INT128 using TInt128 = TWide<i64>; using TUint128 = TWide<ui64>; -#else +#else using TInt128 = signed __int128; using TUint128 = unsigned __int128; #endif - + template<ui8 Scale> struct TDivider; #if defined(__clang__) && defined(DONT_USE_NATIVE_INT128) template<> struct TDivider<0> { static inline constexpr TUint128 Value = 1U; }; @@ -30,12 +30,12 @@ template<ui8 Scale> struct TDivider { static inline constexpr TInt128 Value = TD #else template<> struct TDivider<0> { static constexpr TUint128 Value = 1U; }; template<ui8 Scale> struct TDivider { static constexpr TUint128 Value = TDivider<Scale - 1U>::Value * 10U; }; -#endif - +#endif + constexpr ui8 MaxPrecision = 35; -static_assert(sizeof(TInt128) == 16, "Wrong size of TInt128, expected 16"); - +static_assert(sizeof(TInt128) == 16, "Wrong size of TInt128, expected 16"); + inline constexpr TInt128 Inf() { return TInt128(100000000000000000ULL) * TInt128(1000000000000000000ULL); } @@ -73,20 +73,20 @@ inline bool IsNormal(TInt128 v) { return v > b.first && v < b.second; } -const char* ToString(TInt128 v, ui8 precision, ui8 scale = 0); -TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale = 0); - +const char* ToString(TInt128 v, ui8 precision, ui8 scale = 0); +TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale = 0); + // Accept string representation with exponent. TInt128 FromStringEx(const TStringBuf& str, ui8 precision, ui8 scale); -template<typename TMkqlProto> -inline TInt128 FromProto(const TMkqlProto& val) { - ui64 half[2] = {val.GetLow128(), val.GetHi128()}; - TInt128 val128; +template<typename TMkqlProto> +inline TInt128 FromProto(const TMkqlProto& val) { + ui64 half[2] = {val.GetLow128(), val.GetHi128()}; + TInt128 val128; std::memcpy(&val128, half, sizeof(val128)); return val128; } - + template<typename TValue> inline constexpr TValue YtDecimalNan() { return std::numeric_limits<TValue>::max(); @@ -129,13 +129,13 @@ inline TValue ToYtDecimal(TInt128 val) { return YtDecimalNan<TValue>(); } -inline TInt128 FromHalfs(ui64 lo, i64 hi) { - ui64 half[2] = {lo, static_cast<ui64>(hi)}; - TInt128 val128; - std::memcpy(&val128, half, sizeof(val128)); - return val128; -} - +inline TInt128 FromHalfs(ui64 lo, i64 hi) { + ui64 half[2] = {lo, static_cast<ui64>(hi)}; + TInt128 val128; + std::memcpy(&val128, half, sizeof(val128)); + return val128; +} + inline std::pair<ui64, ui64> MakePair(const TInt128 v) { std::pair<ui64, ui64> r; std::memcpy(&r, &v, sizeof(v)); @@ -155,5 +155,5 @@ TInt128 MulAndDivNormalDivider(TInt128 a, TInt128 b, TInt128 c); // a*b/c Only for non zero normal positive multiplier. TInt128 MulAndDivNormalMultiplier(TInt128 a, TInt128 b, TInt128 c); -} +} } diff --git a/ydb/library/yql/public/issue/protos/issue_message.proto b/ydb/library/yql/public/issue/protos/issue_message.proto index 3fb90ad840..d581a5d8f0 100644 --- a/ydb/library/yql/public/issue/protos/issue_message.proto +++ b/ydb/library/yql/public/issue/protos/issue_message.proto @@ -1,17 +1,17 @@ -package NYql.NIssue.NProto; +package NYql.NIssue.NProto; option java_package = "com.yandex.yql.issue.proto"; - -message IssueMessage { - message Position { - optional uint32 row = 1; - optional uint32 column = 2; - optional string file = 3; - } - - optional Position position = 1; - optional string message = 2; - optional Position end_position = 3; - optional uint32 issue_code = 4; - optional uint32 severity = 5; + +message IssueMessage { + message Position { + optional uint32 row = 1; + optional uint32 column = 2; + optional string file = 3; + } + + optional Position position = 1; + optional string message = 2; + optional Position end_position = 3; + optional uint32 issue_code = 4; + optional uint32 severity = 5; repeated IssueMessage issues = 6; -} +} diff --git a/ydb/library/yql/public/issue/protos/issue_severity.proto b/ydb/library/yql/public/issue/protos/issue_severity.proto index f1b916f669..70fd61f5d9 100644 --- a/ydb/library/yql/public/issue/protos/issue_severity.proto +++ b/ydb/library/yql/public/issue/protos/issue_severity.proto @@ -1,14 +1,14 @@ -syntax = "proto3"; - -package NYql; - +syntax = "proto3"; + +package NYql; + option java_package = "com.yandex.yql.proto"; - -message TSeverityIds { - enum ESeverityId { - S_FATAL = 0; - S_ERROR = 1; - S_WARNING = 2; + +message TSeverityIds { + enum ESeverityId { + S_FATAL = 0; + S_ERROR = 1; + S_WARNING = 2; S_INFO = 3; - } -} + } +} diff --git a/ydb/library/yql/public/issue/protos/ya.make b/ydb/library/yql/public/issue/protos/ya.make index 8544401f5f..f50541d000 100644 --- a/ydb/library/yql/public/issue/protos/ya.make +++ b/ydb/library/yql/public/issue/protos/ya.make @@ -1,12 +1,12 @@ -PROTO_LIBRARY() - +PROTO_LIBRARY() + OWNER(g:yql) - -SRCS( - issue_message.proto - issue_severity.proto -) - + +SRCS( + issue_message.proto + issue_severity.proto +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/library/yql/public/issue/ut/ya.make b/ydb/library/yql/public/issue/ut/ya.make index c1dba6363a..4fc5841a51 100644 --- a/ydb/library/yql/public/issue/ut/ya.make +++ b/ydb/library/yql/public/issue/ut/ya.make @@ -1,17 +1,17 @@ UNITTEST_FOR(ydb/library/yql/public/issue) - + OWNER(g:yql) - -FORK_SUBTESTS() - -SRCS( - yql_issue_ut.cpp - yql_issue_manager_ut.cpp + +FORK_SUBTESTS() + +SRCS( + yql_issue_ut.cpp + yql_issue_manager_ut.cpp yql_issue_utils_ut.cpp yql_warning_ut.cpp -) - -PEERDIR( -) - -END() +) + +PEERDIR( +) + +END() diff --git a/ydb/library/yql/public/issue/ya.make b/ydb/library/yql/public/issue/ya.make index 63baa00272..04a84bee88 100644 --- a/ydb/library/yql/public/issue/ya.make +++ b/ydb/library/yql/public/issue/ya.make @@ -1,30 +1,30 @@ -LIBRARY() - +LIBRARY() + OWNER( g:yql g:yql_ydb_core ) - -SRCS( - yql_issue.cpp - yql_issue_message.cpp - yql_issue_manager.cpp + +SRCS( + yql_issue.cpp + yql_issue_message.cpp + yql_issue_manager.cpp yql_issue_utils.cpp yql_warning.cpp -) - -PEERDIR( +) + +PEERDIR( contrib/libs/protobuf library/cpp/colorizer library/cpp/resource ydb/public/api/protos ydb/library/yql/public/issue/protos ydb/library/yql/utils -) - +) + GENERATE_ENUM_SERIALIZATION(yql_warning.h) -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/public/issue/yql_issue.cpp b/ydb/library/yql/public/issue/yql_issue.cpp index 8f1e1a33e2..3b89156874 100644 --- a/ydb/library/yql/public/issue/yql_issue.cpp +++ b/ydb/library/yql/public/issue/yql_issue.cpp @@ -9,14 +9,14 @@ #include <util/string/split.h> #include <util/string/strip.h> #include <util/string/subst.h> -#include <util/system/compiler.h> -#include <util/generic/map.h> -#include <util/generic/stack.h> -#include <cstdlib> - - -namespace NYql { - +#include <util/system/compiler.h> +#include <util/generic/map.h> +#include <util/generic/stack.h> +#include <cstdlib> + + +namespace NYql { + void SanitizeNonAscii(TString& s) { if (!NYql::IsUtf8(s)) { for (size_t i = 0; i < s.size(); ++i) { @@ -78,18 +78,18 @@ void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren)); } issuesStack.push(std::make_tuple(0, &topIssue, EFnType::Main)); - while (!issuesStack.empty()) { + while (!issuesStack.empty()) { auto level = std::get<0>(issuesStack.top()); const auto& curIssue = *std::get<1>(issuesStack.top()); const EFnType fnType = std::get<2>(issuesStack.top()); - issuesStack.pop(); - if (!leafOnly || curIssue.GetSubIssues().empty()) { + issuesStack.pop(); + if (!leafOnly || curIssue.GetSubIssues().empty()) { if (fnType == EFnType::Main) { fn(curIssue, level); } else { afterChildrenFn(curIssue, level); } - } + } if (fnType == EFnType::Main) { level++; const auto& subIssues = curIssue.GetSubIssues(); @@ -99,52 +99,52 @@ void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void } issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::Main)); } - } - } -} - -namespace { - + } + } +} + +namespace { + Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { char* whitespaces = reinterpret_cast<char*>(alloca(indentation)); - memset(whitespaces, ' ', indentation); - out.Write(whitespaces, indentation); -} - -void ProgramLinesWithErrors( - const TString& programText, + memset(whitespaces, ' ', indentation); + out.Write(whitespaces, indentation); +} + +void ProgramLinesWithErrors( + const TString& programText, const TVector<TIssue>& errors, TMap<ui32, TStringBuf>& lines) -{ +{ TVector<ui32> rows; - for (const auto& topIssue: errors) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { - for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { - rows.push_back(row); - } - }); - } - std::sort(rows.begin(), rows.end()); - - auto prog = StringSplitter(programText).Split('\n'); + for (const auto& topIssue: errors) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { + for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { + rows.push_back(row); + } + }); + } + std::sort(rows.begin(), rows.end()); + + auto prog = StringSplitter(programText).Split('\n'); auto progIt = prog.begin(), progEnd = prog.end(); - ui32 progRow = 1; - - for (ui32 row: rows) { - while (progRow < row && progIt != progEnd) { - ++progRow; - ++progIt; - } - if (progIt != progEnd) { - lines[row] = progIt->Token(); - } - } -} - -} // namspace - + ui32 progRow = 1; + + for (ui32 row: rows) { + while (progRow < row && progIt != progEnd) { + ++progRow; + ++progIt; + } + if (progIt != progEnd) { + lines[row] = progIt->Token(); + } + } +} + +} // namspace + void TIssues::PrintTo(IOutputStream& out, bool oneLine) const -{ +{ if (oneLine) { bool printWithSpace = false; if (Issues_.size() > 1) { @@ -175,42 +175,42 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const out << issue << Endl; }); } - } -} - -void TIssues::PrintWithProgramTo( + } +} + +void TIssues::PrintWithProgramTo( IOutputStream& out, - const TString& programFilename, - const TString& programText) const -{ - using namespace NColorizer; - + const TString& programFilename, + const TString& programText) const +{ + using namespace NColorizer; + TMap<ui32, TStringBuf> lines; - ProgramLinesWithErrors(programText, Issues_, lines); - - for (const TIssue& topIssue: Issues_) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { - auto shift = level * 4; - Indent(out, shift); - out << DarkGray() << programFilename << Old() << ':'; - out << Purple() << issue.Range() << Old(); - auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed(); - auto severityName = SeverityToString(issue.GetSeverity()); - out << color << ": "<< severityName << ": " << issue.Message << Old() << '\n'; - Indent(out, shift); - if (issue.Position.HasValue()) { - out << '\t' << lines[issue.Position.Row] << '\n'; - out << '\t'; - if (issue.Position.Column > 0) { - Indent(out, issue.Position.Column - 1); - } - out << '^'; - } - out << Endl; - }); - } -} - + ProgramLinesWithErrors(programText, Issues_, lines); + + for (const TIssue& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + auto shift = level * 4; + Indent(out, shift); + out << DarkGray() << programFilename << Old() << ':'; + out << Purple() << issue.Range() << Old(); + auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed(); + auto severityName = SeverityToString(issue.GetSeverity()); + out << color << ": "<< severityName << ": " << issue.Message << Old() << '\n'; + Indent(out, shift); + if (issue.Position.HasValue()) { + out << '\t' << lines[issue.Position.Row] << '\n'; + out << '\t'; + if (issue.Position.Column > 0) { + Indent(out, issue.Position.Column - 1); + } + out << '^'; + } + out << Endl; + }); + } +} + TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { TStringBuf messageBuf = e.what(); auto parsedPos = TryParseTerminationMessage(messageBuf); @@ -262,26 +262,26 @@ TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) { return Nothing(); } -} // namspace NYql - -template <> +} // namspace NYql + +template <> void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) { out << (pos.File ? pos.File : "<main>"); if (pos) { out << ":" << pos.Row << ':' << pos.Column; } -} - -template<> +} + +template<> void Out<NYql::TRange>(IOutputStream & out, const NYql::TRange & range) { - if (range.IsRange()) { - out << '[' << range.Position << '-' << range.EndPosition << ']'; - } else { - out << range.Position; - } -} - -template <> + if (range.IsRange()) { + out << '[' << range.Position << '-' << range.EndPosition << ']'; + } else { + out << range.Position; + } +} + +template <> void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error) { error.PrintTo(out); -} +} diff --git a/ydb/library/yql/public/issue/yql_issue.h b/ydb/library/yql/public/issue/yql_issue.h index 0cc1cbcd95..7b97674c9d 100644 --- a/ydb/library/yql/public/issue/yql_issue.h +++ b/ydb/library/yql/public/issue/yql_issue.h @@ -1,58 +1,58 @@ -#pragma once - -#include <util/system/types.h> +#pragma once + +#include <util/system/types.h> #include <util/generic/hash.h> #include <util/generic/maybe.h> -#include <util/generic/vector.h> -#include <util/generic/string.h> -#include <util/generic/strbuf.h> -#include <util/generic/ptr.h> -#include <util/stream/output.h> -#include <util/stream/str.h> -#include <util/digest/numeric.h> +#include <util/generic/vector.h> +#include <util/generic/string.h> +#include <util/generic/strbuf.h> +#include <util/generic/ptr.h> +#include <util/stream/output.h> +#include <util/stream/str.h> +#include <util/digest/numeric.h> #include <google/protobuf/message.h> - -#include "yql_issue_id.h" - -namespace NYql { - + +#include "yql_issue_id.h" + +namespace NYql { + void SanitizeNonAscii(TString& s); -/////////////////////////////////////////////////////////////////////////////// -// TPosition -/////////////////////////////////////////////////////////////////////////////// -struct TPosition { - ui32 Column = 0U; - ui32 Row = 0U; +/////////////////////////////////////////////////////////////////////////////// +// TPosition +/////////////////////////////////////////////////////////////////////////////// +struct TPosition { + ui32 Column = 0U; + ui32 Row = 0U; TString File; - - TPosition() = default; - + + TPosition() = default; + TPosition(ui32 column, ui32 row, const TString& file = {}) - : Column(column) - , Row(row) + : Column(column) + , Row(row) , File(file) - { + { SanitizeNonAscii(File); - } - - explicit operator bool() const { - return HasValue(); - } - - inline bool HasValue() const { - return Row | Column; - } - - inline bool operator==(const TPosition& other) const { + } + + explicit operator bool() const { + return HasValue(); + } + + inline bool HasValue() const { + return Row | Column; + } + + inline bool operator==(const TPosition& other) const { return Column == other.Column && Row == other.Row && File == other.File; - } - - inline bool operator<(const TPosition& other) const { + } + + inline bool operator<(const TPosition& other) const { return std::tie(Row, Column, File) < std::tie(other.Row, other.Column, other.File); - } -}; - + } +}; + class TTextWalker { public: TTextWalker(TPosition& position) @@ -78,47 +78,47 @@ private: ui32 LfCount; }; -struct TRange { - TPosition Position; - TPosition EndPosition; - - TRange() = default; - - TRange(TPosition position) - : Position(position) - , EndPosition(position) - { - } - - TRange(TPosition position, TPosition endPosition) - : Position(position) - , EndPosition(endPosition) - { - } - - inline bool IsRange() const { - return !(Position == EndPosition); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// TIssue -/////////////////////////////////////////////////////////////////////////////// - -class TIssue; -using TIssuePtr = TIntrusivePtr<TIssue>; -class TIssue: public TThrRefBase { +struct TRange { + TPosition Position; + TPosition EndPosition; + + TRange() = default; + + TRange(TPosition position) + : Position(position) + , EndPosition(position) + { + } + + TRange(TPosition position, TPosition endPosition) + : Position(position) + , EndPosition(endPosition) + { + } + + inline bool IsRange() const { + return !(Position == EndPosition); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// TIssue +/////////////////////////////////////////////////////////////////////////////// + +class TIssue; +using TIssuePtr = TIntrusivePtr<TIssue>; +class TIssue: public TThrRefBase { TVector<TIntrusivePtr<TIssue>> Children_; -public: - TPosition Position; - TPosition EndPosition; - TString Message; +public: + TPosition Position; + TPosition EndPosition; + TString Message; TIssueCode IssueCode = 0U; ESeverity Severity = TSeverityIds::S_ERROR; - - TIssue() = default; - - template <typename T> + + TIssue() = default; + + template <typename T> explicit TIssue(const T& message) : Position(TPosition()) , EndPosition(TPosition()) @@ -128,64 +128,64 @@ public: } template <typename T> - TIssue(TPosition position, const T& message) - : Position(position) - , EndPosition(position) - , Message(message) - { + TIssue(TPosition position, const T& message) + : Position(position) + , EndPosition(position) + , Message(message) + { SanitizeNonAscii(Message); - } - - inline TRange Range() const { - return{ Position, EndPosition }; - } - - template <typename T> - TIssue(TPosition position, TPosition endPosition, const T& message) - : Position(position) - , EndPosition(endPosition) - , Message(message) - { + } + + inline TRange Range() const { + return{ Position, EndPosition }; + } + + template <typename T> + TIssue(TPosition position, TPosition endPosition, const T& message) + : Position(position) + , EndPosition(endPosition) + , Message(message) + { SanitizeNonAscii(Message); - } - - inline bool operator==(const TIssue& other) const { - return Position == other.Position && Message == other.Message - && IssueCode == other.IssueCode; - } - - ui64 Hash() const noexcept { - return CombineHashes( + } + + inline bool operator==(const TIssue& other) const { + return Position == other.Position && Message == other.Message + && IssueCode == other.IssueCode; + } + + ui64 Hash() const noexcept { + return CombineHashes( CombineHashes( (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), ComputeHash(Position.File) ), (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message))); - } - + } + TIssue& SetCode(TIssueCode id, ESeverity severity) { - IssueCode = id; - Severity = severity; + IssueCode = id; + Severity = severity; return *this; - } - - ESeverity GetSeverity() const { - return Severity; - } - - TIssueCode GetCode() const { - return IssueCode; - } - + } + + ESeverity GetSeverity() const { + return Severity; + } + + TIssueCode GetCode() const { + return IssueCode; + } + TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) { Severity = (ESeverity)Min((ui32)issue->GetSeverity(), (ui32)Severity); - Children_.push_back(issue); + Children_.push_back(issue); return *this; - } - + } + const TVector<TIntrusivePtr<TIssue>>& GetSubIssues() const { - return Children_; - } + return Children_; + } void PrintTo(IOutputStream& out, bool oneLine = false) const; @@ -194,17 +194,17 @@ public: PrintTo(out, oneLine); return out.Str(); } -}; - +}; + void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn = {}); - -/////////////////////////////////////////////////////////////////////////////// -// TIssues -/////////////////////////////////////////////////////////////////////////////// -class TIssues { -public: - TIssues() = default; - + +/////////////////////////////////////////////////////////////////////////////// +// TIssues +/////////////////////////////////////////////////////////////////////////////// +class TIssues { +public: + TIssues() = default; + inline TIssues(const TVector<TIssue>& issues) : Issues_(issues) { @@ -215,125 +215,125 @@ public: { } - inline TIssues(const TIssues& rhs) - : Issues_(rhs.Issues_) - { - } - - inline TIssues& operator=(const TIssues& rhs) { - Issues_ = rhs.Issues_; - return *this; - } - + inline TIssues(const TIssues& rhs) + : Issues_(rhs.Issues_) + { + } + + inline TIssues& operator=(const TIssues& rhs) { + Issues_ = rhs.Issues_; + return *this; + } + inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) - { - } - - inline TIssues& operator=(TIssues&& rhs) { + { + } + + inline TIssues& operator=(TIssues&& rhs) { Issues_ = std::move(rhs.Issues_); - return *this; - } - + return *this; + } + template <typename ... Args> void AddIssue(Args&& ... args) { - Issues_.emplace_back(std::forward<Args>(args)...); - } - + Issues_.emplace_back(std::forward<Args>(args)...); + } + inline void AddIssues(const TIssues& errors) { - Issues_.insert(Issues_.end(), - errors.Issues_.begin(), errors.Issues_.end()); - } - + Issues_.insert(Issues_.end(), + errors.Issues_.begin(), errors.Issues_.end()); + } + inline void AddIssues(const TPosition& pos, const TIssues& errors) { - Issues_.reserve(Issues_.size() + errors.Size()); - for (const auto& e: errors) { - Issues_.push_back(TIssue(pos, e.Message)); - } - } - - inline const TIssue* begin() const { - return Issues_.begin(); - } - - inline const TIssue* end() const { - return Issues_.end(); - } - - inline TIssue& back() { - return Issues_.back(); - } - - inline const TIssue& back() const { - return Issues_.back(); - } - - inline bool Empty() const { - return Issues_.empty(); - } - + Issues_.reserve(Issues_.size() + errors.Size()); + for (const auto& e: errors) { + Issues_.push_back(TIssue(pos, e.Message)); + } + } + + inline const TIssue* begin() const { + return Issues_.begin(); + } + + inline const TIssue* end() const { + return Issues_.end(); + } + + inline TIssue& back() { + return Issues_.back(); + } + + inline const TIssue& back() const { + return Issues_.back(); + } + + inline bool Empty() const { + return Issues_.empty(); + } + explicit operator bool() const noexcept { return !Issues_.empty(); } - inline size_t Size() const { - return Issues_.size(); - } - + inline size_t Size() const { + return Issues_.size(); + } + void PrintTo(IOutputStream& out, bool oneLine = false) const; - void PrintWithProgramTo( + void PrintWithProgramTo( IOutputStream& out, - const TString& programFilename, - const TString& programText) const; - + const TString& programFilename, + const TString& programText) const; + inline TString ToString(bool oneLine = false) const { - TStringStream out; + TStringStream out; PrintTo(out, oneLine); - return out.Str(); - } - + return out.Str(); + } + TString ToOneLineString() const { return ToString(true); } - inline void Clear() { - Issues_.clear(); - } - + inline void Clear() { + Issues_.clear(); + } + void Reserve(size_t capacity) { Issues_.reserve(capacity); } -private: +private: TVector<TIssue> Issues_; -}; - -class TErrorException : public yexception { - const TIssueCode Code_; -public: - explicit TErrorException(TIssueCode code) - : Code_(code) - {} - TIssueCode GetCode() const { - return Code_; - } -}; - +}; + +class TErrorException : public yexception { + const TIssueCode Code_; +public: + explicit TErrorException(TIssueCode code) + : Code_(code) + {} + TIssueCode GetCode() const { + return Code_; + } +}; + TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos = TPosition()); TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message); - -} // namespace NYql - -template <> + +} // namespace NYql + +template <> void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos); - -template <> + +template <> void Out<NYql::TRange>(IOutputStream& out, const NYql::TRange& pos); - -template <> + +template <> void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error); - -template <> -struct THash<NYql::TIssue> { - inline size_t operator()(const NYql::TIssue& err) const { - return err.Hash(); - } -}; + +template <> +struct THash<NYql::TIssue> { + inline size_t operator()(const NYql::TIssue& err) const { + return err.Hash(); + } +}; diff --git a/ydb/library/yql/public/issue/yql_issue_id.h b/ydb/library/yql/public/issue/yql_issue_id.h index e84d36a9e6..9f4300f06b 100644 --- a/ydb/library/yql/public/issue/yql_issue_id.h +++ b/ydb/library/yql/public/issue/yql_issue_id.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/protos/issue_severity.pb.h> #include <library/cpp/resource/resource.h> @@ -7,116 +7,116 @@ #include <google/protobuf/repeated_field.h> #include <google/protobuf/text_format.h> #include <google/protobuf/message.h> - + #include <util/generic/hash.h> #include <util/generic/singleton.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/string/subst.h> -namespace NYql { - -using TIssueCode = ui32; -using ESeverity = NYql::TSeverityIds::ESeverityId; -const TIssueCode DEFAULT_ERROR = 0; -const TIssueCode UNEXPECTED_ERROR = 1; - -inline TString SeverityToString(ESeverity severity) -{ - auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); +namespace NYql { + +using TIssueCode = ui32; +using ESeverity = NYql::TSeverityIds::ESeverityId; +const TIssueCode DEFAULT_ERROR = 0; +const TIssueCode UNEXPECTED_ERROR = 1; + +inline TString SeverityToString(ESeverity severity) +{ + auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); return ret.empty() ? "Unknown" : to_title(ret.substr(2)); //remove prefix "S_" -} - -template <typename T> -inline TString IssueCodeToString(TIssueCode id) { - auto ret = T::EIssueCode_Name(static_cast<typename T::EIssueCode>(id)); +} + +template <typename T> +inline TString IssueCodeToString(TIssueCode id) { + auto ret = T::EIssueCode_Name(static_cast<typename T::EIssueCode>(id)); if (!ret.empty()) { SubstGlobal(ret, '_', ' '); return to_title(ret); } else { return "Unknown"; } -} - -template<typename TProto, const char* ResourceName> -class TIssueId { - TProto ProtoIssues_; +} + +template<typename TProto, const char* ResourceName> +class TIssueId { + TProto ProtoIssues_; THashMap<TIssueCode, NYql::TSeverityIds::ESeverityId> IssuesMap_; THashMap<TIssueCode, TString> IssuesFormatMap_; - - const google::protobuf::Descriptor* GetProtoDescriptor() const { - auto ret = ProtoIssues_.GetDescriptor(); - Y_ENSURE(ret != nullptr, "Bad proto file"); - return ret; - } - - bool CheckSeverityNameFormat(const TString& name) const { - if (name.size() > 2 && name.substr(0,2) == "S_") { - return true; - } - return false; - } - -public: - ESeverity GetSeverity(TIssueCode id) const { - auto it = IssuesMap_.find(id); - Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: " - << id << "(" << IssueCodeToString<TProto>(id) << ")"); - return it->second; - } - + + const google::protobuf::Descriptor* GetProtoDescriptor() const { + auto ret = ProtoIssues_.GetDescriptor(); + Y_ENSURE(ret != nullptr, "Bad proto file"); + return ret; + } + + bool CheckSeverityNameFormat(const TString& name) const { + if (name.size() > 2 && name.substr(0,2) == "S_") { + return true; + } + return false; + } + +public: + ESeverity GetSeverity(TIssueCode id) const { + auto it = IssuesMap_.find(id); + Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: " + << id << "(" << IssueCodeToString<TProto>(id) << ")"); + return it->second; + } + TString GetMessage(TIssueCode id) const { auto it = IssuesFormatMap_.find(id); - Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: " + Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: " << id << "(" << IssueCodeToString<TProto>(id) << ")"); return it->second; } - TIssueId() { - auto configData = NResource::Find(TStringBuf(ResourceName)); - if (!::google::protobuf::TextFormat::ParseFromString(configData, &ProtoIssues_)) { - Y_ENSURE(false, "Bad format of protobuf data file, resource: " << ResourceName); - } - - auto sDesc = TSeverityIds::ESeverityId_descriptor(); - for (int i = 0; i < sDesc->value_count(); i++) { - const auto& name = sDesc->value(i)->name(); - Y_ENSURE(CheckSeverityNameFormat(name), - "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); - } - - for (const auto& x : ProtoIssues_.ids()) { - auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity())); - Y_ENSURE(rv.second, "Duplicate issue code found, code: " - << static_cast<int>(x.code()) - << "(" << IssueCodeToString<TProto>(x.code()) <<")"); - } - - // Check all IssueCodes have mapping to severity - auto eDesc = TProto::EIssueCode_descriptor(); - for (int i = 0; i < eDesc->value_count(); i++) { - auto it = IssuesMap_.find(eDesc->value(i)->number()); - Y_ENSURE(it != IssuesMap_.end(), "IssueCode: " - << eDesc->value(i)->name() - << " is not found in protobuf data file"); - } + TIssueId() { + auto configData = NResource::Find(TStringBuf(ResourceName)); + if (!::google::protobuf::TextFormat::ParseFromString(configData, &ProtoIssues_)) { + Y_ENSURE(false, "Bad format of protobuf data file, resource: " << ResourceName); + } + + auto sDesc = TSeverityIds::ESeverityId_descriptor(); + for (int i = 0; i < sDesc->value_count(); i++) { + const auto& name = sDesc->value(i)->name(); + Y_ENSURE(CheckSeverityNameFormat(name), + "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); + } + + for (const auto& x : ProtoIssues_.ids()) { + auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity())); + Y_ENSURE(rv.second, "Duplicate issue code found, code: " + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + } + + // Check all IssueCodes have mapping to severity + auto eDesc = TProto::EIssueCode_descriptor(); + for (int i = 0; i < eDesc->value_count(); i++) { + auto it = IssuesMap_.find(eDesc->value(i)->number()); + Y_ENSURE(it != IssuesMap_.end(), "IssueCode: " + << eDesc->value(i)->name() + << " is not found in protobuf data file"); + } for (const auto& x : ProtoIssues_.ids()) { auto rv = IssuesFormatMap_.insert(std::make_pair(x.code(), x.format())); - Y_ENSURE(rv.second, "Duplicate issue code found, code: " + Y_ENSURE(rv.second, "Duplicate issue code found, code: " << static_cast<int>(x.code()) << "(" << IssueCodeToString<TProto>(x.code()) <<")"); } - } -}; - -template<typename TProto, const char* ResourceName> -inline ESeverity GetSeverity(TIssueCode id) { - return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id); -} - + } +}; + +template<typename TProto, const char* ResourceName> +inline ESeverity GetSeverity(TIssueCode id) { + return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id); +} + template<typename TProto, const char* ResourceName> inline TString GetMessage(TIssueCode id) { return Singleton<TIssueId<TProto, ResourceName>>()->GetMessage(id); -} +} } diff --git a/ydb/library/yql/public/issue/yql_issue_manager.cpp b/ydb/library/yql/public/issue/yql_issue_manager.cpp index 66572c0a9f..7375d2d30f 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager.cpp +++ b/ydb/library/yql/public/issue/yql_issue_manager.cpp @@ -1,21 +1,21 @@ -#include "yql_issue_manager.h" - +#include "yql_issue_manager.h" + #include <util/string/cast.h> #include <util/string/builder.h> -using namespace NYql; - -void TIssueManager::AddScope(std::function<TIssuePtr()> fn) { - RawIssues_.emplace(std::make_pair(TMaybe<TIssuePtr>(), fn)); -} - -void TIssueManager::LeaveScope() { - Y_ASSERT(HasOpenScopes()); - - if (RawIssues_.size() == 1) { - // Last scope, just dump it - auto maybeIssue = RawIssues_.top().first; - if (maybeIssue) { +using namespace NYql; + +void TIssueManager::AddScope(std::function<TIssuePtr()> fn) { + RawIssues_.emplace(std::make_pair(TMaybe<TIssuePtr>(), fn)); +} + +void TIssueManager::LeaveScope() { + Y_ASSERT(HasOpenScopes()); + + if (RawIssues_.size() == 1) { + // Last scope, just dump it + auto maybeIssue = RawIssues_.top().first; + if (maybeIssue) { if ((*maybeIssue)->GetCode() == Max<ui32>()) { for (const auto& nestedIssue : (*maybeIssue.Get())->GetSubIssues()) { CompletedIssues_.AddIssue(*nestedIssue); @@ -23,17 +23,17 @@ void TIssueManager::LeaveScope() { } else { CompletedIssues_.AddIssue(**maybeIssue); } - } - RawIssues_.pop(); - return; - } - - if (RawIssues_.top().first.Empty()) { - // No issues in this scope - RawIssues_.pop(); - return; - } - + } + RawIssues_.pop(); + return; + } + + if (RawIssues_.top().first.Empty()) { + // No issues in this scope + RawIssues_.pop(); + return; + } + auto subIssue = *RawIssues_.top().first; if (subIssue->GetSubIssues().size() == 1) { auto nestedIssue = subIssue->GetSubIssues().front(); @@ -49,16 +49,16 @@ void TIssueManager::LeaveScope() { subIssue->Message = msg; } } - RawIssues_.pop(); - if (RawIssues_.top().first.Empty()) { - RawIssues_.top().first = RawIssues_.top().second(); + RawIssues_.pop(); + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); if (!*RawIssues_.top().first) { RawIssues_.top().first = new TIssue(); (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); } else { (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; } - } + } if (subIssue->GetCode() == Max<ui32>()) { for (const auto& nestedIssue : subIssue->GetSubIssues()) { @@ -67,14 +67,14 @@ void TIssueManager::LeaveScope() { } else { RawIssues_.top().first->Get()->AddSubIssue(subIssue); } -} - -void TIssueManager::LeaveAllScopes() { - while (!RawIssues_.empty()) { - LeaveScope(); - } -} - +} + +void TIssueManager::LeaveAllScopes() { + while (!RawIssues_.empty()) { + LeaveScope(); + } +} + TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) { const auto severity = issue.GetSeverity(); if (OverflowIssues_[severity]) { @@ -94,71 +94,71 @@ TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) { return p; } -void TIssueManager::RaiseIssue(const TIssue& issue) { +void TIssueManager::RaiseIssue(const TIssue& issue) { TIssuePtr p = CheckUniqAndLimit(issue); if (!p) { return; } - if (RawIssues_.empty()) { + if (RawIssues_.empty()) { CompletedIssues_.AddIssue(issue); - return; - } - if (RawIssues_.top().first.Empty()) { - RawIssues_.top().first = RawIssues_.top().second(); + return; + } + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); if (!*RawIssues_.top().first) { RawIssues_.top().first = new TIssue(); (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); } else { (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; } - } + } RawIssues_.top().first->Get()->AddSubIssue(p); -} - +} + void TIssueManager::RaiseIssues(const TIssues& issues) { for (const auto& x : issues) { RaiseIssue(x); } } -bool TIssueManager::RaiseWarning(TIssue issue) { - bool isWarning = true; - if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) { +bool TIssueManager::RaiseWarning(TIssue issue) { + bool isWarning = true; + if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) { const auto action = WarningPolicy_.GetAction(issue.GetCode()); switch (action) { case EWarningAction::DISABLE: - return isWarning; + return isWarning; case EWarningAction::ERROR: - issue.Severity = ESeverity::TSeverityIds_ESeverityId_S_ERROR; - if (WarningToErrorTreatMessage_) { - TIssue newIssue; - newIssue.SetCode(issue.GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); - newIssue.Message = WarningToErrorTreatMessage_.GetRef(); - newIssue.AddSubIssue(new TIssue(issue)); - issue = newIssue; - } - isWarning = false; - break; + issue.Severity = ESeverity::TSeverityIds_ESeverityId_S_ERROR; + if (WarningToErrorTreatMessage_) { + TIssue newIssue; + newIssue.SetCode(issue.GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + newIssue.Message = WarningToErrorTreatMessage_.GetRef(); + newIssue.AddSubIssue(new TIssue(issue)); + issue = newIssue; + } + isWarning = false; + break; case EWarningAction::DEFAULT: break; - default: - Y_ENSURE(false, "Unknown action"); - } - } - - RaiseIssue(issue); - return isWarning; -} - -bool TIssueManager::HasOpenScopes() const { - return !RawIssues_.empty(); -} - + default: + Y_ENSURE(false, "Unknown action"); + } + } + + RaiseIssue(issue); + return isWarning; +} + +bool TIssueManager::HasOpenScopes() const { + return !RawIssues_.empty(); +} + TIssues TIssueManager::GetIssues() { - LeaveAllScopes(); + LeaveAllScopes(); return GetCompletedIssues(); -} - +} + TIssues TIssueManager::GetCompletedIssues() const { TIssues res; for (auto& p: OverflowIssues_) { @@ -176,16 +176,16 @@ void TIssueManager::AddIssues(const TIssues& issues) { CompletedIssues_.AddIssue(*p); } } -} - -void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { +} + +void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { for (auto& issue: issues) { if (auto p = CheckUniqAndLimit(TIssue(pos, issue.Message))) { CompletedIssues_.AddIssue(*p); } } -} - +} + void TIssueManager::Reset(const TIssues& issues) { for (auto& p: OverflowIssues_) { p.Drop(); @@ -196,21 +196,21 @@ void TIssueManager::Reset(const TIssues& issues) { } CompletedIssues_.Clear(); - while (!RawIssues_.empty()) { - RawIssues_.pop(); - } + while (!RawIssues_.empty()) { + RawIssues_.pop(); + } AddIssues(issues); -} - -void TIssueManager::Reset() { - Reset(TIssues()); -} - +} + +void TIssueManager::Reset() { + Reset(TIssues()); +} + void TIssueManager::AddWarningRule(const TWarningRule &rule) { WarningPolicy_.AddRule(rule); -} - -void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) { - WarningToErrorTreatMessage_ = msg; -} +} + +void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) { + WarningToErrorTreatMessage_ = msg; +} diff --git a/ydb/library/yql/public/issue/yql_issue_manager.h b/ydb/library/yql/public/issue/yql_issue_manager.h index 2a2195e592..9ad5ac7bb4 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager.h +++ b/ydb/library/yql/public/issue/yql_issue_manager.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include "yql_issue.h" #include "yql_warning.h" -#include <util/generic/maybe.h> -#include <util/generic/stack.h> +#include <util/generic/maybe.h> +#include <util/generic/stack.h> #include <util/generic/hash_set.h> - + #include <array> - -namespace NYql { - -class TIssueManager: private TNonCopyable { -public: - void AddScope(std::function<TIssuePtr()> fn); - void LeaveScope(); - void LeaveAllScopes(); - void RaiseIssue(const TIssue& issue); + +namespace NYql { + +class TIssueManager: private TNonCopyable { +public: + void AddScope(std::function<TIssuePtr()> fn); + void LeaveScope(); + void LeaveAllScopes(); + void RaiseIssue(const TIssue& issue); void RaiseIssues(const TIssues& issues); - bool RaiseWarning(TIssue issue); - void AddIssues(const TIssues& errors); - void AddIssues(const TPosition& pos, const TIssues& issues); - bool HasOpenScopes() const; - + bool RaiseWarning(TIssue issue); + void AddIssues(const TIssues& errors); + void AddIssues(const TPosition& pos, const TIssues& issues); + bool HasOpenScopes() const; + TIssues GetIssues(); TIssues GetCompletedIssues() const; - + void Reset(const TIssues& issues); - void Reset(); + void Reset(); void AddWarningRule(const TWarningRule &rule); - void SetWarningToErrorTreatMessage(const TString& msg); + void SetWarningToErrorTreatMessage(const TString& msg); void SetIssueCountLimit(size_t limit) { IssueLimit_ = limit; } -private: +private: TIssuePtr CheckUniqAndLimit(const TIssue& issue); struct TIssueHash { @@ -50,26 +50,26 @@ private: } }; TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_; - TIssues CompletedIssues_; - TMaybe<TString> WarningToErrorTreatMessage_; + TIssues CompletedIssues_; + TMaybe<TString> WarningToErrorTreatMessage_; TWarningPolicy WarningPolicy_; std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_; std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_; size_t IssueLimit_ = 0; -}; - -class TIssueScopeGuard: private TNonCopyable { - TIssueManager& Manager_; -public: - TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn) - : Manager_(manager) - { - Manager_.AddScope(fn); - } - ~TIssueScopeGuard() - { - Manager_.LeaveScope(); - } -}; - -} +}; + +class TIssueScopeGuard: private TNonCopyable { + TIssueManager& Manager_; +public: + TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn) + : Manager_(manager) + { + Manager_.AddScope(fn); + } + ~TIssueScopeGuard() + { + Manager_.LeaveScope(); + } +}; + +} diff --git a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp index aa4b741661..287f63fedd 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp +++ b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp @@ -1,132 +1,132 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "yql_issue_manager.h" - -using namespace NYql; - + +#include "yql_issue_manager.h" + +using namespace NYql; + static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) { return [name, column, row]() { return new TIssue(TPosition(column, row), name); - }; -} - + }; +} + Y_UNIT_TEST_SUITE(TIssueManagerTest) { Y_UNIT_TEST(NoErrorNoLevelTest) { - TIssueManager issueManager; + TIssueManager issueManager; auto completedIssues = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } - + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + Y_UNIT_TEST(NoErrorOneLevelTest) { - TIssueManager issueManager; + TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); auto completedIssues = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } - + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + Y_UNIT_TEST(NoErrorTwoLevelsTest) { - TIssueManager issueManager; + TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.LeaveScope(); + issueManager.LeaveScope(); auto completedIssues = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } - + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + Y_UNIT_TEST(OneErrorOneLevelTest) { - TIssueManager issueManager; + TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); auto completedIssues1 = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); auto completedIssues2 = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); - issueManager.LeaveScope(); + issueManager.LeaveScope(); auto completedIssues3 = issueManager.GetCompletedIssues(); UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "IssueOne"); - } - + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "IssueOne"); + } + Y_UNIT_TEST(OneErrorTwoLevelsTest) { - TIssueManager issueManager; + TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "B"); - - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->Message, "IssueOne"); - } - + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "B"); + + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->Message, "IssueOne"); + } + Y_UNIT_TEST(MultiErrorsMultiLevelsTest) { - TIssueManager issueManager; + TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); + issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2)); - issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); - issueManager.LeaveScope(); - issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1 - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "WarningScope1"); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->Message, "ErrorScope3"); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->Message, "ErrorScope2"); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1 + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "WarningScope1"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->Message, "ErrorScope3"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->Message, "ErrorScope2"); auto ref = R"___(<main>: Error: A <main>: Error: WarningScope1 <main>:1:1: Error: B <main>:2:2: Error: C <main>: Error: ErrorScope3 <main>: Error: ErrorScope2 -)___"; - UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); - } - +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + Y_UNIT_TEST(TIssueScopeGuardSimpleTest) { - TIssueManager issueManager; - { + TIssueManager issueManager; + { TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1")); - } - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "ErrorScope1"); - } + issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1")); + } + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->Message, "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "ErrorScope1"); + } Y_UNIT_TEST(FuseScopesTest) { TIssueManager issueManager; @@ -203,4 +203,4 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) { )___"; UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); } -} +} diff --git a/ydb/library/yql/public/issue/yql_issue_message.cpp b/ydb/library/yql/public/issue/yql_issue_message.cpp index a94806f5ce..f9d512ca5a 100644 --- a/ydb/library/yql/public/issue/yql_issue_message.cpp +++ b/ydb/library/yql/public/issue/yql_issue_message.cpp @@ -5,47 +5,47 @@ #include <util/generic/deque.h> #include <util/generic/yexception.h> - + #include <tuple> - -namespace NYql { - -using namespace NIssue::NProto; - -template<typename TIssueMessage> -TIssue IssueFromMessage(const TIssueMessage& issueMessage) { - TIssue topIssue; + +namespace NYql { + +using namespace NIssue::NProto; + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage) { + TIssue topIssue; TDeque<std::pair<TIssue*, const TIssueMessage*>> queue; - queue.push_front(std::make_pair(&topIssue, &issueMessage)); - while (!queue.empty()) { - TIssue& issue = *queue.back().first; - const auto& message = *queue.back().second; - queue.pop_back(); - TPosition position(message.position().column(), message.position().row(), message.position().file()); - TPosition endPosition(message.end_position().column(), message.end_position().row()); - if (position.HasValue()) { - if (endPosition.HasValue()) { - issue = TIssue(position, endPosition, message.message()); - } else { - issue = TIssue(position, message.message()); - } - } else { - issue = TIssue(message.message()); - } - + queue.push_front(std::make_pair(&topIssue, &issueMessage)); + while (!queue.empty()) { + TIssue& issue = *queue.back().first; + const auto& message = *queue.back().second; + queue.pop_back(); + TPosition position(message.position().column(), message.position().row(), message.position().file()); + TPosition endPosition(message.end_position().column(), message.end_position().row()); + if (position.HasValue()) { + if (endPosition.HasValue()) { + issue = TIssue(position, endPosition, message.message()); + } else { + issue = TIssue(position, message.message()); + } + } else { + issue = TIssue(message.message()); + } + for (const auto& subMessage : message.issues()) { - auto subIssue = new TIssue(); - issue.AddSubIssue(subIssue); - queue.push_front(std::make_pair(subIssue, &subMessage)); - } - - issue.SetCode(message.issue_code(), static_cast<ESeverity>(message.severity())); - } - return topIssue; -} - -template<typename TIssueMessage> -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) { + auto subIssue = new TIssue(); + issue.AddSubIssue(subIssue); + queue.push_front(std::make_pair(subIssue, &subMessage)); + } + + issue.SetCode(message.issue_code(), static_cast<ESeverity>(message.severity())); + } + return topIssue; +} + +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) { issues.Clear(); if (message.size()) { issues.Reserve(message.size()); @@ -54,7 +54,7 @@ void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> } } -template<typename TIssueMessage> +template<typename TIssueMessage> void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { TDeque<std::pair<const TIssue*, TIssueMessage*>> queue; queue.push_front(std::make_pair(&topIssue, issueMessage)); @@ -81,10 +81,10 @@ void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { TIssueMessage* subMessage = message.add_issues(); queue.push_front(std::make_pair(subIssue.Get(), subMessage)); } - } -} - -template<typename TIssueMessage> + } +} + +template<typename TIssueMessage> void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) { message->Clear(); if (!issues) @@ -93,48 +93,48 @@ void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField for (const auto &issue : issues) { IssueToMessage(issue, message->Add()); } -} - -template -TIssue IssueFromMessage<Ydb::Issue::IssueMessage>(const Ydb::Issue::IssueMessage& issueMessage); -template -TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); - -template +} + +template +TIssue IssueFromMessage<Ydb::Issue::IssueMessage>(const Ydb::Issue::IssueMessage& issueMessage); +template +TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); + +template void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues); -template +template void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues); - -template + +template void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage); -template +template void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); - -template + +template void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message); -template +template void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message); - + NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { - NIssue::NProto::IssueMessage issueMessage; + NIssue::NProto::IssueMessage issueMessage; IssueToMessage(topIssue, &issueMessage); - return issueMessage; + return issueMessage; } - + TString IssueToBinaryMessage(const TIssue& issue) { - TString result; - Ydb::Issue::IssueMessage protobuf; + TString result; + Ydb::Issue::IssueMessage protobuf; IssueToMessage(issue, &protobuf); Y_PROTOBUF_SUPPRESS_NODISCARD protobuf.SerializeToString(&result); - return result; -} - -TIssue IssueFromBinaryMessage(const TString& binaryMessage) { - Ydb::Issue::IssueMessage protobuf; - if (!protobuf.ParseFromString(binaryMessage)) { - ythrow yexception() << "unable to parse binary string as issue protobuf"; - } - return IssueFromMessage(protobuf); -} - -} + return result; +} + +TIssue IssueFromBinaryMessage(const TString& binaryMessage) { + Ydb::Issue::IssueMessage protobuf; + if (!protobuf.ParseFromString(binaryMessage)) { + ythrow yexception() << "unable to parse binary string as issue protobuf"; + } + return IssueFromMessage(protobuf); +} + +} diff --git a/ydb/library/yql/public/issue/yql_issue_message.h b/ydb/library/yql/public/issue/yql_issue_message.h index 4d112e4f51..b9d1ba529c 100644 --- a/ydb/library/yql/public/issue/yql_issue_message.h +++ b/ydb/library/yql/public/issue/yql_issue_message.h @@ -1,30 +1,30 @@ -#pragma once - -#include "yql_issue.h" - +#pragma once + +#include "yql_issue.h" + #include <util/generic/ylimits.h> -namespace NYql { - -namespace NIssue { -namespace NProto { -class IssueMessage; -} -} - -template<typename TIssueMessage> -TIssue IssueFromMessage(const TIssueMessage& issueMessage); -template<typename TIssueMessage> -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues); - +namespace NYql { + +namespace NIssue { +namespace NProto { +class IssueMessage; +} +} + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage); +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues); + NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue); - -template<typename TIssueMessage> + +template<typename TIssueMessage> void IssueToMessage(const TIssue& topIssue, TIssueMessage* message); -template<typename TIssueMessage> +template<typename TIssueMessage> void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message); - + TString IssueToBinaryMessage(const TIssue& issue); -TIssue IssueFromBinaryMessage(const TString& binaryMessage); - -} +TIssue IssueFromBinaryMessage(const TString& binaryMessage); + +} diff --git a/ydb/library/yql/public/issue/yql_issue_ut.cpp b/ydb/library/yql/public/issue/yql_issue_ut.cpp index 64efaa990f..87b417da39 100644 --- a/ydb/library/yql/public/issue/yql_issue_ut.cpp +++ b/ydb/library/yql/public/issue/yql_issue_ut.cpp @@ -4,45 +4,45 @@ #include <ydb/library/yql/public/issue/protos/issue_message.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/public/api/protos/ydb_issue_message.pb.h> - + #include <google/protobuf/message.h> #include <google/protobuf/descriptor.h> - -using namespace google::protobuf; + +using namespace google::protobuf; using namespace NYql; - -void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes); -void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) { - UNIT_ASSERT(a); - UNIT_ASSERT(b); - - UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type())); - UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label()); - UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); - UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number()); - UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated()); - UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed()); - UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index()); - if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) { - ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes); - } -} - -void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes) { - UNIT_ASSERT(a); - UNIT_ASSERT(b); - if (!visitedTypes->insert(a->name()).second) { - return; - } - - UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); - UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count()); - - for (int i = 0; i < a->field_count(); i++) { - ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes); - } -} - + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes); +void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + + UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type())); + UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label()); + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number()); + UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated()); + UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed()); + UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index()); + if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) { + ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes); + } +} + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + if (!visitedTypes->insert(a->name()).second) { + return; + } + + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count()); + + for (int i = 0; i < a->field_count(); i++) { + ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes); + } +} + Y_UNIT_TEST_SUITE(IssueTest) { Y_UNIT_TEST(Ascii) { TIssue issue1("тест abc"); @@ -56,25 +56,25 @@ Y_UNIT_TEST_SUITE(IssueTest) { } } -Y_UNIT_TEST_SUITE(IssueProtoTest) { - Y_UNIT_TEST(KikimrYqlSameLayout) { - Ydb::Issue::IssueMessage yqlIssue; - NYql::NIssue::NProto::IssueMessage kikimrIssue; - THashSet<TString> visitedTypes; - ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes); - } - - Y_UNIT_TEST(BinarySerialization) { - TIssue issueTo("root_issue"); - TString bin = IssueToBinaryMessage(issueTo); - TIssue issueFrom = IssueFromBinaryMessage(bin); - UNIT_ASSERT_EQUAL(issueTo, issueFrom); - } - - Y_UNIT_TEST(WrongBinStringException) { - UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); - } -} +Y_UNIT_TEST_SUITE(IssueProtoTest) { + Y_UNIT_TEST(KikimrYqlSameLayout) { + Ydb::Issue::IssueMessage yqlIssue; + NYql::NIssue::NProto::IssueMessage kikimrIssue; + THashSet<TString> visitedTypes; + ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes); + } + + Y_UNIT_TEST(BinarySerialization) { + TIssue issueTo("root_issue"); + TString bin = IssueToBinaryMessage(issueTo); + TIssue issueFrom = IssueFromBinaryMessage(bin); + UNIT_ASSERT_EQUAL(issueTo, issueFrom); + } + + Y_UNIT_TEST(WrongBinStringException) { + UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); + } +} Y_UNIT_TEST_SUITE(TextWalkerTest) { diff --git a/ydb/library/yql/public/types/yql_types.proto b/ydb/library/yql/public/types/yql_types.proto index a701259485..f9ebaf1390 100644 --- a/ydb/library/yql/public/types/yql_types.proto +++ b/ydb/library/yql/public/types/yql_types.proto @@ -1,9 +1,9 @@ -syntax = "proto3"; +syntax = "proto3"; package NYql.NProto; option java_package = "ru.yandex.yql.proto"; -enum TypeIds { - UNUSED = 0x0000; +enum TypeIds { + UNUSED = 0x0000; Bool = 0x0006; Int8 = 0x0007; Uint8 = 0x0005; diff --git a/ydb/library/yql/public/udf/udf_value.h b/ydb/library/yql/public/udf/udf_value.h index 824b65fb0c..705ac41133 100644 --- a/ydb/library/yql/public/udf/udf_value.h +++ b/ydb/library/yql/public/udf/udf_value.h @@ -6,7 +6,7 @@ #include "udf_version.h" #include <ydb/library/yql/public/decimal/yql_decimal.h> - + #include <util/system/yassert.h> // FAIL, VERIFY_DEBUG #include <util/generic/utility.h> // Min, Max #include <util/generic/yexception.h> // Y_ENSURE diff --git a/ydb/library/yql/public/udf/udf_value_builder.h b/ydb/library/yql/public/udf/udf_value_builder.h index 292a05264b..096f735a2b 100644 --- a/ydb/library/yql/public/udf/udf_value_builder.h +++ b/ydb/library/yql/public/udf/udf_value_builder.h @@ -144,12 +144,12 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 14) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 14) class IValueBuilder3: public IValueBuilder2 { public: - virtual bool GetSecureParam(TStringRef key, TStringRef& value) const = 0; + virtual bool GetSecureParam(TStringRef key, TStringRef& value) const = 0; }; -#endif +#endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 17) class IValueBuilder4: public IValueBuilder3 { diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g index 55576429fb..3b616b9c9f 100644 --- a/ydb/library/yql/sql/v0/SQL.g +++ b/ydb/library/yql/sql/v0/SQL.g @@ -32,7 +32,7 @@ sql_stmt_core: | declare_stmt | import_stmt | export_stmt - | alter_table_stmt + | alter_table_stmt | do_stmt | define_action_or_subquery_stmt | evaluate_if_stmt @@ -299,12 +299,12 @@ simple_values_source: expr_list | select_stmt; create_table_stmt: CREATE TABLE simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN; create_table_entry: column_schema | table_constraint; - + alter_table_stmt: ALTER TABLE simple_table_ref alter_table_action; -alter_table_action: alter_table_add_column | alter_table_drop_column; -alter_table_add_column: ADD COLUMN? column_schema (COMMA ADD COLUMN? column_schema)*; +alter_table_action: alter_table_add_column | alter_table_drop_column; +alter_table_add_column: ADD COLUMN? column_schema (COMMA ADD COLUMN? column_schema)*; alter_table_drop_column: DROP COLUMN? id; - + column_schema: id_schema flex_type (NOT? NULL)?; column_order_by_specification: id (ASC | DESC)?; @@ -470,7 +470,7 @@ keyword_in_uncompat: keyword_compat: ( ABORT | ACTION - | ADD + | ADD | AFTER | ALTER | ANALYZE @@ -699,7 +699,7 @@ fragment Z:('z'|'Z'); ABORT: A B O R T; ACTION: A C T I O N; -ADD: A D D; +ADD: A D D; AFTER: A F T E R; ALL: A L L; ALTER: A L T E R; diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp index 961ba34db2..a0945f9d05 100644 --- a/ydb/library/yql/sql/v0/builtin.cpp +++ b/ydb/library/yql/sql/v0/builtin.cpp @@ -418,7 +418,7 @@ public: return nullptr; } - auto parsed = ParseType(*literal, *ctx.Pool, ctx.Issues, Args[0]->GetPos()); + auto parsed = ParseType(*literal, *ctx.Pool, ctx.Issues, Args[0]->GetPos()); if (!parsed) { ctx.Error(Args[0]->GetPos()) << "Failed to parse type"; return nullptr; @@ -599,9 +599,9 @@ TNodePtr BuildFileNameArgument(TPosition pos, const TNodePtr& argument) { return new TLiteralStringAtom(pos, argument, "FilePath requires string literal as parameter"); } -class TYqlAtom final: public TCallNode { +class TYqlAtom final: public TCallNode { public: - TYqlAtom(TPosition pos, const TString& opName, const TVector<TNodePtr>& args) + TYqlAtom(TPosition pos, const TString& opName, const TVector<TNodePtr>& args) : TCallNode(pos, opName, 1, 1, args) {} @@ -613,7 +613,7 @@ public: } TNodePtr DoClone() const final { - return new TYqlAtom(Pos, OpName, Args); + return new TYqlAtom(Pos, OpName, Args); } }; @@ -1909,7 +1909,7 @@ struct TBuiltinFuncData { // Atom builtins {"asatom", BuildSimpleBuiltinFactoryCallback<TYqlAsAtom>()}, - {"secureparam", BuildNamedBuiltinFactoryCallback<TYqlAtom>("SecureParam")}, + {"secureparam", BuildNamedBuiltinFactoryCallback<TYqlAtom>("SecureParam")}, {"void", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Void", 0, 0)}, {"callable", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Callable", 2, 2)}, @@ -2007,10 +2007,10 @@ struct TBuiltinFuncData { {"flattenmembers", BuildNamedBuiltinFactoryCallback<TFlattenMembers>("FlattenMembers")}, // File builtins - {"filepath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FilePath")}, - {"filecontent", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FileContent")}, - {"folderpath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FolderPath") }, - {"files", BuildNamedBuiltinFactoryCallback<TYqlAtom>("Files")}, + {"filepath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FilePath")}, + {"filecontent", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FileContent")}, + {"folderpath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FolderPath") }, + {"files", BuildNamedBuiltinFactoryCallback<TYqlAtom>("Files")}, {"parsefile", BuildSimpleBuiltinFactoryCallback<TYqlParseFileOp>()}, // Misc builtins @@ -2192,7 +2192,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec TString normalizedName(name); TString ns = to_lower(nameSpace); if (ns.empty()) { - TMaybe<TIssue> error = NormalizeName(pos, normalizedName); + TMaybe<TIssue> error = NormalizeName(pos, normalizedName); if (!error.Empty()) { return new TInvalidBuiltin(pos, error->Message); } diff --git a/ydb/library/yql/sql/v0/context.cpp b/ydb/library/yql/sql/v0/context.cpp index 98763d6261..a3a9013f2b 100644 --- a/ydb/library/yql/sql/v0/context.cpp +++ b/ydb/library/yql/sql/v0/context.cpp @@ -45,7 +45,7 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { } // namespace TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, - TIssues& issues) + TIssues& issues) : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) @@ -54,7 +54,7 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, , Issues(issues) , IncrementMonCounterFunction(settings.IncrementCounter) , CurrCluster(settings.DefaultCluster) - , HasPendingErrors(false) + , HasPendingErrors(false) , Libraries(settings.Libraries) { Position.File = settings.File; @@ -97,7 +97,7 @@ IOutputStream& TContext::Error() { } IOutputStream& TContext::Error(NYql::TPosition pos) { - HasPendingErrors = true; + HasPendingErrors = true; return MakeIssue(TSeverityIds::S_ERROR, TIssuesIds::DEFAULT_ERROR, pos); } diff --git a/ydb/library/yql/sql/v0/context.h b/ydb/library/yql/sql/v0/context.h index aa93af7bc6..45f40fae61 100644 --- a/ydb/library/yql/sql/v0/context.h +++ b/ydb/library/yql/sql/v0/context.h @@ -23,7 +23,7 @@ namespace NSQLTranslationV0 { class TContext { public: TContext(const NSQLTranslation::TTranslationSettings& settings, - NYql::TIssues& issues); + NYql::TIssues& issues); virtual ~TContext(); @@ -128,7 +128,7 @@ namespace NSQLTranslationV0 { THashMap<TString, TNodePtr> Variables; NSQLTranslation::TTranslationSettings Settings; std::unique_ptr<TMemoryPool> Pool; - NYql::TIssues& Issues; + NYql::TIssues& Issues; TMap<TString, TStack<TNodePtr>> NamedNodes; TMap<TString, TNodePtr> UniversalAliases; THashSet<TString> Exports; @@ -136,7 +136,7 @@ namespace NSQLTranslationV0 { TMap<TString, TString> SimpleUdfs; NSQLTranslation::TIncrementMonCounterFunction IncrementMonCounterFunction; TString CurrCluster; - bool HasPendingErrors; + bool HasPendingErrors; THashMap<TString, ui32> GenIndexes; bool PragmaRefSelect = false; bool PragmaSampleSelect = false; diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index c0ef9a2d8a..4c7cfc273b 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -62,11 +62,11 @@ namespace NSQLTranslationV0 { DeleteOn, }; - enum class EAlterTableIntentnt { - AddColumn, - DropColumn - }; - + enum class EAlterTableIntentnt { + AddColumn, + DropColumn + }; + class TContext; class ITableKeys; class ISource; diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 61a7493873..7752b28b01 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -327,7 +327,7 @@ public: Y_UNUSED(src); TSet<TString> used; for (auto& hint: Hints) { - TMaybe<TIssue> normalizeError = NormalizeName(Pos, hint); + TMaybe<TIssue> normalizeError = NormalizeName(Pos, hint); if (!normalizeError.Empty()) { ctx.Error() << normalizeError->Message; ctx.IncrementMonCounter("sql_errors", "NormalizeHintError"); @@ -575,94 +575,94 @@ TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TVector<TCol return new TCreateTableNode(pos, tr, columns, pkColumns, partitionByColumns, orderByColumns); } -class TAlterTableNode final: public TAstListNode { -public: +class TAlterTableNode final: public TAstListNode { +public: TAlterTableNode(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, EAlterTableIntentnt intent) - : TAstListNode(pos) - , Table(tr) - , Columns(columns) - , Intent(intent) - {} - bool DoInit(TContext& ctx, ISource* src) override { - if (!Table.Check(ctx)) { - return false; - } - - auto keys = Table.Keys->GetTableKeys()->BuildKeys(ctx, ITableKeys::EBuildKeysMode::CREATE); - ctx.PushBlockShortcuts(); - if (!keys || !keys->Init(ctx, src)) { - return false; - } - keys = ctx.GroundBlockShortcutsForExpr(keys); - + : TAstListNode(pos) + , Table(tr) + , Columns(columns) + , Intent(intent) + {} + bool DoInit(TContext& ctx, ISource* src) override { + if (!Table.Check(ctx)) { + return false; + } + + auto keys = Table.Keys->GetTableKeys()->BuildKeys(ctx, ITableKeys::EBuildKeysMode::CREATE); + ctx.PushBlockShortcuts(); + if (!keys || !keys->Init(ctx, src)) { + return false; + } + keys = ctx.GroundBlockShortcutsForExpr(keys); + auto actions = Y(); - if (Intent == EAlterTableIntentnt::DropColumn) { + if (Intent == EAlterTableIntentnt::DropColumn) { auto columns = Y(); for (auto& col : Columns) { columns = L(columns, BuildQuotedAtom(Pos, col.Name)); - } + } actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); - } else { + } else { auto columns = Y(); - for (auto& col: Columns) { + for (auto& col: Columns) { auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); - if (!type) { - return false; - } - Y_ASSERT(type->IsList()); - Y_ASSERT(type->GetChildrenCount() > 1); - auto typeName = type->GetChild(0); - Y_ASSERT(typeName->IsAtom()); - if (typeName->GetContent() == "OptionalType") { - ctx.Error(col.Pos) << "ALTER TABLE clause requires non-optional column types in scheme"; - return false; - } - if (col.Nullable) { - type = TAstNode::NewList( - col.Pos, - *ctx.Pool, - TAstNode::NewLiteralAtom( - col.Pos, - "OptionalType", - *ctx.Pool - ), - type - ); - } - columns = L(columns, Q(Y(BuildQuotedAtom(Pos, col.Name), AstNode(type)))); - } + if (!type) { + return false; + } + Y_ASSERT(type->IsList()); + Y_ASSERT(type->GetChildrenCount() > 1); + auto typeName = type->GetChild(0); + Y_ASSERT(typeName->IsAtom()); + if (typeName->GetContent() == "OptionalType") { + ctx.Error(col.Pos) << "ALTER TABLE clause requires non-optional column types in scheme"; + return false; + } + if (col.Nullable) { + type = TAstNode::NewList( + col.Pos, + *ctx.Pool, + TAstNode::NewLiteralAtom( + col.Pos, + "OptionalType", + *ctx.Pool + ), + type + ); + } + columns = L(columns, Q(Y(BuildQuotedAtom(Pos, col.Name), AstNode(type)))); + } actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); - } - + } + auto opts = Y(); - opts = L(opts, Q(Y(Q("mode"), Q("alter")))); + opts = L(opts, Q(Y(Q("mode"), Q("alter")))); opts = L(opts, Q(Y(Q("actions"), Q(actions)))); - - Add("block", Q(Y( - Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.ServiceName(ctx)), BuildQuotedAtom(Pos, Table.Cluster))), + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.ServiceName(ctx)), BuildQuotedAtom(Pos, Table.Cluster))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - ctx.UsedClusters.insert(Table.Cluster); - return TAstListNode::DoInit(ctx, src); - } - TPtr DoClone() const final { - return {}; - } -private: - TTableRef Table; + ))); + + ctx.UsedClusters.insert(Table.Cluster); + return TAstListNode::DoInit(ctx, src); + } + TPtr DoClone() const final { + return {}; + } +private: + TTableRef Table; TVector<TColumnSchema> Columns; - EAlterTableIntentnt Intent; -}; - + EAlterTableIntentnt Intent; +}; + TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, EAlterTableIntentnt intent) -{ - return new TAlterTableNode(pos, tr, columns, intent); -} - +{ + return new TAlterTableNode(pos, tr, columns, intent); +} + class TDropTableNode final: public TAstListNode { public: TDropTableNode(TPosition pos, const TTableRef& tr) @@ -990,7 +990,7 @@ public: BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); } - + if (ctx.ResultSizeLimit > 0) { Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink, BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit))))); diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index 116c4d46e9..b028fc0439 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -41,7 +41,7 @@ using NALP::SQLLexerTokens; #endif using namespace NSQLGenerated; - + static TPosition GetPos(const TToken& token) { return TPosition(token.GetColumn(), token.GetLine()); } @@ -68,26 +68,26 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword_restricte } inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { - switch (node.Alt_case()) { + switch (node.Alt_case()) { case TRule_keyword::kAltKeyword1: return GetKeywordId(ctx, node.GetAlt_keyword1().GetRule_keyword_restricted1()); case TRule_keyword::kAltKeyword2: return GetIdentifier(ctx, node.GetAlt_keyword2().GetRule_keyword_alter_uncompat1()); case TRule_keyword::kAltKeyword3: return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); - default: - Y_FAIL("You should change implementation according grammar changes"); - } -} - + default: + Y_FAIL("You should change implementation according grammar changes"); + } +} + inline TString GetKeyword(TTranslation& ctx, const TRule_keyword& node) { return GetKeywordId(ctx, node).Name; } -inline TString GetKeyword(TTranslation& ctx, const TRule_keyword_restricted& node) { - return GetKeywordId(ctx, node).Name; -} - +inline TString GetKeyword(TTranslation& ctx, const TRule_keyword_restricted& node) { + return GetKeywordId(ctx, node).Name; +} + static TString Id(const TRule_id& node, TTranslation& ctx) { // id: IDENTIFIER | keyword; switch (node.Alt_case()) { @@ -101,16 +101,16 @@ static TString Id(const TRule_id& node, TTranslation& ctx) { } static TString Id(const TRule_id_schema& node, TTranslation& ctx) { - switch (node.Alt_case()) { + switch (node.Alt_case()) { case TRule_id_schema::kAltIdSchema1: return ctx.Identifier(node.GetAlt_id_schema1().GetToken1()); case TRule_id_schema::kAltIdSchema2: return GetKeyword(ctx, node.GetAlt_id_schema2().GetRule_keyword_restricted1()); - default: - Y_FAIL("You should change implementation according grammar changes"); - } -} - + default: + Y_FAIL("You should change implementation according grammar changes"); + } +} + static std::pair<bool, TString> Id(const TRule_id_or_at& node, TTranslation& ctx) { bool hasAt = node.HasBlock1(); return std::make_pair(hasAt, Id(node.GetRule_id2(), ctx) ); @@ -4431,32 +4431,32 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } break; - case TRule_sql_stmt_core::kAltSqlStmtCore15: { - Ctx.BodyPart(); - const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1(); + case TRule_sql_stmt_core::kAltSqlStmtCore15: { + Ctx.BodyPart(); + const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1(); TTableRef tr(SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), Mode, *this)); - const auto& ruleAction = rule.GetRule_alter_table_action4(); - switch (ruleAction.Alt_case()) { - case TRule_alter_table_action::kAltAlterTableAction1: { - const auto& addRule = ruleAction.GetAlt_alter_table_action1().GetRule_alter_table_add_column1(); - if (!AlterTableAddColumns(blocks, addRule, tr)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction2: { - const auto& dropRule = ruleAction.GetAlt_alter_table_action2().GetRule_alter_table_drop_column1(); - if (!AlterTableDropColumn(blocks, dropRule, tr)) { - return false; - } - break; - } - default: - AltNotImplemented("alter_table_action", core); - return false; - } - break; - } + const auto& ruleAction = rule.GetRule_alter_table_action4(); + switch (ruleAction.Alt_case()) { + case TRule_alter_table_action::kAltAlterTableAction1: { + const auto& addRule = ruleAction.GetAlt_alter_table_action1().GetRule_alter_table_add_column1(); + if (!AlterTableAddColumns(blocks, addRule, tr)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction2: { + const auto& dropRule = ruleAction.GetAlt_alter_table_action2().GetRule_alter_table_drop_column1(); + if (!AlterTableDropColumn(blocks, dropRule, tr)) { + return false; + } + break; + } + default: + AltNotImplemented("alter_table_action", core); + return false; + } + break; + } case TRule_sql_stmt_core::kAltSqlStmtCore16: { Ctx.BodyPart(); auto node = DoStatement(core.GetAlt_sql_stmt_core16().GetRule_do_stmt1(), false); @@ -4502,7 +4502,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } Ctx.IncrementMonCounter("sql_features", internalStatementName); - return !Ctx.HasPendingErrors; + return !Ctx.HasPendingErrors; } bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) { @@ -4549,30 +4549,30 @@ bool TSqlQuery::ExportStatement(const TRule_export_stmt& stmt) { bool TSqlQuery::AlterTableAddColumns(TVector<TNodePtr>& blocks, const TRule_alter_table_add_column& rule, const TTableRef& tr) { TVector<TColumnSchema> columns; - - columns.push_back(ColumnSchemaImpl(rule.GetRule_column_schema3(), *this)); - for (const auto& block: rule.GetBlock4()) { - columns.push_back(ColumnSchemaImpl(block.GetRule_column_schema4(), *this)); - } - - AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, columns, EAlterTableIntentnt::AddColumn)); - return true; -} - + + columns.push_back(ColumnSchemaImpl(rule.GetRule_column_schema3(), *this)); + for (const auto& block: rule.GetBlock4()) { + columns.push_back(ColumnSchemaImpl(block.GetRule_column_schema4(), *this)); + } + + AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, columns, EAlterTableIntentnt::AddColumn)); + return true; +} + bool TSqlQuery::AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alter_table_drop_column& node, const TTableRef& tr) { TString name = Id(node.GetRule_id3(), *this); TColumnSchema column(Ctx.Pos(), name, "", false, false); AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, TVector<TColumnSchema>{column}, EAlterTableIntentnt::DropColumn)); - return true; -} - + return true; +} + TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) { success = false; const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix2(), *this); const TString& lowerPrefix = to_lower(prefix); const TString pragma(IdOrString(stmt.GetRule_id_or_string3(), *this)); TString normalizedPragma(pragma); - TMaybe<TIssue> normalizeError = NormalizeName(Ctx.Pos(), normalizedPragma); + TMaybe<TIssue> normalizeError = NormalizeName(Ctx.Pos(), normalizedPragma); if (!normalizeError.Empty()) { Error() << normalizeError->Message; Ctx.IncrementMonCounter("sql_errors", "NormalizePragmaError"); @@ -4730,7 +4730,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.ResultRowsLimit)) { Error() << "Expected single unsigned integer argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; + return {}; } Ctx.IncrementMonCounter("sql_pragma", "ResultRowsLimit"); @@ -4738,16 +4738,16 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.ResultSizeLimit)) { Error() << "Expected single unsigned integer argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; + return {}; } Ctx.IncrementMonCounter("sql_pragma", "ResultSizeLimit"); - } else if (normalizedPragma == "warning") { + } else if (normalizedPragma == "warning") { if (values.size() != 2U || values.front().Empty() || values.back().Empty()) { - Error() << "Expected arguments <action>, <issueId> for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } + Error() << "Expected arguments <action>, <issueId> for: " << pragma; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } TString codePattern = *values[1].GetLiteral(); TString action = *values[0].GetLiteral(); @@ -4775,7 +4775,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Y_ENSURE(false, "Unknown parse result"); } - Ctx.IncrementMonCounter("sql_pragma", "warning"); + Ctx.IncrementMonCounter("sql_pragma", "warning"); } else if (normalizedPragma == "greetings") { if (values.size() > 1) { Error() << "Not expect few arguments for: " << pragma; @@ -5269,11 +5269,11 @@ TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message& protoAst, TContext& ctx) { - YQL_ENSURE(!ctx.Issues.Size()); + YQL_ENSURE(!ctx.Issues.Size()); res.Root = SqlASTToYql(protoAst, ctx); res.Pool = std::move(ctx.Pool); if (!res.Root) { - if (ctx.Issues.Size()) { + if (ctx.Issues.Size()) { ctx.IncrementMonCounter("sql_errors", "AstToYqlError"); } else { ctx.IncrementMonCounter("sql_errors", "AstToYqlSilentError"); diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 3dc27809f3..3a70aef93c 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -789,24 +789,24 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); } - Y_UNIT_TEST(ProcessUserTypeAuth) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData($ROWS, AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "YDB.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("inputRowsList")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); - } - }; - - TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); - } - + Y_UNIT_TEST(ProcessUserTypeAuth) { + NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData($ROWS, AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "YDB.PushData") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("inputRowsList")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); + } + }; + + TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); + } + Y_UNIT_TEST(SelectStreamRtmr) { NYql::TAstParseResult res = SqlToYql( "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 01e3383eea..ef394c7a00 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -33,7 +33,7 @@ sql_stmt_core: | declare_stmt | import_stmt | export_stmt - | alter_table_stmt + | alter_table_stmt | do_stmt | define_action_or_subquery_stmt | if_stmt @@ -460,7 +460,7 @@ create_table_entry: with_table_settings: WITH LPAREN table_settings_entry (COMMA table_settings_entry)* RPAREN; table_settings_entry: an_id EQUALS table_setting_value; - + alter_table_stmt: ALTER TABLE simple_table_ref alter_table_action (COMMA alter_table_action)*; alter_table_action: alter_table_add_column @@ -471,8 +471,8 @@ alter_table_action: | alter_table_set_table_setting_uncompat | alter_table_set_table_setting_compat | alter_table_reset_table_setting - | alter_table_add_index - | alter_table_drop_index + | alter_table_add_index + | alter_table_drop_index | alter_table_rename_to | alter_table_add_changefeed | alter_table_alter_changefeed @@ -487,13 +487,13 @@ alter_table_alter_column_family: ALTER FAMILY an_id SET an_id family_setting_val alter_table_set_table_setting_uncompat: SET an_id table_setting_value; alter_table_set_table_setting_compat: SET LPAREN alter_table_setting_entry (COMMA alter_table_setting_entry)* RPAREN; alter_table_reset_table_setting: RESET LPAREN an_id (COMMA an_id)* RPAREN; -alter_table_add_index: ADD table_index; -alter_table_drop_index: DROP INDEX an_id; +alter_table_add_index: ADD table_index; +alter_table_drop_index: DROP INDEX an_id; alter_table_rename_to: RENAME TO an_id_table; alter_table_add_changefeed: ADD changefeed; alter_table_alter_changefeed: ALTER CHANGEFEED an_id changefeed_alter_settings; alter_table_drop_changefeed: DROP CHANGEFEED an_id; - + column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; family_relation: FAMILY an_id; column_order_by_specification: an_id (ASC | DESC)?; @@ -503,12 +503,12 @@ table_constraint: | PARTITION BY LPAREN an_id (COMMA an_id)* RPAREN | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN ; - + table_index: INDEX an_id table_index_type (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? ON LPAREN an_id_schema (COMMA an_id_schema)* RPAREN (COVER LPAREN an_id_schema (COMMA an_id_schema)* RPAREN)?; - + table_index_type: global_index | local_index @@ -857,7 +857,7 @@ keyword_schema_uncompat: keyword_compat: ( ABORT | ACTION - | ADD + | ADD | AFTER | ALTER | ANALYZE @@ -880,7 +880,7 @@ keyword_compat: ( | CONDITIONAL | CONFLICT | CONSTRAINT - | COVER + | COVER | CREATE | CROSS | CURRENT @@ -1113,7 +1113,7 @@ fragment Z:('z'|'Z'); ABORT: A B O R T; ACTION: A C T I O N; -ADD: A D D; +ADD: A D D; AFTER: A F T E R; ALL: A L L; ALTER: A L T E R; @@ -1149,7 +1149,7 @@ COMPACT: C O M P A C T; CONDITIONAL: C O N D I T I O N A L; CONFLICT: C O N F L I C T; CONSTRAINT: C O N S T R A I N T; -COVER: C O V E R; +COVER: C O V E R; CREATE: C R E A T E; CROSS: C R O S S; CUBE: C U B E; @@ -1204,7 +1204,7 @@ FROM: F R O M; FULL: F U L L; FUNCTION: F U N C T I O N; GLOB: G L O B; -GLOBAL: G L O B A L; +GLOBAL: G L O B A L; GROUP: G R O U P; GROUPING: G R O U P I N G; GROUPS: G R O U P S; @@ -1235,7 +1235,7 @@ LEFT: L E F T; LIKE: L I K E; LIMIT: L I M I T; LIST: L I S T; -LOCAL: L O C A L; +LOCAL: L O C A L; MATCH: M A T C H; NATURAL: N A T U R A L; NO: N O; diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index aba1fc9734..43ef1243f1 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -529,7 +529,7 @@ public: return nullptr; } - auto parsed = ParseType(*literal, *ctx.Pool, ctx.Issues, Args[0]->GetPos()); + auto parsed = ParseType(*literal, *ctx.Pool, ctx.Issues, Args[0]->GetPos()); if (!parsed) { ctx.Error(Args[0]->GetPos()) << "Failed to parse type"; return nullptr; @@ -814,9 +814,9 @@ TNodePtr BuildFileNameArgument(TPosition pos, const TNodePtr& argument) { return new TLiteralStringAtom(pos, argument, "FilePath requires string literal as parameter"); } -class TYqlAtom final: public TCallNode { +class TYqlAtom final: public TCallNode { public: - TYqlAtom(TPosition pos, const TString& opName, const TVector<TNodePtr>& args) + TYqlAtom(TPosition pos, const TString& opName, const TVector<TNodePtr>& args) : TCallNode(pos, opName, 1, 1, args) {} @@ -828,7 +828,7 @@ public: } TNodePtr DoClone() const final { - return new TYqlAtom(Pos, OpName, Args); + return new TYqlAtom(Pos, OpName, Args); } bool IsLiteral() const override { @@ -2565,7 +2565,7 @@ struct TBuiltinFuncData { // Atom builtins {"asatom", BuildSimpleBuiltinFactoryCallback<TYqlAsAtom>()}, - {"secureparam", BuildNamedBuiltinFactoryCallback<TYqlAtom>("SecureParam")}, + {"secureparam", BuildNamedBuiltinFactoryCallback<TYqlAtom>("SecureParam")}, {"void", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Void", 0, 0)}, {"emptylist", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("EmptyList", 0, 0)}, @@ -2692,10 +2692,10 @@ struct TBuiltinFuncData { {"staticzip", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticZip", 1, -1) }, // File builtins - {"filepath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FilePath")}, - {"filecontent", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FileContent")}, - {"folderpath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FolderPath") }, - {"files", BuildNamedBuiltinFactoryCallback<TYqlAtom>("Files")}, + {"filepath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FilePath")}, + {"filecontent", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FileContent")}, + {"folderpath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FolderPath") }, + {"files", BuildNamedBuiltinFactoryCallback<TYqlAtom>("Files")}, {"parsefile", BuildSimpleBuiltinFactoryCallback<TYqlParseFileOp>()}, // Misc builtins @@ -2888,7 +2888,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec TString nameSpace(originalNameSpace); TString ns = to_lower(nameSpace); if (ns.empty()) { - TMaybe<TIssue> error = NormalizeName(pos, normalizedName); + TMaybe<TIssue> error = NormalizeName(pos, normalizedName); if (!error.Empty()) { return new TInvalidBuiltin(pos, error->Message); } diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp index 1b8221204e..c08fe96cae 100644 --- a/ydb/library/yql/sql/v1/context.cpp +++ b/ydb/library/yql/sql/v1/context.cpp @@ -66,7 +66,7 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = { } // namespace TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, - TIssues& issues) + TIssues& issues) : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) @@ -74,7 +74,7 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, , Pool(new TMemoryPool(4096)) , Issues(issues) , IncrementMonCounterFunction(settings.IncrementCounter) - , HasPendingErrors(false) + , HasPendingErrors(false) , DqEngineEnable(Settings.DqDefaultAuto->Allow()) , AnsiQuotedIdentifiers(settings.AnsiLexer) { @@ -139,7 +139,7 @@ IOutputStream& TContext::Error(NYql::TIssueCode code) { } IOutputStream& TContext::Error(NYql::TPosition pos, NYql::TIssueCode code) { - HasPendingErrors = true; + HasPendingErrors = true; return MakeIssue(TSeverityIds::S_ERROR, code, pos); } diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index ada32012df..8951757486 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -79,7 +79,7 @@ namespace NSQLTranslationV1 { class TContext { public: TContext(const NSQLTranslation::TTranslationSettings& settings, - NYql::TIssues& issues); + NYql::TIssues& issues); virtual ~TContext(); @@ -204,7 +204,7 @@ namespace NSQLTranslationV1 { THashMap<TString, TNodePtr> Variables; NSQLTranslation::TTranslationSettings Settings; std::unique_ptr<TMemoryPool> Pool; - NYql::TIssues& Issues; + NYql::TIssues& Issues; TMap<TString, TNodePtr> UniversalAliases; THashSet<TString> Exports; THashMap<TString, TString> ImportModuleAliases; @@ -214,7 +214,7 @@ namespace NSQLTranslationV1 { int ScopeLevel = 0; size_t AnonymousNameIndex = 0; TDeque<TScopedStatePtr> AllScopes; - bool HasPendingErrors; + bool HasPendingErrors; THashMap<TString, ui32> GenIndexes; using TWinSpecsRef = std::reference_wrapper<TWinSpecs>; TDeque<TWinSpecsRef> WinSpecsScopes; diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 538222aa8e..de82b45a23 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -68,11 +68,11 @@ namespace NSQLTranslationV1 { DeleteOn, }; - enum class EAlterTableIntentnt { - AddColumn, - DropColumn - }; - + enum class EAlterTableIntentnt { + AddColumn, + DropColumn + }; + class TContext; class ITableKeys; class ISource; @@ -1146,7 +1146,7 @@ namespace NSQLTranslationV1 { TVector<TFamilyEntry> AlterColumnFamilies; TTableSettings TableSettings; TVector<TIndexDescription> AddIndexes; - TVector<TIdentifier> DropIndexes; + TVector<TIdentifier> DropIndexes; TMaybe<TIdentifier> RenameTo; TVector<TChangefeedDescription> AddChangefeeds; TVector<TChangefeedDescription> AlterChangefeeds; diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 13f35593b3..725356b9f5 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -99,22 +99,22 @@ static INode::TPtr CreateIndexType(TIndexDescription::EType type, const INode& n } static INode::TPtr CreateIndexDesc(const TIndexDescription& index, const INode& node) { - auto indexColumns = node.Y(); + auto indexColumns = node.Y(); for (const auto& col : index.IndexColumns) { - indexColumns = node.L(indexColumns, BuildQuotedAtom(col.Pos, col.Name)); - } - auto dataColumns = node.Y(); + indexColumns = node.L(indexColumns, BuildQuotedAtom(col.Pos, col.Name)); + } + auto dataColumns = node.Y(); for (const auto& col : index.DataColumns) { - dataColumns = node.L(dataColumns, BuildQuotedAtom(col.Pos, col.Name)); - } + dataColumns = node.L(dataColumns, BuildQuotedAtom(col.Pos, col.Name)); + } const auto& indexType = node.Y(node.Q("indexType"), CreateIndexType(index.Type, node)); const auto& indexName = node.Y(node.Q("indexName"), BuildQuotedAtom(index.Name.Pos, index.Name.Name)); - return node.Y(node.Q(indexName), - node.Q(indexType), - node.Q(node.Y(node.Q("indexColumns"), node.Q(indexColumns))), - node.Q(node.Y(node.Q("dataColumns"), node.Q(dataColumns)))); -} - + return node.Y(node.Q(indexName), + node.Q(indexType), + node.Q(node.Y(node.Q("indexColumns"), node.Q(indexColumns))), + node.Q(node.Y(node.Q("dataColumns"), node.Q(dataColumns)))); +} + static INode::TPtr CreateChangefeedDesc(const TChangefeedDescription& desc, const INode& node) { auto settings = node.Y(); if (desc.Settings.Mode) { @@ -606,28 +606,28 @@ public: return false; } } - - THashSet<TString> indexNames; + + THashSet<TString> indexNames; for (const auto& index : Params.Indexes) { if (!indexNames.insert(index.Name.Name).second) { ctx.Error(index.Name.Pos) << "Index " << index.Name.Name << " must be defined once"; - return false; - } - + return false; + } + for (const auto& indexColumn : index.IndexColumns) { - if (!columnsSet.contains(indexColumn.Name)) { - ctx.Error(indexColumn.Pos) << "Undefined column: " << indexColumn.Name; - return false; - } - } - + if (!columnsSet.contains(indexColumn.Name)) { + ctx.Error(indexColumn.Pos) << "Undefined column: " << indexColumn.Name; + return false; + } + } + for (const auto& dataColumn : index.DataColumns) { - if (!columnsSet.contains(dataColumn.Name)) { - ctx.Error(dataColumn.Pos) << "Undefined column: " << dataColumn.Name; - return false; - } - } - } + if (!columnsSet.contains(dataColumn.Name)) { + ctx.Error(dataColumn.Pos) << "Undefined column: " << dataColumn.Name; + return false; + } + } + } THashSet<TString> cfNames; for (const auto& cf : Params.Changefeeds) { @@ -708,9 +708,9 @@ public: for (const auto& index : Params.Indexes) { const auto& desc = CreateIndexDesc(index, *this); - opts = L(opts, Q(Y(Q("index"), Q(desc)))); - } - + opts = L(opts, Q(Y(Q("index"), Q(desc)))); + } + for (const auto& cf : Params.Changefeeds) { const auto& desc = CreateChangefeedDesc(cf, *this); opts = L(opts, Q(Y(Q("changefeed"), Q(desc)))); @@ -731,7 +731,7 @@ public: } opts = L(opts, Q(Y(Q("columnFamilies"), Q(columnFamilies)))); } - + if (Params.TableSettings.IsSet()) { auto settings = Y(); @@ -806,7 +806,7 @@ public: return {}; } private: - const TTableRef Table; + const TTableRef Table; const TCreateTableParameters Params; TScopedStatePtr Scoped; }; @@ -816,23 +816,23 @@ TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTable return new TCreateTableNode(pos, tr, params, scoped); } -class TAlterTableNode final: public TAstListNode { -public: +class TAlterTableNode final: public TAstListNode { +public: TAlterTableNode(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) - : TAstListNode(pos) - , Table(tr) + : TAstListNode(pos) + , Table(tr) , Params(params) , Scoped(scoped) { scoped->UseCluster(Table.Service, Table.Cluster); } - bool DoInit(TContext& ctx, ISource* src) override { - auto keys = Table.Keys->GetTableKeys()->BuildKeys(ctx, ITableKeys::EBuildKeysMode::CREATE); - if (!keys || !keys->Init(ctx, src)) { - return false; - } - + bool DoInit(TContext& ctx, ISource* src) override { + auto keys = Table.Keys->GetTableKeys()->BuildKeys(ctx, ITableKeys::EBuildKeysMode::CREATE); + if (!keys || !keys->Init(ctx, src)) { + return false; + } + auto actions = Y(); if (Params.AddColumns) { @@ -841,9 +841,9 @@ public: auto columnDesc = Y(); columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); auto type = col.Type; - if (col.Nullable) { + if (col.Nullable) { type = Y("OptionalType", type); - } + } columnDesc = L(columnDesc, type); if (col.Families) { auto familiesDesc = Y(); @@ -853,18 +853,18 @@ public: columnDesc = L(columnDesc, Q(familiesDesc)); } columns = L(columns, Q(columnDesc)); - } + } actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); - } - + } + if (Params.DropColumns) { auto columns = Y(); for (auto& colName : Params.DropColumns) { columns = L(columns, BuildQuotedAtom(Pos, colName)); } actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); - } - + } + if (Params.AlterColumns) { auto columns = Y(); for (auto& col : Params.AlterColumns) { @@ -959,14 +959,14 @@ public: for (const auto& index : Params.AddIndexes) { const auto& desc = CreateIndexDesc(index, *this); - actions = L(actions, Q(Y(Q("addIndex"), Q(desc)))); - } - - for (const auto& id : Params.DropIndexes) { - auto indexName = BuildQuotedAtom(id.Pos, id.Name); - actions = L(actions, Q(Y(Q("dropIndex"), indexName))); - } - + actions = L(actions, Q(Y(Q("addIndex"), Q(desc)))); + } + + for (const auto& id : Params.DropIndexes) { + auto indexName = BuildQuotedAtom(id.Pos, id.Name); + actions = L(actions, Q(Y(Q("dropIndex"), indexName))); + } + if (Params.RenameTo) { auto destination = ctx.GetPrefixedPath(Scoped->CurrService, Scoped->CurrCluster, TDeferredAtom(Params.RenameTo->Pos, Params.RenameTo->Name)); @@ -993,28 +993,28 @@ public: opts = L(opts, Q(Y(Q("mode"), Q("alter")))); opts = L(opts, Q(Y(Q("actions"), Q(actions)))); - Add("block", Q(Y( + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - return TAstListNode::DoInit(ctx, src); - } - TPtr DoClone() const final { - return {}; - } -private: - TTableRef Table; + ))); + + return TAstListNode::DoInit(ctx, src); + } + TPtr DoClone() const final { + return {}; + } +private: + TTableRef Table; const TAlterTableParameters Params; TScopedStatePtr Scoped; -}; - +}; + TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) -{ +{ return new TAlterTableNode(pos, tr, params, scoped); -} - +} + class TDropTableNode final: public TAstListNode { public: TDropTableNode(TPosition pos, const TTableRef& tr, TScopedStatePtr scoped) @@ -1645,7 +1645,7 @@ public: BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); } - + if (ctx.ResultSizeLimit > 0) { Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink, BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit))))); diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index b7da20a5a8..ab5a864788 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -49,7 +49,7 @@ using NALPDefault::SQLv1LexerTokens; #endif using namespace NSQLv1Generated; - + static TPosition GetPos(const TToken& token) { return TPosition(token.GetColumn(), token.GetLine()); } @@ -72,7 +72,7 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { // | keyword_hint_uncompat // | keyword_schema_uncompat //; - switch (node.Alt_case()) { + switch (node.Alt_case()) { case TRule_keyword::kAltKeyword1: return GetIdentifier(ctx, node.GetAlt_keyword1().GetRule_keyword_compat1()); case TRule_keyword::kAltKeyword2: @@ -91,11 +91,11 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { return GetIdentifier(ctx, node.GetAlt_keyword8().GetRule_keyword_hint_uncompat1()); case TRule_keyword::kAltKeyword9: return GetIdentifier(ctx, node.GetAlt_keyword9().GetRule_keyword_schema_uncompat1()); - default: + default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - + } +} + inline TString GetKeyword(TTranslation& ctx, const TRule_keyword& node) { return GetKeywordId(ctx, node).Name; } @@ -103,8 +103,8 @@ inline TString GetKeyword(TTranslation& ctx, const TRule_keyword& node) { template <typename TRule> inline TString GetKeyword(TTranslation& ctx, const TRule& node) { return GetIdentifier(ctx, node).Name; -} - +} + static TString Id(const TRule_identifier& node, TTranslation& ctx) { // identifier: ID_PLAIN | ID_QUOTED; return ctx.Identifier(node.GetToken1()); @@ -146,7 +146,7 @@ static TString Id(const TRule_id_schema& node, TTranslation& ctx) { // | keyword_hint_uncompat // // | keyword_schema_uncompat //; - switch (node.Alt_case()) { + switch (node.Alt_case()) { case TRule_id_schema::kAltIdSchema1: return Id(node.GetAlt_id_schema1().GetRule_identifier1(), ctx); case TRule_id_schema::kAltIdSchema2: @@ -161,11 +161,11 @@ static TString Id(const TRule_id_schema& node, TTranslation& ctx) { return GetKeyword(ctx, node.GetAlt_id_schema6().GetRule_keyword_window_uncompat1()); case TRule_id_schema::kAltIdSchema7: return GetKeyword(ctx, node.GetAlt_id_schema7().GetRule_keyword_hint_uncompat1()); - default: + default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - + } +} + static TString Id(const TRule_an_id_or_type& node, TTranslation& ctx) { // an_id_or_type: id_or_type | STRING_VALUE; switch (node.Alt_case()) { @@ -509,11 +509,11 @@ static TString Id(const TRule_an_id_pure& node, TTranslation& ctx) { template<typename TRule> static TIdentifier IdEx(const TRule& node, TTranslation& ctx) { - const TString name(Id(node, ctx)); - const TPosition pos(ctx.Context().Pos()); - return TIdentifier(pos, name); -} - + const TString name(Id(node, ctx)); + const TPosition pos(ctx.Context().Pos()); + return TIdentifier(pos, name); +} + static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& ctx, const TString& defaultStr = {}) { if (!node.HasBlock1()) { return defaultStr; @@ -598,15 +598,15 @@ static bool PureColumnOrNamedListStr(const TRule_pure_column_or_named_list& node static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, TVector<TIndexDescription>& indexes) { indexes.emplace_back(IdEx(node.GetRule_an_id2(), ctx)); - + const auto& indexType = node.GetRule_table_index_type3(); - switch (indexType.Alt_case()) { + switch (indexType.Alt_case()) { case TRule_table_index_type::kAltTableIndexType1: { auto globalIndex = indexType.GetAlt_table_index_type1().GetRule_global_index1(); if (globalIndex.HasBlock2()) { - ctx.AltNotImplemented("unique", indexType); - return false; - } + ctx.AltNotImplemented("unique", indexType); + return false; + } if (globalIndex.HasBlock3()) { const TString token = to_lower(ctx.Token(globalIndex.GetBlock3().GetToken1())); if (token == "sync") { @@ -617,36 +617,36 @@ static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, T Y_FAIL("You should change implementation according to grammar changes"); } } - } - break; + } + break; case TRule_table_index_type::kAltTableIndexType2: - ctx.AltNotImplemented("local", indexType); - return false; - default: + ctx.AltNotImplemented("local", indexType); + return false; + default: Y_FAIL("You should change implementation according to grammar changes"); - } - - if (node.HasBlock4()) { - ctx.AltNotImplemented("with", indexType); - return false; - } - + } + + if (node.HasBlock4()) { + ctx.AltNotImplemented("with", indexType); + return false; + } + indexes.back().IndexColumns.emplace_back(IdEx(node.GetRule_an_id_schema7(), ctx)); - for (const auto& block : node.GetBlock8()) { + for (const auto& block : node.GetBlock8()) { indexes.back().IndexColumns.emplace_back(IdEx(block.GetRule_an_id_schema2(), ctx)); - } - - if (node.HasBlock10()) { - const auto& block = node.GetBlock10(); + } + + if (node.HasBlock10()) { + const auto& block = node.GetBlock10(); indexes.back().DataColumns.emplace_back(IdEx(block.GetRule_an_id_schema3(), ctx)); - for (const auto& inner : block.GetBlock4()) { + for (const auto& inner : block.GetBlock4()) { indexes.back().DataColumns.emplace_back(IdEx(inner.GetRule_an_id_schema2(), ctx)); - } - } - - return true; -} - + } + } + + return true; +} + static bool ChangefeedSettingsEntry(const TRule_changefeed_settings_entry& node, TTranslation& ctx, TChangefeedSettings& settings, bool alter) { const auto id = IdEx(node.GetRule_an_id1(), ctx); const TString value(ctx.Token(node.GetRule_changefeed_setting_value3().GetToken1())); @@ -8157,8 +8157,8 @@ private: bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params); bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, TAlterTableParameters& params); bool AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, TAlterTableParameters& params); - bool AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params); - void AlterTableDropIndex(const TRule_alter_table_drop_index& node, TAlterTableParameters& params); + bool AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params); + void AlterTableDropIndex(const TRule_alter_table_drop_index& node, TAlterTableParameters& params); void AlterTableRenameTo(const TRule_alter_table_rename_to& node, TAlterTableParameters& params); bool AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params); bool AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params); @@ -8483,9 +8483,9 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } break; - case TRule_sql_stmt_core::kAltSqlStmtCore15: { - Ctx.BodyPart(); - const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1(); + case TRule_sql_stmt_core::kAltSqlStmtCore15: { + Ctx.BodyPart(); + const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1(); TTableRef tr; if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), tr)) { return false; @@ -8499,12 +8499,12 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& for (auto& block : rule.GetBlock5()) { if (!AlterTableAction(block.GetRule_alter_table_action2(), params)) { return false; - } - } + } + } AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, params, Ctx.Scoped)); - break; - } + break; + } case TRule_sql_stmt_core::kAltSqlStmtCore16: { Ctx.BodyPart(); auto node = DoStatement(core.GetAlt_sql_stmt_core16().GetRule_do_stmt1(), false); @@ -8764,7 +8764,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } Ctx.IncrementMonCounter("sql_features", internalStatementName); - return !Ctx.HasPendingErrors; + return !Ctx.HasPendingErrors; } bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) { @@ -8900,20 +8900,20 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab } break; } - case TRule_alter_table_action::kAltAlterTableAction9: { - // ADD INDEX - const auto& addIndex = node.GetAlt_alter_table_action9().GetRule_alter_table_add_index1(); - if (!AlterTableAddIndex(addIndex, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction10: { - // DROP INDEX - const auto& dropIndex = node.GetAlt_alter_table_action10().GetRule_alter_table_drop_index1(); - AlterTableDropIndex(dropIndex, params); - break; - } + case TRule_alter_table_action::kAltAlterTableAction9: { + // ADD INDEX + const auto& addIndex = node.GetAlt_alter_table_action9().GetRule_alter_table_add_index1(); + if (!AlterTableAddIndex(addIndex, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction10: { + // DROP INDEX + const auto& dropIndex = node.GetAlt_alter_table_action10().GetRule_alter_table_drop_index1(); + AlterTableDropIndex(dropIndex, params); + break; + } case TRule_alter_table_action::kAltAlterTableAction11: { // RENAME TO if (!params.IsEmpty()) { @@ -8921,7 +8921,7 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab Error() << "RENAME TO can not be used together with another table action"; return false; } - + const auto& renameTo = node.GetAlt_alter_table_action11().GetRule_alter_table_rename_to1(); AlterTableRenameTo(renameTo, params); break; @@ -8955,7 +8955,7 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab } return true; } - + bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params) { auto columnSchema = ColumnSchemaImpl(node.GetRule_column_schema3()); if (!columnSchema) { @@ -8993,11 +8993,11 @@ bool TSqlQuery::AlterTableAddFamily(const TRule_family_entry& node, TAlterTableP if (!FillFamilySettings(node.GetBlock3().GetRule_family_settings1(), family)) { return false; } - } + } params.AddColumnFamilies.push_back(family); return true; } - + bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, TAlterTableParameters& params) { @@ -9034,9 +9034,9 @@ bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_famil Ctx.Error() << "Unknown table setting: " << settingName.Name; return false; } - return true; -} - + return true; +} + bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params) { @@ -9044,9 +9044,9 @@ bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_sett params.TableSettings, true)) { return false; } - return true; -} - + return true; +} + bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, TAlterTableParameters& params) { @@ -9081,17 +9081,17 @@ bool TSqlQuery::AlterTableResetTableSetting(const TRule_alter_table_reset_table_ return true; } -bool TSqlQuery::AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params) { - if (!CreateTableIndex(node.GetRule_table_index2(), *this, params.AddIndexes)) { - return false; - } - return true; -} - -void TSqlQuery::AlterTableDropIndex(const TRule_alter_table_drop_index& node, TAlterTableParameters& params) { - params.DropIndexes.emplace_back(IdEx(node.GetRule_an_id3(), *this)); -} - +bool TSqlQuery::AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params) { + if (!CreateTableIndex(node.GetRule_table_index2(), *this, params.AddIndexes)) { + return false; + } + return true; +} + +void TSqlQuery::AlterTableDropIndex(const TRule_alter_table_drop_index& node, TAlterTableParameters& params) { + params.DropIndexes.emplace_back(IdEx(node.GetRule_an_id3(), *this)); +} + void TSqlQuery::AlterTableRenameTo(const TRule_alter_table_rename_to& node, TAlterTableParameters& params) { params.RenameTo = IdEx(node.GetRule_an_id_table3(), *this); } @@ -9137,7 +9137,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success const TString& lowerPrefix = to_lower(prefix); const TString pragma(Id(stmt.GetRule_an_id3(), *this)); TString normalizedPragma(pragma); - TMaybe<TIssue> normalizeError = NormalizeName(Ctx.Pos(), normalizedPragma); + TMaybe<TIssue> normalizeError = NormalizeName(Ctx.Pos(), normalizedPragma); if (!normalizeError.Empty()) { Error() << normalizeError->Message; Ctx.IncrementMonCounter("sql_errors", "NormalizePragmaError"); @@ -9357,7 +9357,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultRowsLimit)) { Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; + return {}; } Ctx.IncrementMonCounter("sql_pragma", "ResultRowsLimit"); @@ -9365,16 +9365,16 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultSizeLimit)) { Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; + return {}; } Ctx.IncrementMonCounter("sql_pragma", "ResultSizeLimit"); - } else if (normalizedPragma == "warning") { + } else if (normalizedPragma == "warning") { if (values.size() != 2U || values.front().Empty() || values.back().Empty()) { - Error() << "Expected arguments <action>, <issueId> for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } + Error() << "Expected arguments <action>, <issueId> for: " << pragma; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } TString action; TString codePattern; @@ -9403,7 +9403,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.SetWarningPolicyFor(TIssuesIds::YQL_UNUSED_SYMBOL, EWarningAction::DEFAULT); } - Ctx.IncrementMonCounter("sql_pragma", "warning"); + Ctx.IncrementMonCounter("sql_pragma", "warning"); } else if (normalizedPragma == "greetings") { if (values.size() > 1) { Error() << "Multiple arguments are not expected for " << pragma; @@ -10092,11 +10092,11 @@ TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message& protoAst, TContext& ctx) { - YQL_ENSURE(!ctx.Issues.Size()); + YQL_ENSURE(!ctx.Issues.Size()); res.Root = SqlASTToYql(protoAst, ctx); res.Pool = std::move(ctx.Pool); if (!res.Root) { - if (ctx.Issues.Size()) { + if (ctx.Issues.Size()) { ctx.IncrementMonCounter("sql_errors", "AstToYqlError"); } else { ctx.IncrementMonCounter("sql_errors", "AstToYqlSilentError"); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 1f45a14130..85b6e0dfcd 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -154,10 +154,10 @@ Y_UNIT_TEST_SUITE(AnsiMode) { } Y_UNIT_TEST_SUITE(SqlParsingOnly) { - Y_UNIT_TEST(CoverColumnName) { - UNIT_ASSERT(SqlToYql("SELECT cover FROM plato.Input").IsOk()); - } - + Y_UNIT_TEST(CoverColumnName) { + UNIT_ASSERT(SqlToYql("SELECT cover FROM plato.Input").IsOk()); + } + Y_UNIT_TEST(TableHints) { UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH INFER_SCHEMA").IsOk()); UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEMA)").IsOk()); @@ -861,23 +861,23 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); } - Y_UNIT_TEST(ProcessUserTypeAuth) { + Y_UNIT_TEST(ProcessUserTypeAuth) { NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "YDB.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); - } - }; - - TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); - } - + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "YDB.PushData") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); + } + }; + + TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); + } + Y_UNIT_TEST(SelectStreamRtmr) { NYql::TAstParseResult res = SqlToYql( "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", diff --git a/ydb/public/api/grpc/draft/dummy.proto b/ydb/public/api/grpc/draft/dummy.proto index b1f5edfafe..9a2a17f0d8 100644 --- a/ydb/public/api/grpc/draft/dummy.proto +++ b/ydb/public/api/grpc/draft/dummy.proto @@ -1,19 +1,19 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/ydb_operation.proto"; -package Draft.Dummy; - -message PingRequest { - bool copy = 1; - bytes payload = 2; -} - -message PingResponse { - bytes payload = 1; -} - +package Draft.Dummy; + +message PingRequest { + bool copy = 1; + bytes payload = 2; +} + +message PingResponse { + bytes payload = 1; +} + message InfiniteRequest { Ydb.Operations.OperationParams operation_params = 1; } @@ -22,9 +22,9 @@ message InfiniteResponse { Ydb.Operations.Operation operation = 1; } -service DummyService { - rpc Ping(PingRequest) returns (PingResponse); +service DummyService { + rpc Ping(PingRequest) returns (PingResponse); rpc Infinite(InfiniteRequest) returns (InfiniteResponse); - rpc BiStreamPing(stream PingRequest) returns (stream PingResponse); -} - + rpc BiStreamPing(stream PingRequest) returns (stream PingResponse); +} + diff --git a/ydb/public/api/grpc/draft/ya.make b/ydb/public/api/grpc/draft/ya.make index 588583c2b5..f63be96521 100644 --- a/ydb/public/api/grpc/draft/ya.make +++ b/ydb/public/api/grpc/draft/ya.make @@ -2,17 +2,17 @@ PROTO_LIBRARY(api-grpc-draft) MAVEN_GROUP_ID(com.yandex.ydb) -GRPC() - +GRPC() + OWNER( dcherednik fomichev vvvv g:kikimr ) - -SRCS( - dummy.proto + +SRCS( + dummy.proto ydb_clickhouse_internal_v1.proto ydb_persqueue_v1.proto ydb_datastreams_v1.proto @@ -21,12 +21,12 @@ SRCS( ydb_long_tx_v1.proto ydb_logstore_v1.proto yql_db_v1.proto -) - -PEERDIR( +) + +PEERDIR( ydb/public/api/protos -) - +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/api/grpc/ya.make b/ydb/public/api/grpc/ya.make index 4f4a628ab6..24173a0264 100644 --- a/ydb/public/api/grpc/ya.make +++ b/ydb/public/api/grpc/ya.make @@ -2,35 +2,35 @@ PROTO_LIBRARY(api-grpc) MAVEN_GROUP_ID(com.yandex.ydb) -GRPC() - +GRPC() + OWNER( dcherednik fomichev vvvv g:kikimr ) - -SRCS( + +SRCS( ydb_auth_v1.proto ydb_coordination_v1.proto - ydb_discovery_v1.proto + ydb_discovery_v1.proto ydb_export_v1.proto ydb_import_v1.proto ydb_monitoring_v1.proto - ydb_operation_v1.proto - ydb_cms_v1.proto + ydb_operation_v1.proto + ydb_cms_v1.proto ydb_rate_limiter_v1.proto - ydb_scheme_v1.proto + ydb_scheme_v1.proto ydb_scripting_v1.proto - ydb_table_v1.proto + ydb_table_v1.proto yq_v1.proto -) - -PEERDIR( +) + +PEERDIR( ydb/public/api/protos -) - +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/api/grpc/ydb_cms_v1.proto b/ydb/public/api/grpc/ydb_cms_v1.proto index d3ec482124..11e4517381 100644 --- a/ydb/public/api/grpc/ydb_cms_v1.proto +++ b/ydb/public/api/grpc/ydb_cms_v1.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package Ydb.Cms.V1; +package Ydb.Cms.V1; option java_package = "com.yandex.ydb.cms.v1"; import "ydb/public/api/protos/ydb_cms.proto"; @@ -10,20 +10,20 @@ import "ydb/public/api/protos/ydb_cms.proto"; // instances for example. service CmsService { - // Create a new database. - rpc CreateDatabase(Cms.CreateDatabaseRequest) returns (Cms.CreateDatabaseResponse); + // Create a new database. + rpc CreateDatabase(Cms.CreateDatabaseRequest) returns (Cms.CreateDatabaseResponse); - // Get current database's status. - rpc GetDatabaseStatus(Cms.GetDatabaseStatusRequest) returns (Cms.GetDatabaseStatusResponse); + // Get current database's status. + rpc GetDatabaseStatus(Cms.GetDatabaseStatusRequest) returns (Cms.GetDatabaseStatusResponse); - // Alter database resources. - rpc AlterDatabase(Cms.AlterDatabaseRequest) returns (Cms.AlterDatabaseResponse); + // Alter database resources. + rpc AlterDatabase(Cms.AlterDatabaseRequest) returns (Cms.AlterDatabaseResponse); - // List all databases. - rpc ListDatabases(Cms.ListDatabasesRequest) returns (Cms.ListDatabasesResponse); + // List all databases. + rpc ListDatabases(Cms.ListDatabasesRequest) returns (Cms.ListDatabasesResponse); - // Remove database. - rpc RemoveDatabase(Cms.RemoveDatabaseRequest) returns (Cms.RemoveDatabaseResponse); + // Remove database. + rpc RemoveDatabase(Cms.RemoveDatabaseRequest) returns (Cms.RemoveDatabaseResponse); // Describe supported database options. rpc DescribeDatabaseOptions(Cms.DescribeDatabaseOptionsRequest) returns (Cms.DescribeDatabaseOptionsResponse); diff --git a/ydb/public/api/grpc/ydb_discovery_v1.proto b/ydb/public/api/grpc/ydb_discovery_v1.proto index 9b3563d8c2..dc06a4f678 100644 --- a/ydb/public/api/grpc/ydb_discovery_v1.proto +++ b/ydb/public/api/grpc/ydb_discovery_v1.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package Ydb.Discovery.V1; +package Ydb.Discovery.V1; option java_package = "com.yandex.ydb.discovery.v1"; import "ydb/public/api/protos/ydb_discovery.proto"; diff --git a/ydb/public/api/grpc/ydb_operation_v1.proto b/ydb/public/api/grpc/ydb_operation_v1.proto index 7e2945802b..8202d08484 100644 --- a/ydb/public/api/grpc/ydb_operation_v1.proto +++ b/ydb/public/api/grpc/ydb_operation_v1.proto @@ -1,29 +1,29 @@ -syntax = "proto3"; - -package Ydb.Operation.V1; +syntax = "proto3"; + +package Ydb.Operation.V1; option java_package = "com.yandex.ydb.operation.v1"; - + import "ydb/public/api/protos/ydb_operation.proto"; - + // All rpc calls to YDB are allowed to be asynchronous. Response message -// of an rpc call contains Operation structure and OperationService +// of an rpc call contains Operation structure and OperationService // is used for polling operation completion. // -// Operation has a field 'ready' to notify client if operation has been +// Operation has a field 'ready' to notify client if operation has been // completed or not. If result is ready a client has to handle 'result' field, // otherwise it is expected that client continues polling result via -// GetOperation rpc of OperationService. Polling is made via unique -// operation id provided in 'id' field of Operation. +// GetOperation rpc of OperationService. Polling is made via unique +// operation id provided in 'id' field of Operation. // // Note: Currently some operations have synchronous implementation and their result // is available when response is obtained. But a client must not make any // assumptions about synchronous or asynchronous nature of any operation and // be ready to poll operation status. - -service OperationService { - + +service OperationService { + // Check status for a given operation. - rpc GetOperation(Operations.GetOperationRequest) returns (Operations.GetOperationResponse); + rpc GetOperation(Operations.GetOperationRequest) returns (Operations.GetOperationResponse); // Starts cancellation of a long-running operation, // Clients can use GetOperation to check whether the cancellation succeeded @@ -36,4 +36,4 @@ service OperationService { // Lists operations that match the specified filter in the request. rpc ListOperations(Operations.ListOperationsRequest) returns (Operations.ListOperationsResponse); -} +} diff --git a/ydb/public/api/grpc/ydb_scheme_v1.proto b/ydb/public/api/grpc/ydb_scheme_v1.proto index 11fcc4e531..634b16e849 100644 --- a/ydb/public/api/grpc/ydb_scheme_v1.proto +++ b/ydb/public/api/grpc/ydb_scheme_v1.proto @@ -1,10 +1,10 @@ -syntax = "proto3"; - -package Ydb.Scheme.V1; +syntax = "proto3"; + +package Ydb.Scheme.V1; option java_package = "com.yandex.ydb.scheme.v1"; - + import "ydb/public/api/protos/ydb_scheme.proto"; - + // Every YDB Database Instance has set of objects organized a tree. // SchemeService provides some functionality to browse and modify // this tree. @@ -12,20 +12,20 @@ import "ydb/public/api/protos/ydb_scheme.proto"; // SchemeService provides a generic tree functionality, to create specific // objects like YDB Table or Persistent Queue use corresponding services. -service SchemeService { - +service SchemeService { + // Make Directory. - rpc MakeDirectory(Scheme.MakeDirectoryRequest) returns (Scheme.MakeDirectoryResponse); - + rpc MakeDirectory(Scheme.MakeDirectoryRequest) returns (Scheme.MakeDirectoryResponse); + // Remove Directory. - rpc RemoveDirectory(Scheme.RemoveDirectoryRequest) returns (Scheme.RemoveDirectoryResponse); - + rpc RemoveDirectory(Scheme.RemoveDirectoryRequest) returns (Scheme.RemoveDirectoryResponse); + // Returns information about given directory and objects inside it. - rpc ListDirectory(Scheme.ListDirectoryRequest) returns (Scheme.ListDirectoryResponse); - + rpc ListDirectory(Scheme.ListDirectoryRequest) returns (Scheme.ListDirectoryResponse); + // Returns information about object with given path. - rpc DescribePath(Scheme.DescribePathRequest) returns (Scheme.DescribePathResponse); - - // Modify permissions. - rpc ModifyPermissions(Scheme.ModifyPermissionsRequest) returns (Scheme.ModifyPermissionsResponse); -} + rpc DescribePath(Scheme.DescribePathRequest) returns (Scheme.DescribePathResponse); + + // Modify permissions. + rpc ModifyPermissions(Scheme.ModifyPermissionsRequest) returns (Scheme.ModifyPermissionsResponse); +} diff --git a/ydb/public/api/grpc/ydb_table_v1.proto b/ydb/public/api/grpc/ydb_table_v1.proto index b02eac5f10..dbbf62c5c8 100644 --- a/ydb/public/api/grpc/ydb_table_v1.proto +++ b/ydb/public/api/grpc/ydb_table_v1.proto @@ -1,78 +1,78 @@ -syntax = "proto3"; - -package Ydb.Table.V1; +syntax = "proto3"; + +package Ydb.Table.V1; option java_package = "com.yandex.ydb.table.v1"; - + import "ydb/public/api/protos/ydb_table.proto"; - -service TableService { - - // Create new session. Implicit session creation is forbidden, + +service TableService { + + // Create new session. Implicit session creation is forbidden, // so user must create new session before execute any query, // otherwise BAD_SESSION status will be returned. - // Simultaneous execution of requests are forbiden. + // Simultaneous execution of requests are forbiden. // Sessions are volatile, can be invalidated by server, for example in case - // of fatal errors. All requests with this session will fail with BAD_SESSION status. - // So, client must be able to handle BAD_SESSION status. - rpc CreateSession(Table.CreateSessionRequest) returns (Table.CreateSessionResponse); - - // Ends a session, releasing server resources associated with it. - rpc DeleteSession(Table.DeleteSessionRequest) returns (Table.DeleteSessionResponse); - - // Idle sessions can be kept alive by calling KeepAlive periodically. - rpc KeepAlive(Table.KeepAliveRequest) returns (Table.KeepAliveResponse); - - // Creates new table. - rpc CreateTable(Table.CreateTableRequest) returns (Table.CreateTableResponse); - - // Drop table. - rpc DropTable(Table.DropTableRequest) returns (Table.DropTableResponse); - - // Modifies schema of given table. - rpc AlterTable(Table.AlterTableRequest) returns (Table.AlterTableResponse); - - // Creates copy of given table. - rpc CopyTable(Table.CopyTableRequest) returns (Table.CopyTableResponse); - + // of fatal errors. All requests with this session will fail with BAD_SESSION status. + // So, client must be able to handle BAD_SESSION status. + rpc CreateSession(Table.CreateSessionRequest) returns (Table.CreateSessionResponse); + + // Ends a session, releasing server resources associated with it. + rpc DeleteSession(Table.DeleteSessionRequest) returns (Table.DeleteSessionResponse); + + // Idle sessions can be kept alive by calling KeepAlive periodically. + rpc KeepAlive(Table.KeepAliveRequest) returns (Table.KeepAliveResponse); + + // Creates new table. + rpc CreateTable(Table.CreateTableRequest) returns (Table.CreateTableResponse); + + // Drop table. + rpc DropTable(Table.DropTableRequest) returns (Table.DropTableResponse); + + // Modifies schema of given table. + rpc AlterTable(Table.AlterTableRequest) returns (Table.AlterTableResponse); + + // Creates copy of given table. + rpc CopyTable(Table.CopyTableRequest) returns (Table.CopyTableResponse); + // Creates consistent copy of given tables. rpc CopyTables(Table.CopyTablesRequest) returns (Table.CopyTablesResponse); // Creates consistent move of given tables. rpc RenameTables(Table.RenameTablesRequest) returns (Table.RenameTablesResponse); - // Returns information about given table (metadata). - rpc DescribeTable(Table.DescribeTableRequest) returns (Table.DescribeTableResponse); - + // Returns information about given table (metadata). + rpc DescribeTable(Table.DescribeTableRequest) returns (Table.DescribeTableResponse); + // Explains data query. // SessionId of previously created session must be provided. - rpc ExplainDataQuery(Table.ExplainDataQueryRequest) returns (Table.ExplainDataQueryResponse); + rpc ExplainDataQuery(Table.ExplainDataQueryRequest) returns (Table.ExplainDataQueryResponse); // Prepares data query, returns query id. // SessionId of previously created session must be provided. - rpc PrepareDataQuery(Table.PrepareDataQueryRequest) returns (Table.PrepareDataQueryResponse); + rpc PrepareDataQuery(Table.PrepareDataQueryRequest) returns (Table.PrepareDataQueryResponse); // Executes data query. // SessionId of previously created session must be provided. - rpc ExecuteDataQuery(Table.ExecuteDataQueryRequest) returns (Table.ExecuteDataQueryResponse); + rpc ExecuteDataQuery(Table.ExecuteDataQueryRequest) returns (Table.ExecuteDataQueryResponse); // Executes scheme query. // SessionId of previously created session must be provided. - rpc ExecuteSchemeQuery(Table.ExecuteSchemeQueryRequest) returns (Table.ExecuteSchemeQueryResponse); + rpc ExecuteSchemeQuery(Table.ExecuteSchemeQueryRequest) returns (Table.ExecuteSchemeQueryResponse); // Begins new transaction. - rpc BeginTransaction(Table.BeginTransactionRequest) returns (Table.BeginTransactionResponse); + rpc BeginTransaction(Table.BeginTransactionRequest) returns (Table.BeginTransactionResponse); // Commits specified active transaction. - rpc CommitTransaction(Table.CommitTransactionRequest) returns (Table.CommitTransactionResponse); + rpc CommitTransaction(Table.CommitTransactionRequest) returns (Table.CommitTransactionResponse); // Performs a rollback of the specified active transaction. - rpc RollbackTransaction(Table.RollbackTransactionRequest) returns (Table.RollbackTransactionResponse); + rpc RollbackTransaction(Table.RollbackTransactionRequest) returns (Table.RollbackTransactionResponse); // Describe supported table options. rpc DescribeTableOptions(Table.DescribeTableOptionsRequest) returns (Table.DescribeTableOptionsResponse); - - // Streaming read table - rpc StreamReadTable(Table.ReadTableRequest) returns (stream Table.ReadTableResponse); + + // Streaming read table + rpc StreamReadTable(Table.ReadTableRequest) returns (stream Table.ReadTableResponse); // Upserts a batch of rows non-transactionally. // Returns success only when all rows were successfully upserted. In case of an error some rows might @@ -81,4 +81,4 @@ service TableService { // Executes scan query with streaming result. rpc StreamExecuteScanQuery(Table.ExecuteScanQueryRequest) returns (stream Table.ExecuteScanQueryPartialResponse); -} +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index a1a0abaedf..8a156403a3 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -1,19 +1,19 @@ PROTO_LIBRARY(api-protos) MAVEN_GROUP_ID(com.yandex.ydb) - + OWNER( dcherednik fomichev vvvv g:kikimr ) - + PEERDIR( ydb/public/api/protos/annotations ) -SRCS( +SRCS( draft/datastreams.proto draft/persqueue_common.proto draft/persqueue_error_codes.proto @@ -25,28 +25,28 @@ SRCS( ydb_persqueue_v1.proto ydb_persqueue_cluster_discovery.proto ydb_clickhouse_internal.proto - ydb_cms.proto + ydb_cms.proto ydb_common.proto ydb_coordination.proto - ydb_discovery.proto + ydb_discovery.proto ydb_experimental.proto ydb_export.proto ydb_formats.proto ydb_import.proto - ydb_issue_message.proto + ydb_issue_message.proto ydb_monitoring.proto - ydb_operation.proto + ydb_operation.proto ydb_query_stats.proto ydb_rate_limiter.proto - ydb_scheme.proto + ydb_scheme.proto ydb_scripting.proto ydb_status_codes.proto - ydb_table.proto + ydb_table.proto ydb_value.proto ydb_s3_internal.proto yq.proto -) - +) + CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation) # .pb.h are only available in C++ variant of PROTO_LIBRARY @@ -58,4 +58,4 @@ ENDIF() EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/api/protos/ydb_cms.proto b/ydb/public/api/protos/ydb_cms.proto index 3c959d8ad3..1a569c7081 100644 --- a/ydb/public/api/protos/ydb_cms.proto +++ b/ydb/public/api/protos/ydb_cms.proto @@ -29,7 +29,7 @@ message ComputationalUnits { uint64 count = 3; } -// Computational unit allocated for database. Used to register +// Computational unit allocated for database. Used to register // externally allocated computational resources in CMS. message AllocatedComputationalUnit { // Required. Computational unit host name. @@ -96,12 +96,12 @@ message DatabaseQuotas { uint32 ttl_min_run_internal_seconds = 4; } -// Request to create a new database. For successfull creation -// specified database shouldn't exist. At least one storage -// unit should be requested for the database. -message CreateDatabaseRequest { - Ydb.Operations.OperationParams operation_params = 1; - // Required. Full path to database's home dir. Used as database ID. +// Request to create a new database. For successfull creation +// specified database shouldn't exist. At least one storage +// unit should be requested for the database. +message CreateDatabaseRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Required. Full path to database's home dir. Used as database ID. string path = 2; oneof resources_kind { // Resources to allocate for database by CMS. @@ -114,7 +114,7 @@ message CreateDatabaseRequest { // Additional database options. DatabaseOptions options = 4; // Attach attributes to database. - map<string, string> attributes = 5; + map<string, string> attributes = 5; // Optional quotas for schema operations SchemaOperationQuotas schema_operation_quotas = 8; // Optional idempotency key @@ -123,26 +123,26 @@ message CreateDatabaseRequest { DatabaseQuotas database_quotas = 10; } -message CreateDatabaseResponse { - Ydb.Operations.Operation operation = 1; +message CreateDatabaseResponse { + Ydb.Operations.Operation operation = 1; } -// Get current database status. -message GetDatabaseStatusRequest { - // Required. Full path to database's home dir. +// Get current database status. +message GetDatabaseStatusRequest { + // Required. Full path to database's home dir. string path = 1; // Operation parameters Ydb.Operations.OperationParams operation_params = 2; } -message GetDatabaseStatusResponse { +message GetDatabaseStatusResponse { // operation.result holds GetDatabaseStatusResult - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } -message GetDatabaseStatusResult { +message GetDatabaseStatusResult { enum State { - STATE_UNSPECIFIED = 0; + STATE_UNSPECIFIED = 0; CREATING = 1; RUNNING = 2; REMOVING = 3; @@ -150,9 +150,9 @@ message GetDatabaseStatusResult { CONFIGURING = 5; } - // Full path to database's home dir. + // Full path to database's home dir. string path = 1; - // Current database state. + // Current database state. State state = 2; oneof resources_kind { // Database resources requested for allocation. @@ -160,9 +160,9 @@ message GetDatabaseStatusResult { Resources required_shared_resources = 7; ServerlessResources serverless_resources = 8; } - // Database resources allocated by CMS. + // Database resources allocated by CMS. Resources allocated_resources = 4; - // Externally allocated database resources registered in CMS. + // Externally allocated database resources registered in CMS. repeated AllocatedComputationalUnit registered_resources = 5; // Current database generation. Incremented at each successful // alter request. @@ -173,17 +173,17 @@ message GetDatabaseStatusResult { DatabaseQuotas database_quotas = 10; } -// Change resources allocated for database. -message AlterDatabaseRequest { - // Required. Full path to database's home dir. +// Change resources allocated for database. +message AlterDatabaseRequest { + // Required. Full path to database's home dir. string path = 1; - // Additional computational units to allocate for database. + // Additional computational units to allocate for database. repeated ComputationalUnits computational_units_to_add = 2; // Computational units to deallocate. repeated ComputationalUnits computational_units_to_remove = 3; - // Additional storage units to allocate for database. + // Additional storage units to allocate for database. repeated StorageUnits storage_units_to_add = 4; - // Externally allocated computational units to register for database. + // Externally allocated computational units to register for database. repeated AllocatedComputationalUnit computational_units_to_register = 5; // Externally allocated computational units to deregister. repeated AllocatedComputationalUnit computational_units_to_deregister = 6; @@ -197,39 +197,39 @@ message AlterDatabaseRequest { string idempotency_key = 10; // Change quotas for the database DatabaseQuotas database_quotas = 11; - // Alter attributes. Leave the value blank to drop an attribute. - map<string, string> alter_attributes = 12; + // Alter attributes. Leave the value blank to drop an attribute. + map<string, string> alter_attributes = 12; } -message AlterDatabaseResponse { - Ydb.Operations.Operation operation = 1; +message AlterDatabaseResponse { + Ydb.Operations.Operation operation = 1; } -// List all databases known by CMS. -message ListDatabasesRequest { +// List all databases known by CMS. +message ListDatabasesRequest { // Operation parameters Ydb.Operations.OperationParams operation_params = 1; } -message ListDatabasesResponse { +message ListDatabasesResponse { // operation.result holds ListDatabasesResult - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } -message ListDatabasesResult { +message ListDatabasesResult { repeated string paths = 1; } -// Completely remove database and all his data. -message RemoveDatabaseRequest { - // Required. Full path to database's home dir. +// Completely remove database and all his data. +message RemoveDatabaseRequest { + // Required. Full path to database's home dir. string path = 1; - - Ydb.Operations.OperationParams operation_params = 2; + + Ydb.Operations.OperationParams operation_params = 2; } -message RemoveDatabaseResponse { - Ydb.Operations.Operation operation = 1; +message RemoveDatabaseResponse { + Ydb.Operations.Operation operation = 1; } message StorageUnitDescription { diff --git a/ydb/public/api/protos/ydb_issue_message.proto b/ydb/public/api/protos/ydb_issue_message.proto index c7ffbc33fe..1c294d7c59 100644 --- a/ydb/public/api/protos/ydb_issue_message.proto +++ b/ydb/public/api/protos/ydb_issue_message.proto @@ -1,26 +1,26 @@ syntax = "proto3"; option cc_enable_arenas = true; -package Ydb.Issue; +package Ydb.Issue; option java_package = "com.yandex.ydb"; - + // IssueMessage is a transport format for ydb/library/yql/public/issue library -message IssueMessage { - message Position { +message IssueMessage { + message Position { uint32 row = 1; uint32 column = 2; string file = 3; - } - + } + Position position = 1; string message = 2; Position end_position = 3; uint32 issue_code = 4; // Severity values from ydb/library/yql/public/issue/protos/issue_severity.proto - // FATAL = 0; - // ERROR = 1; - // WARNING = 2; - // INFO = 3; + // FATAL = 0; + // ERROR = 1; + // WARNING = 2; + // INFO = 3; uint32 severity = 5; repeated IssueMessage issues = 6; -} +} diff --git a/ydb/public/api/protos/ydb_operation.proto b/ydb/public/api/protos/ydb_operation.proto index f3aedf374d..30ddd002a2 100644 --- a/ydb/public/api/protos/ydb_operation.proto +++ b/ydb/public/api/protos/ydb_operation.proto @@ -1,31 +1,31 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -import "google/protobuf/any.proto"; +syntax = "proto3"; +option cc_enable_arenas = true; + +import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; - + import "ydb/public/api/protos/annotations/validation.proto"; import "ydb/public/api/protos/ydb_common.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; - -package Ydb.Operations; - + +package Ydb.Operations; + option java_package = "com.yandex.ydb"; -option java_outer_classname = "OperationProtos"; +option java_outer_classname = "OperationProtos"; -message OperationParams { - enum OperationMode { - OPERATION_MODE_UNSPECIFIED = 0; +message OperationParams { + enum OperationMode { + OPERATION_MODE_UNSPECIFIED = 0; // Server will only reply once operation is finished (ready=true), and operation object won't be // accessible after the reply. This is a basic request-response mode. SYNC = 1; ASYNC = 2; - } - - OperationMode operation_mode = 1; + } + + OperationMode operation_mode = 1; // Indicates that client is no longer interested in the result of operation after the specified duration // starting from the time operation arrives at the server. @@ -48,16 +48,16 @@ message OperationParams { // If enabled, server will report cost information, if supported by the operation. // This flag is mostly useful for SYNC operations, to get the cost information in the response. Ydb.FeatureFlag.Status report_cost_info = 5; -} - -message GetOperationRequest { +} + +message GetOperationRequest { string id = 1 [(required) = true]; -} - -message GetOperationResponse { - Operation operation = 1; -} - +} + +message GetOperationResponse { + Operation operation = 1; +} + message CancelOperationRequest { string id = 1 [(required) = true]; } @@ -89,21 +89,21 @@ message ListOperationsResponse { string next_page_token = 4; } -message Operation { +message Operation { // Identifier of the operation, empty value means no active operation object is present (it was forgotten or // not created in the first place, as in SYNC operation mode). - string id = 1; - - // true - this operation has beed finished (doesn't matter successful or not), - // so Status field has status code, and Result field can contains result data. - // false - this operation still running. You can repeat request using operation Id. - bool ready = 2; - - StatusIds.StatusCode status = 3; - - repeated Ydb.Issue.IssueMessage issues = 4; - // Result data - google.protobuf.Any result = 5; + string id = 1; + + // true - this operation has beed finished (doesn't matter successful or not), + // so Status field has status code, and Result field can contains result data. + // false - this operation still running. You can repeat request using operation Id. + bool ready = 2; + + StatusIds.StatusCode status = 3; + + repeated Ydb.Issue.IssueMessage issues = 4; + // Result data + google.protobuf.Any result = 5; google.protobuf.Any metadata = 6; @@ -111,5 +111,5 @@ message Operation { // For completed operations, it shows the final cost of the operation. // For operations in progress, it might indicate the current cost of the operation (if supported). CostInfo cost_info = 7; -} - +} + diff --git a/ydb/public/api/protos/ydb_scheme.proto b/ydb/public/api/protos/ydb_scheme.proto index 134b555794..d66b765725 100644 --- a/ydb/public/api/protos/ydb_scheme.proto +++ b/ydb/public/api/protos/ydb_scheme.proto @@ -1,70 +1,70 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb.Scheme; +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.Scheme; option java_package = "com.yandex.ydb.scheme"; option java_outer_classname = "SchemeOperationProtos"; - + import "ydb/public/api/protos/ydb_operation.proto"; - -// Create directory. -// All intermediate directories must be created -message MakeDirectoryRequest { - Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + +// Create directory. +// All intermediate directories must be created +message MakeDirectoryRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; } - -message MakeDirectoryResponse { - Ydb.Operations.Operation operation = 1; + +message MakeDirectoryResponse { + Ydb.Operations.Operation operation = 1; } - -// Remove directory -message RemoveDirectoryRequest { - Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + +// Remove directory +message RemoveDirectoryRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; } - -message RemoveDirectoryResponse { - Ydb.Operations.Operation operation = 1; + +message RemoveDirectoryResponse { + Ydb.Operations.Operation operation = 1; } - -// List directory -message ListDirectoryRequest { - Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + +// List directory +message ListDirectoryRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; } - -message ListDirectoryResponse { - // Holds ListDirectoryResult in case of successful call - Ydb.Operations.Operation operation = 1; + +message ListDirectoryResponse { + // Holds ListDirectoryResult in case of successful call + Ydb.Operations.Operation operation = 1; } - -message Permissions { + +message Permissions { // SID (Security ID) of user or group - string subject = 1; - repeated string permission_names = 2; + string subject = 1; + repeated string permission_names = 2; } - -message Entry { - enum Type { - TYPE_UNSPECIFIED = 0; - DIRECTORY = 1; - TABLE = 2; - PERS_QUEUE_GROUP = 3; - DATABASE = 4; - RTMR_VOLUME = 5; - BLOCK_STORE_VOLUME = 6; + +message Entry { + enum Type { + TYPE_UNSPECIFIED = 0; + DIRECTORY = 1; + TABLE = 2; + PERS_QUEUE_GROUP = 3; + DATABASE = 4; + RTMR_VOLUME = 5; + BLOCK_STORE_VOLUME = 6; COORDINATION_NODE = 7; SEQUENCE = 15; REPLICATION = 16; } // Name of scheme entry (dir2 of /dir1/dir2) - string name = 1; + string name = 1; // SID (Security ID) of user or group - string owner = 2; - Type type = 5; - repeated Permissions effective_permissions = 6; + string owner = 2; + Type type = 5; + repeated Permissions effective_permissions = 6; repeated Permissions permissions = 7; // Size of entry in bytes. Currently filled for: @@ -73,49 +73,49 @@ message Entry { // Empty (zero) in other cases. uint64 size_bytes = 8; } - -message ListDirectoryResult { - Entry self = 1; - repeated Entry children = 2; + +message ListDirectoryResult { + Entry self = 1; + repeated Entry children = 2; } - -// Returns information about object with given path -message DescribePathRequest { - Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + +// Returns information about object with given path +message DescribePathRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; } - -message DescribePathResponse { - // Holds DescribePathResult in case of DescribePathResult - Ydb.Operations.Operation operation = 1; + +message DescribePathResponse { + // Holds DescribePathResult in case of DescribePathResult + Ydb.Operations.Operation operation = 1; } - -message DescribePathResult { - Entry self = 1; + +message DescribePathResult { + Entry self = 1; } - -message PermissionsAction { - oneof action { - // Grant permissions - Permissions grant = 1; - // Revoke permissions - Permissions revoke = 2; - // Rewrite permissions for given subject (last set win in case of multiple set for one subject) - Permissions set = 3; - // New owner for object - string change_owner = 4; - } + +message PermissionsAction { + oneof action { + // Grant permissions + Permissions grant = 1; + // Revoke permissions + Permissions revoke = 2; + // Rewrite permissions for given subject (last set win in case of multiple set for one subject) + Permissions set = 3; + // New owner for object + string change_owner = 4; + } } - -// Modify permissions of given object -message ModifyPermissionsRequest { - Ydb.Operations.OperationParams operation_params = 1; - string path = 2; - repeated PermissionsAction actions = 3; - // Clear all permissions on the object for all subjects - bool clear_permissions = 4; + +// Modify permissions of given object +message ModifyPermissionsRequest { + Ydb.Operations.OperationParams operation_params = 1; + string path = 2; + repeated PermissionsAction actions = 3; + // Clear all permissions on the object for all subjects + bool clear_permissions = 4; } - -message ModifyPermissionsResponse { - Ydb.Operations.Operation operation = 1; + +message ModifyPermissionsResponse { + Ydb.Operations.Operation operation = 1; } diff --git a/ydb/public/api/protos/ydb_status_codes.proto b/ydb/public/api/protos/ydb_status_codes.proto index 36f1ba80ec..4c500dc43a 100644 --- a/ydb/public/api/protos/ydb_status_codes.proto +++ b/ydb/public/api/protos/ydb_status_codes.proto @@ -1,13 +1,13 @@ -syntax = "proto3"; - +syntax = "proto3"; + package Ydb; option java_package = "com.yandex.ydb"; option java_outer_classname = "StatusCodesProtos"; - -message StatusIds { - // reserved range [400000, 400999] - enum StatusCode { - STATUS_CODE_UNSPECIFIED = 0; + +message StatusIds { + // reserved range [400000, 400999] + enum StatusCode { + STATUS_CODE_UNSPECIFIED = 0; SUCCESS = 400000; BAD_REQUEST = 400010; UNAUTHORIZED = 400020; @@ -27,6 +27,6 @@ message StatusIds { UNDETERMINED = 400170; UNSUPPORTED = 400180; SESSION_BUSY = 400190; - } - // reserved range [401000, 402999] for internal client status -} + } + // reserved range [401000, 402999] for internal client status +} diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 9d21bb6d7e..c8063d9475 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -1,6 +1,6 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/annotations/validation.proto"; import "ydb/public/api/protos/ydb_common.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; @@ -10,113 +10,113 @@ import "ydb/public/api/protos/ydb_value.proto"; import "ydb/public/api/protos/ydb_scheme.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_formats.proto"; - + import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - -package Ydb.Table; +import "google/protobuf/timestamp.proto"; + +package Ydb.Table; option java_package = "com.yandex.ydb.table"; - -// Create new session -message CreateSessionRequest { + +// Create new session +message CreateSessionRequest { Ydb.Operations.OperationParams operation_params = 1; } - -// Create new session -message CreateSessionResponse { - // Holds CreateSessionResult in case of CreateSessionResult - Ydb.Operations.Operation operation = 1; + +// Create new session +message CreateSessionResponse { + // Holds CreateSessionResult in case of CreateSessionResult + Ydb.Operations.Operation operation = 1; } - -message CreateSessionResult { + +message CreateSessionResult { // Session identifier - string session_id = 1; + string session_id = 1; } - -// Delete session with given id string -message DeleteSessionRequest { + +// Delete session with given id string +message DeleteSessionRequest { // Session identifier - string session_id = 1; + string session_id = 1; Ydb.Operations.OperationParams operation_params = 2; } - -message DeleteSessionResponse { - Ydb.Operations.Operation operation = 1; + +message DeleteSessionResponse { + Ydb.Operations.Operation operation = 1; } - -message GlobalIndex { + +message GlobalIndex { } - + message GlobalAsyncIndex { } -// Represent secondary index -message TableIndex { - // Name of index - string name = 1; - // list of columns - repeated string index_columns = 2; - // Type of index - oneof type { - GlobalIndex global_index = 3; +// Represent secondary index +message TableIndex { + // Name of index + string name = 1; + // list of columns + repeated string index_columns = 2; + // Type of index + oneof type { + GlobalIndex global_index = 3; GlobalAsyncIndex global_async_index = 4; - } - // list of columns content to be copied in to index table - repeated string data_columns = 5; -} - -// Represent secondary index with index state -message TableIndexDescription { - enum Status { - STATUS_UNSPECIFIED = 0; - // Index is ready to use - STATUS_READY = 1; - // index is being built - STATUS_BUILDING = 2; - } - // Name of index - string name = 1; - // list of columns - repeated string index_columns = 2; - // Type of index - oneof type { - GlobalIndex global_index = 3; + } + // list of columns content to be copied in to index table + repeated string data_columns = 5; +} + +// Represent secondary index with index state +message TableIndexDescription { + enum Status { + STATUS_UNSPECIFIED = 0; + // Index is ready to use + STATUS_READY = 1; + // index is being built + STATUS_BUILDING = 2; + } + // Name of index + string name = 1; + // list of columns + repeated string index_columns = 2; + // Type of index + oneof type { + GlobalIndex global_index = 3; GlobalAsyncIndex global_async_index = 5; - } - Status status = 4; - // list of columns content to be copied in to index table - repeated string data_columns = 6; + } + Status status = 4; + // list of columns content to be copied in to index table + repeated string data_columns = 6; // Size of index data in bytes uint64 size_bytes = 7; -} - -// State of index building operation -message IndexBuildState { - enum State { - STATE_UNSPECIFIED = 0; - STATE_PREPARING = 1; - STATE_TRANSFERING_DATA = 2; - STATE_APPLYING = 3; - STATE_DONE = 4; - STATE_CANCELLATION = 5; - STATE_CANCELLED = 6; +} + +// State of index building operation +message IndexBuildState { + enum State { + STATE_UNSPECIFIED = 0; + STATE_PREPARING = 1; + STATE_TRANSFERING_DATA = 2; + STATE_APPLYING = 3; + STATE_DONE = 4; + STATE_CANCELLATION = 5; + STATE_CANCELLED = 6; STATE_REJECTION = 7; STATE_REJECTED = 8; - } -} - -// Description of index building operation -message IndexBuildDescription { - string path = 1; - TableIndex index = 2; -} - -message IndexBuildMetadata { - IndexBuildDescription description = 1; - IndexBuildState.State state = 2; - float progress = 3; -} - + } +} + +// Description of index building operation +message IndexBuildDescription { + string path = 1; + TableIndex index = 2; +} + +message IndexBuildMetadata { + IndexBuildDescription description = 1; + IndexBuildState.State state = 2; + float progress = 3; +} + message StoragePool { string media = 1; } @@ -167,34 +167,34 @@ message ExplicitPartitions { repeated TypedValue split_points = 1; } -message PartitionStats { - // Approximate number of rows in shard - uint64 rows_estimate = 1; - // Approximate size of shard (bytes) - uint64 store_size = 2; -} - -message TableStats { - // Stats for each partition - repeated PartitionStats partition_stats = 1; - // Approximate number of rows in table - uint64 rows_estimate = 2; - // Approximate size of table (bytes) - uint64 store_size = 3; - // Number of partitions in table - uint64 partitions = 4; - // Timestamp of table creation - google.protobuf.Timestamp creation_time = 5; - // Timestamp of last modification - google.protobuf.Timestamp modification_time = 6; -} - +message PartitionStats { + // Approximate number of rows in shard + uint64 rows_estimate = 1; + // Approximate size of shard (bytes) + uint64 store_size = 2; +} + +message TableStats { + // Stats for each partition + repeated PartitionStats partition_stats = 1; + // Approximate number of rows in table + uint64 rows_estimate = 2; + // Approximate size of table (bytes) + uint64 store_size = 3; + // Number of partitions in table + uint64 partitions = 4; + // Timestamp of table creation + google.protobuf.Timestamp creation_time = 5; + // Timestamp of last modification + google.protobuf.Timestamp modification_time = 6; +} + message PartitioningPolicy { enum AutoPartitioningPolicy { - AUTO_PARTITIONING_POLICY_UNSPECIFIED = 0; - DISABLED = 1; - AUTO_SPLIT = 2; - AUTO_SPLIT_MERGE = 3; + AUTO_PARTITIONING_POLICY_UNSPECIFIED = 0; + DISABLED = 1; + AUTO_SPLIT = 2; + AUTO_SPLIT_MERGE = 3; } string preset_name = 1; @@ -393,20 +393,20 @@ message ReadReplicasSettings { reserved 3; // cluster_replicas_settings (part of oneof settings) } -message CreateTableRequest { +message CreateTableRequest { // Session identifier - string session_id = 1; - // Full path - string path = 2; - // Columns (name, type) + string session_id = 1; + // Full path + string path = 2; + // Columns (name, type) repeated ColumnMeta columns = 3; - // List of columns used as primary key - repeated string primary_key = 4; - // Table profile + // List of columns used as primary key + repeated string primary_key = 4; + // Table profile TableProfile profile = 5; Ydb.Operations.OperationParams operation_params = 6; - // List of secondary indexes - repeated TableIndex indexes = 7; + // List of secondary indexes + repeated TableIndex indexes = 7; // Table rows time to live settings TtlSettings ttl_settings = 8; // Storage settings for table @@ -432,35 +432,35 @@ message CreateTableRequest { // Read replicas settings for table ReadReplicasSettings read_replicas_settings = 17; } - -message CreateTableResponse { - Ydb.Operations.Operation operation = 1; + +message CreateTableResponse { + Ydb.Operations.Operation operation = 1; } - -// Drop table with given path -message DropTableRequest { + +// Drop table with given path +message DropTableRequest { // Session identifier - string session_id = 1; - // Full path - string path = 2; - reserved 3; + string session_id = 1; + // Full path + string path = 2; + reserved 3; Ydb.Operations.OperationParams operation_params = 4; } - -message DropTableResponse { - Ydb.Operations.Operation operation = 1; + +message DropTableResponse { + Ydb.Operations.Operation operation = 1; } - -// Alter table with given path -message AlterTableRequest { + +// Alter table with given path +message AlterTableRequest { // Session identifier - string session_id = 1; - // Full path - string path = 2; - // Columns (name, type) to add + string session_id = 1; + // Full path + string path = 2; + // Columns (name, type) to add repeated ColumnMeta add_columns = 3; - // Columns to remove - repeated string drop_columns = 4; + // Columns to remove + repeated string drop_columns = 4; Ydb.Operations.OperationParams operation_params = 5; // Columns to alter repeated ColumnMeta alter_columns = 6; @@ -469,10 +469,10 @@ message AlterTableRequest { TtlSettings set_ttl_settings = 7; google.protobuf.Empty drop_ttl_settings = 8; } - // Add secondary indexes - repeated TableIndex add_indexes = 9; - // Remove secondary indexes - repeated string drop_indexes = 10; + // Add secondary indexes + repeated TableIndex add_indexes = 9; + // Remove secondary indexes + repeated string drop_indexes = 10; // Change table storage settings StorageSettings alter_storage_settings = 11; // Add/alter column families @@ -491,26 +491,26 @@ message AlterTableRequest { // Set read replicas settings for table ReadReplicasSettings set_read_replicas_settings = 18; } - -message AlterTableResponse { - Ydb.Operations.Operation operation = 1; + +message AlterTableResponse { + Ydb.Operations.Operation operation = 1; } - -// Copy table with given path -message CopyTableRequest { + +// Copy table with given path +message CopyTableRequest { // Session identifier - string session_id = 1; - // Copy from path - string source_path = 2; - // Copy to path - string destination_path = 3; + string session_id = 1; + // Copy from path + string source_path = 2; + // Copy to path + string destination_path = 3; Ydb.Operations.OperationParams operation_params = 4; } - -message CopyTableResponse { - Ydb.Operations.Operation operation = 1; + +message CopyTableResponse { + Ydb.Operations.Operation operation = 1; } - + message CopyTableItem { // Copy from path string source_path = 1; @@ -555,39 +555,39 @@ message RenameTablesResponse { Ydb.Operations.Operation operation = 1; } -// Describe table with given path -message DescribeTableRequest { +// Describe table with given path +message DescribeTableRequest { // Session identifier - string session_id = 1; - // Full path - string path = 2; + string session_id = 1; + // Full path + string path = 2; Ydb.Operations.OperationParams operation_params = 4; - // Includes shard key distribution info - bool include_shard_key_bounds = 5; - // Includes table statistics - bool include_table_stats = 6; - // Includes partition statistics (required include_table_statistics) - bool include_partition_stats = 7; -} - -message DescribeTableResponse { - // Holds DescribeTableResult in case of successful call - Ydb.Operations.Operation operation = 1; -} - -message DescribeTableResult { - // Description of scheme object - Ydb.Scheme.Entry self = 1; - // List of columns + // Includes shard key distribution info + bool include_shard_key_bounds = 5; + // Includes table statistics + bool include_table_stats = 6; + // Includes partition statistics (required include_table_statistics) + bool include_partition_stats = 7; +} + +message DescribeTableResponse { + // Holds DescribeTableResult in case of successful call + Ydb.Operations.Operation operation = 1; +} + +message DescribeTableResult { + // Description of scheme object + Ydb.Scheme.Entry self = 1; + // List of columns repeated ColumnMeta columns = 2; - // List of primary key columns - repeated string primary_key = 3; - // List of key ranges for shard - repeated TypedValue shard_key_bounds = 4; - // List of indexes - repeated TableIndexDescription indexes = 5; - // Statistics of table - TableStats table_stats = 6; + // List of primary key columns + repeated string primary_key = 3; + // List of key ranges for shard + repeated TypedValue shard_key_bounds = 4; + // List of indexes + repeated TableIndexDescription indexes = 5; + // Statistics of table + TableStats table_stats = 6; // TTL params TtlSettings ttl_settings = 7; // Storage settings for table @@ -605,13 +605,13 @@ message DescribeTableResult { // Read replicas settings for table ReadReplicasSettings read_replicas_settings = 14; } - + message Query { - // Text of query or id prepared query + // Text of query or id prepared query oneof query { - // SQL program + // SQL program string yql_text = 1; - // Prepared query id + // Prepared query id string id = 2; } } @@ -662,46 +662,46 @@ message ExecuteDataQueryRequest { string session_id = 1; TransactionControl tx_control = 2; Query query = 3; - // Map of query parameters (optional) - map<string, TypedValue> parameters = 4; + // Map of query parameters (optional) + map<string, TypedValue> parameters = 4; QueryCachePolicy query_cache_policy = 5; Ydb.Operations.OperationParams operation_params = 6; QueryStatsCollection.Mode collect_stats = 7; } message ExecuteDataQueryResponse { - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } message ExecuteSchemeQueryRequest { // Session identifier string session_id = 1; - // SQL text + // SQL text string yql_text = 2; Ydb.Operations.OperationParams operation_params = 3; } message ExecuteSchemeQueryResponse { - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } -// Holds transaction id +// Holds transaction id message TransactionMeta { // Transaction identifier string id = 1; } -// Holds query id and type of parameters +// Holds query id and type of parameters message QueryMeta { // Query identifier string id = 1; - // Type of parameters + // Type of parameters map<string, Type> parameters_types = 2; } -// One QueryResult can contain multiple tables -message ExecuteQueryResult { - // Result rets (for each table) +// One QueryResult can contain multiple tables +message ExecuteQueryResult { + // Result rets (for each table) repeated Ydb.ResultSet result_sets = 1; // Transaction metadata TransactionMeta tx_meta = 2; @@ -710,58 +710,58 @@ message ExecuteQueryResult { // Query execution statistics Ydb.TableStats.QueryStats query_stats = 4; } - -// Explain data query + +// Explain data query message ExplainDataQueryRequest { // Session identifier string session_id = 1; - // SQL text to explain + // SQL text to explain string yql_text = 2; Ydb.Operations.OperationParams operation_params = 3; } message ExplainDataQueryResponse { - // Holds ExplainQueryResult in case of successful call - Ydb.Operations.Operation operation = 1; + // Holds ExplainQueryResult in case of successful call + Ydb.Operations.Operation operation = 1; } -message ExplainQueryResult { - string query_ast = 1; - string query_plan = 2; +message ExplainQueryResult { + string query_ast = 1; + string query_plan = 2; } - -// Prepare given program to execute + +// Prepare given program to execute message PrepareDataQueryRequest { // Session identifier string session_id = 1; - // SQL text + // SQL text string yql_text = 2; Ydb.Operations.OperationParams operation_params = 3; } message PrepareDataQueryResponse { - // Holds PrepareQueryResult in case of successful call - Ydb.Operations.Operation operation = 1; + // Holds PrepareQueryResult in case of successful call + Ydb.Operations.Operation operation = 1; } -message PrepareQueryResult { - // Query id, used to perform ExecuteDataQuery - string query_id = 1; - // Parameters type, used to fill in parameter values +message PrepareQueryResult { + // Query id, used to perform ExecuteDataQuery + string query_id = 1; + // Parameters type, used to fill in parameter values map<string, Type> parameters_types = 2; } - -// Keep session alive -message KeepAliveRequest { + +// Keep session alive +message KeepAliveRequest { // Session identifier - string session_id = 1; + string session_id = 1; Ydb.Operations.OperationParams operation_params = 2; } - -message KeepAliveResponse { - Ydb.Operations.Operation operation = 1; + +message KeepAliveResponse { + Ydb.Operations.Operation operation = 1; } - + message KeepAliveResult { enum SessionStatus { SESSION_STATUS_UNSPECIFIED = 0; @@ -772,7 +772,7 @@ message KeepAliveResult { SessionStatus session_status = 1; } -// Begin transaction on given session with given settings +// Begin transaction on given session with given settings message BeginTransactionRequest { // Session identifier string session_id = 1; @@ -781,15 +781,15 @@ message BeginTransactionRequest { } message BeginTransactionResponse { - // Holds BeginTransactionResult in case of successful call - Ydb.Operations.Operation operation = 1; + // Holds BeginTransactionResult in case of successful call + Ydb.Operations.Operation operation = 1; } message BeginTransactionResult { TransactionMeta tx_meta = 1; } -// Commit transaction with given session and tx id +// Commit transaction with given session and tx id message CommitTransactionRequest { // Session identifier string session_id = 1; @@ -800,14 +800,14 @@ message CommitTransactionRequest { } message CommitTransactionResponse { - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } message CommitTransactionResult { Ydb.TableStats.QueryStats query_stats = 1; } -// Rollback transaction with given session and tx id +// Rollback transaction with given session and tx id message RollbackTransactionRequest { // Session identifier string session_id = 1; @@ -817,7 +817,7 @@ message RollbackTransactionRequest { } message RollbackTransactionResponse { - Ydb.Operations.Operation operation = 1; + Ydb.Operations.Operation operation = 1; } message StoragePolicyDescription { @@ -885,58 +885,58 @@ message DescribeTableOptionsResult { repeated ReplicationPolicyDescription replication_policy_presets = 6; repeated CachingPolicyDescription caching_policy_presets = 7; } - -// ReadTable request/response - -message KeyRange { - // Left border - oneof from_bound { + +// ReadTable request/response + +message KeyRange { + // Left border + oneof from_bound { // Specify if we don't want to include given key - TypedValue greater = 1; - // Specify if we want to include given key - TypedValue greater_or_equal = 2; - } - - // Right border - oneof to_bound { + TypedValue greater = 1; + // Specify if we want to include given key + TypedValue greater_or_equal = 2; + } + + // Right border + oneof to_bound { // Specify if we don't want to include given key - TypedValue less = 3; - // Specify if we want to include given key - TypedValue less_or_equal = 4; - } -} - -// Request to read table (without SQL) -message ReadTableRequest { + TypedValue less = 3; + // Specify if we want to include given key + TypedValue less_or_equal = 4; + } +} + +// Request to read table (without SQL) +message ReadTableRequest { // Session identifier - string session_id = 1; - // Path to table to read - string path = 2; - // Primary key range to read - KeyRange key_range = 3; - // Output columns - repeated string columns = 4; - // Require ordered reading - bool ordered = 5; - // Limits row count to read - uint64 row_limit = 6; + string session_id = 1; + // Path to table to read + string path = 2; + // Primary key range to read + KeyRange key_range = 3; + // Output columns + repeated string columns = 4; + // Require ordered reading + bool ordered = 5; + // Limits row count to read + uint64 row_limit = 6; // Use a server-side snapshot Ydb.FeatureFlag.Status use_snapshot = 7; } - -// ReadTable doesn't use Operation, returns result directly -message ReadTableResponse { - // Status of request (same as other statuses) - StatusIds.StatusCode status = 1; - // Issues - repeated Ydb.Issue.IssueMessage issues = 2; - // Read table result - ReadTableResult result = 3; -} - -// Result of read table request -message ReadTableResult { - // Result set (same as result of sql request) + +// ReadTable doesn't use Operation, returns result directly +message ReadTableResponse { + // Status of request (same as other statuses) + StatusIds.StatusCode status = 1; + // Issues + repeated Ydb.Issue.IssueMessage issues = 2; + // Read table result + ReadTableResult result = 3; +} + +// Result of read table request +message ReadTableResult { + // Result set (same as result of sql request) Ydb.ResultSet result_set = 1; } diff --git a/ydb/public/api/protos/ydb_value.proto b/ydb/public/api/protos/ydb_value.proto index 520e5a19b6..d885d73f8e 100644 --- a/ydb/public/api/protos/ydb_value.proto +++ b/ydb/public/api/protos/ydb_value.proto @@ -1,84 +1,84 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -import "google/protobuf/struct.proto"; - +syntax = "proto3"; +option cc_enable_arenas = true; + +import "google/protobuf/struct.proto"; + package Ydb; option java_package = "com.yandex.ydb"; option java_outer_classname = "ValueProtos"; - + message DecimalType { uint32 precision = 1; uint32 scale = 2; -} - -message OptionalType { - Type item = 1; -} - -message ListType { - Type item = 1; -} - +} + +message OptionalType { + Type item = 1; +} + +message ListType { + Type item = 1; +} + message VariantType { - oneof type { - TupleType tuple_items = 1; - StructType struct_items = 2; - } -} - -message TupleType { - repeated Type elements = 1; -} - + oneof type { + TupleType tuple_items = 1; + StructType struct_items = 2; + } +} + +message TupleType { + repeated Type elements = 1; +} + message StructMember { - string name = 1; - Type type = 2; -} - -message StructType { + string name = 1; + Type type = 2; +} + +message StructType { repeated StructMember members = 1; -} - -message DictType { - Type key = 1; - Type payload = 2; -} - +} + +message DictType { + Type key = 1; + Type payload = 2; +} + message TaggedType { string tag = 1; Type type = 2; } -message Type { +message Type { enum PrimitiveTypeId { - PRIMITIVE_TYPE_ID_UNSPECIFIED = 0x0000; - BOOL = 0x0006; - INT8 = 0x0007; - UINT8 = 0x0005; - INT16 = 0x0008; - UINT16 = 0x0009; - INT32 = 0x0001; - UINT32 = 0x0002; - INT64 = 0x0003; - UINT64 = 0x0004; - FLOAT = 0x0021; - DOUBLE = 0x0020; - DATE = 0x0030; - DATETIME = 0x0031; - TIMESTAMP = 0x0032; - INTERVAL = 0x0033; - TZ_DATE = 0x0034; - TZ_DATETIME = 0x0035; - TZ_TIMESTAMP = 0x0036; - STRING = 0x1001; - UTF8 = 0x1200; - YSON = 0x1201; - JSON = 0x1202; - UUID = 0x1203; + PRIMITIVE_TYPE_ID_UNSPECIFIED = 0x0000; + BOOL = 0x0006; + INT8 = 0x0007; + UINT8 = 0x0005; + INT16 = 0x0008; + UINT16 = 0x0009; + INT32 = 0x0001; + UINT32 = 0x0002; + INT64 = 0x0003; + UINT64 = 0x0004; + FLOAT = 0x0021; + DOUBLE = 0x0020; + DATE = 0x0030; + DATETIME = 0x0031; + TIMESTAMP = 0x0032; + INTERVAL = 0x0033; + TZ_DATE = 0x0034; + TZ_DATETIME = 0x0035; + TZ_TIMESTAMP = 0x0036; + STRING = 0x1001; + UTF8 = 0x1200; + YSON = 0x1201; + JSON = 0x1202; + UUID = 0x1203; JSON_DOCUMENT = 0x1204; DYNUMBER = 0x1302; - } + } oneof type { /* Data types */ @@ -100,56 +100,56 @@ message Type { google.protobuf.NullValue empty_list_type = 203; google.protobuf.NullValue empty_dict_type = 204; } -} - -/** - * Holds a pair to represent Dict type - */ - -message ValuePair { - Value key = 1; - Value payload = 2; -} - -/** - * This message represents any of the supported by transport value types. - * Note, this is not actually a Ydb types. See NYql.NProto.TypeIds for Ydb types. - * - * For scalar types, just oneof value used. - * For composite types repeated Items or Pairs used. See below. - * - * The idea is, we do not represent explicitly Optional<T> if value is not null (most common case) - * - just represents value of T. Numbers of Optional levels we can get from type. - * Variant<T> type always represent explicitly - */ - -message Value { +} + +/** + * Holds a pair to represent Dict type + */ + +message ValuePair { + Value key = 1; + Value payload = 2; +} + +/** + * This message represents any of the supported by transport value types. + * Note, this is not actually a Ydb types. See NYql.NProto.TypeIds for Ydb types. + * + * For scalar types, just oneof value used. + * For composite types repeated Items or Pairs used. See below. + * + * The idea is, we do not represent explicitly Optional<T> if value is not null (most common case) + * - just represents value of T. Numbers of Optional levels we can get from type. + * Variant<T> type always represent explicitly + */ + +message Value { oneof value { - bool bool_value = 1; - sfixed32 int32_value = 2; - fixed32 uint32_value = 3; - sfixed64 int64_value = 4; - fixed64 uint64_value = 5; - float float_value = 6; - double double_value = 7; - bytes bytes_value = 8; - string text_value = 9; - google.protobuf.NullValue null_flag_value = 10; // Set if current TValue is terminal Null - Value nested_value = 11; // Represents nested TValue for Optional<Optional<T>>(Null), or Variant<T> types - fixed64 low_128 = 15; - } - - repeated Value items = 12; // Used for List, Tuple, Struct types - repeated ValuePair pairs = 13; // Used for Dict type - uint32 variant_index = 14; // Used for Variant type - fixed64 high_128 = 16; -} - -message TypedValue { - Type type = 1; - Value value = 2; + bool bool_value = 1; + sfixed32 int32_value = 2; + fixed32 uint32_value = 3; + sfixed64 int64_value = 4; + fixed64 uint64_value = 5; + float float_value = 6; + double double_value = 7; + bytes bytes_value = 8; + string text_value = 9; + google.protobuf.NullValue null_flag_value = 10; // Set if current TValue is terminal Null + Value nested_value = 11; // Represents nested TValue for Optional<Optional<T>>(Null), or Variant<T> types + fixed64 low_128 = 15; + } + + repeated Value items = 12; // Used for List, Tuple, Struct types + repeated ValuePair pairs = 13; // Used for Dict type + uint32 variant_index = 14; // Used for Variant type + fixed64 high_128 = 16; +} + +message TypedValue { + Type type = 1; + Value value = 2; } - + message Column { // Name of column string name = 1; diff --git a/ydb/public/lib/README.md b/ydb/public/lib/README.md index 3eae0e250d..3fed4b75af 100644 --- a/ydb/public/lib/README.md +++ b/ydb/public/lib/README.md @@ -1,3 +1,3 @@ -# Internal library for Yandex Database (YDB) sdk. DO NOT USE IT DIRECTLY !!! - -- If you need YDB sdk see kikimr/public/sdk +# Internal library for Yandex Database (YDB) sdk. DO NOT USE IT DIRECTLY !!! + +- If you need YDB sdk see kikimr/public/sdk diff --git a/ydb/public/lib/deprecated/client/grpc_client.cpp b/ydb/public/lib/deprecated/client/grpc_client.cpp index 725361776e..dbf808ef75 100644 --- a/ydb/public/lib/deprecated/client/grpc_client.cpp +++ b/ydb/public/lib/deprecated/client/grpc_client.cpp @@ -23,7 +23,7 @@ namespace NKikimr { virtual void Finished() = 0; }; - struct IStreamRequestReadProcessor : public IProcessorBase + struct IStreamRequestReadProcessor : public IProcessorBase { virtual void InvokeProcess() = 0; virtual void InvokeFinish() = 0; @@ -68,7 +68,7 @@ namespace NKikimr { ~TRequestProcessor() { if (!Invoked) { - TGrpcError error = {"request left unhandled", -1}; + TGrpcError error = {"request left unhandled", -1}; Callback(&error, Reply); } } @@ -87,7 +87,7 @@ namespace NKikimr { Callback(nullptr, Reply); } else { const auto& msg = Status.error_message(); - TGrpcError error = {TString(msg.data(), msg.length()), Status.error_code()}; + TGrpcError error = {TString(msg.data(), msg.length()), Status.error_code()}; Callback(&error, Reply); } } @@ -95,7 +95,7 @@ namespace NKikimr { template<typename TRequest, typename TResponse> class TStreamRequestProcessor - : public IStreamRequestReadProcessor + : public IStreamRequestReadProcessor { public: using TAsyncReaderPtr = std::unique_ptr<grpc::ClientAsyncReader<TResponse>>; @@ -137,7 +137,7 @@ namespace NKikimr { ~TStreamRequestProcessor() { if (!Finished) { - TGrpcError error = {"request left unhandled", -1}; + TGrpcError error = {"request left unhandled", -1}; Finish(&error); } } @@ -169,7 +169,7 @@ namespace NKikimr { Finish(nullptr); } else { const auto& msg = Status.error_message(); - TGrpcError error = {TString(msg.data(), msg.length()), Status.error_code()}; + TGrpcError error = {TString(msg.data(), msg.length()), Status.error_code()}; Finish(&error); } } @@ -259,7 +259,7 @@ namespace NKikimr { break; } if (IsStreamTag(tag)) { - THolder<IStreamRequestReadProcessor> processor(static_cast<IStreamRequestReadProcessor*>(tag)); + THolder<IStreamRequestReadProcessor> processor(static_cast<IStreamRequestReadProcessor*>(tag)); if (ok) { processor->InvokeProcess(); Y_UNUSED(processor.Release()); // keep processor alive diff --git a/ydb/public/lib/deprecated/client/grpc_client.h b/ydb/public/lib/deprecated/client/grpc_client.h index 10b364dd4c..b0483a2646 100644 --- a/ydb/public/lib/deprecated/client/grpc_client.h +++ b/ydb/public/lib/deprecated/client/grpc_client.h @@ -10,13 +10,13 @@ namespace NKikimr { using TGRpcClientConfig = NGrpc::TGRpcClientConfig; - using TGrpcError = std::pair<TString, int>; - + using TGrpcError = std::pair<TString, int>; + template<typename T> using TSimpleCallback = std::function<void (const T&)>; template<typename T> - using TCallback = std::function<void (const TGrpcError*, const T&)>; + using TCallback = std::function<void (const TGrpcError*, const T&)>; using TResponseCallback = TCallback<NKikimrClient::TResponse>; using TJSONCallback = TCallback<NKikimrClient::TJSON>; @@ -26,7 +26,7 @@ namespace NKikimr { using TS3ListingResponseCallback = TCallback<NKikimrClient::TS3ListingResponse>; using TConsoleResponseCallback = TCallback<NKikimrClient::TConsoleResponse>; - using TFinishCallback = std::function<void (const TGrpcError*)>; + using TFinishCallback = std::function<void (const TGrpcError*)>; class TGRpcClient { TGRpcClientConfig Config; diff --git a/ydb/public/lib/deprecated/client/ya.make b/ydb/public/lib/deprecated/client/ya.make index 86bb6f09ae..0792f44996 100644 --- a/ydb/public/lib/deprecated/client/ya.make +++ b/ydb/public/lib/deprecated/client/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - msgbus_client.cpp - msgbus_client.h - msgbus_client_config.h - msgbus_player.cpp - msgbus_player.h - grpc_client.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + msgbus_client.cpp + msgbus_client.h + msgbus_client_config.h + msgbus_player.cpp + msgbus_player.h + grpc_client.cpp +) + +PEERDIR( library/cpp/grpc/client library/cpp/messagebus ydb/public/lib/base -) - -END() +) + +END() diff --git a/ydb/public/lib/deprecated/kicli/README.md b/ydb/public/lib/deprecated/kicli/README.md index 1308bde680..7a9c9c54f1 100644 --- a/ydb/public/lib/deprecated/kicli/README.md +++ b/ydb/public/lib/deprecated/kicli/README.md @@ -1,4 +1,4 @@ -# Deprecated Kikimr client. DO NOT USE IT IN NEW PROJECT! API used by this library will be disabled soon! - -- If you need YDB sdk see kikimr/public/sdk - +# Deprecated Kikimr client. DO NOT USE IT IN NEW PROJECT! API used by this library will be disabled soon! + +- If you need YDB sdk see kikimr/public/sdk + diff --git a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp index 97d190a4ab..cb7dc3ed6d 100644 --- a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp +++ b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp @@ -163,9 +163,9 @@ Y_UNIT_TEST_SUITE(ClientLibSchema) { NClient::TKeyColumn("Id", NClient::TType::Uint64), NClient::TColumn("Species", NClient::TType::Utf8), NClient::TColumn("Name", NClient::TType::Utf8), - NClient::TColumn("Weight", NClient::TType::Int64), - NClient::TColumn("MiscY", NClient::TType::Yson), - NClient::TColumn("MiscJ", NClient::TType::Json) + NClient::TColumn("Weight", NClient::TType::Int64), + NClient::TColumn("MiscY", NClient::TType::Yson), + NClient::TColumn("MiscJ", NClient::TType::Json) }); auto children = dc.GetChildren(); UNIT_ASSERT(HasChild(dc, "/dc-1/Animals")); @@ -177,27 +177,27 @@ Y_UNIT_TEST_SUITE(ClientLibSchema) { UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "Species"; })); UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "Name"; })); UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "Weight"; })); - UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "MiscY"; })); - UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "MiscJ"; })); - - try { - dc.CreateTable("FailedCreateTable", { - NClient::TKeyColumn("Id", NClient::TType::Json), - NClient::TColumn("Value", NClient::TType::Uint64) - }); + UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "MiscY"; })); + UNIT_ASSERT(std::find_if(columns.begin(), columns.end(), [](const auto& c) -> bool { return c.Name == "MiscJ"; })); + + try { + dc.CreateTable("FailedCreateTable", { + NClient::TKeyColumn("Id", NClient::TType::Json), + NClient::TColumn("Value", NClient::TType::Uint64) + }); UNIT_FAIL("Unexpected CreateTable success"); - } catch (const yexception& ) { - } - - try { - dc.CreateTable("FailedCreateTable", { - NClient::TKeyColumn("Id", NClient::TType::Yson), - NClient::TColumn("Value", NClient::TType::Uint64) - }); + } catch (const yexception& ) { + } + + try { + dc.CreateTable("FailedCreateTable", { + NClient::TKeyColumn("Id", NClient::TType::Yson), + NClient::TColumn("Value", NClient::TType::Uint64) + }); UNIT_FAIL("Unexpected CreateTable success"); - } catch (const yexception& ) { - } - + } catch (const yexception& ) { + } + #if 1 // TODO: it should be an error auto sameTable = dc.CreateTable("Animals", { NClient::TKeyColumn("Id", NClient::TType::Uint64), @@ -748,7 +748,7 @@ Y_UNIT_TEST_SUITE(ClientLib) { " '('Name (Utf8 '\"Dobby\"))" " '('Weight (Utf8 '350))))" "(let pgmReturn (AsList" - " (UpdateRow '('/dc-1/Zoo/Animals '0 '0:0) row myUpd)" + " (UpdateRow '('/dc-1/Zoo/Animals '0 '0:0) row myUpd)" "))" "(return pgmReturn)" ")"); diff --git a/ydb/public/lib/deprecated/kicli/error.cpp b/ydb/public/lib/deprecated/kicli/error.cpp index 41f9f7e55d..473a981f60 100644 --- a/ydb/public/lib/deprecated/kicli/error.cpp +++ b/ydb/public/lib/deprecated/kicli/error.cpp @@ -280,12 +280,12 @@ void TError::Throw() const { TError::TError(const TResult& result) : Facility(EFacility::FacilityMessageBus) , Code(NBus::MESSAGE_OK) - , YdbStatus(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED) + , YdbStatus(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED) { if (result.TransportStatus != NBus::MESSAGE_OK) { Facility = EFacility::FacilityMessageBus; Code = result.TransportStatus; - Message = result.TransportErrorMessage; + Message = result.TransportErrorMessage; } else { if (result.Reply->GetHeader()->Type == NMsgBusProxy::MTYPE_CLIENT_RESPONSE) { const NKikimrClient::TResponse& response = result.GetResult<NKikimrClient::TResponse>(); @@ -312,11 +312,11 @@ TError::TError(const TResult& result) Message = response.GetDataShardErrors(); } if (response.HasMiniKQLCompileResults() && response.GetMiniKQLCompileResults().ProgramCompileErrorsSize() > 0) { - NYql::TIssues issues; - NYql::IssuesFromMessage(response.GetMiniKQLCompileResults().GetProgramCompileErrors(), issues); - TStringStream message; - issues.PrintTo(message); - Message = message.Str(); + NYql::TIssues issues; + NYql::IssuesFromMessage(response.GetMiniKQLCompileResults().GetProgramCompileErrors(), issues); + TStringStream message; + issues.PrintTo(message); + Message = message.Str(); } if (response.HasMiniKQLErrors()) { if (!Message.empty()) @@ -332,8 +332,8 @@ TError::EFacility TError::GetFacility() const { } Ydb::StatusIds::StatusCode TError::GetYdbStatus() const { - return YdbStatus; + return YdbStatus; +} + } - } -} diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index 426eddb25c..9dc472d231 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -63,8 +63,8 @@ public: static const TType String; static const TType String4k; static const TType String2m; - static const TType Yson; - static const TType Json; + static const TType Yson; + static const TType Json; static const TType JsonDocument; static const TType Timestamp; @@ -204,7 +204,7 @@ public: }; // for compatibility -template <typename ParameterType, NScheme::TTypeId SchemeType = SchemeMapper<ParameterType>::SchemeType> +template <typename ParameterType, NScheme::TTypeId SchemeType = SchemeMapper<ParameterType>::SchemeType> class TParameterValue : public TStructMemberValue<TDataValue<ParameterType, SchemeType>> { public: TParameterValue(const TString& name, ParameterType parameter) @@ -289,7 +289,7 @@ public: TString GetMessage() const; void Throw() const; EFacility GetFacility() const; - // Returns YDB status + // Returns YDB status Ydb::StatusIds::StatusCode GetYdbStatus() const; protected: @@ -342,11 +342,11 @@ public: protected: TResult(NBus::EMessageStatus transportStatus); - TResult(NBus::EMessageStatus transportStatus, const TString& message); + TResult(NBus::EMessageStatus transportStatus, const TString& message); TResult(TAutoPtr<NBus::TBusMessage> reply); NBus::EMessageStatus TransportStatus; - TString TransportErrorMessage; + TString TransportErrorMessage; TAtomicSharedPtr<NBus::TBusMessage> Reply; }; diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp index 9b59847279..4156d72d4b 100644 --- a/ydb/public/lib/deprecated/kicli/kikimr.cpp +++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp @@ -77,9 +77,9 @@ public: , RetryQueue(*this) {} - void OnCall(NBus::EMessageStatus status, const TString& transportErrMessage, NThreading::TPromise<TResult> promise, TAutoPtr<NBus::TBusMessage> request, TAutoPtr<NBus::TBusMessage> reply) { + void OnCall(NBus::EMessageStatus status, const TString& transportErrMessage, NThreading::TPromise<TResult> promise, TAutoPtr<NBus::TBusMessage> request, TAutoPtr<NBus::TBusMessage> reply) { if (status != NBus::MESSAGE_OK) { - promise.SetValue(TResult(status, transportErrMessage)); + promise.SetValue(TResult(status, transportErrMessage)); } if (status == NBus::MESSAGE_OK) { switch (reply->GetHeader()->Type) { @@ -174,7 +174,7 @@ public: NBus::EMessageStatus status, TAutoPtr<NBus::TBusMessage> request, TAutoPtr<NBus::TBusMessage> reply) mutable -> void { - OnCall(status, "", promise, request, reply); + OnCall(status, "", promise, request, reply); }); } @@ -212,14 +212,14 @@ public: RequestType* x = static_cast<RequestType*>(request.Get()); const auto& proto = x->Record; auto callback = [this, request, promise]( - const NGRpcProxy::TGrpcError* error, + const NGRpcProxy::TGrpcError* error, const typename ResponseType::RecordType& proto) { if (error) { - OnCall(MapGrpcStatus(error->second), error->first, promise, request, nullptr); + OnCall(MapGrpcStatus(error->second), error->first, promise, request, nullptr); } else { auto reply = new ResponseType; reply->Record = proto; - OnCall(NBus::MESSAGE_OK, "", promise, request, reply); + OnCall(NBus::MESSAGE_OK, "", promise, request, reply); } }; (GRpcClient.Get()->*func)(proto, std::move(callback)); @@ -330,19 +330,19 @@ public: // we cleanup later to avoid dead lock because we are still inside callback of current client return [gRpcClient]() {}; } -private: - static NBus::EMessageStatus MapGrpcStatus(int status) { - switch (status) { - case grpc::StatusCode::OK: - return NBus::EMessageStatus::MESSAGE_OK; - case grpc::StatusCode::DEADLINE_EXCEEDED: - return NBus::EMessageStatus::MESSAGE_TIMEOUT; - case grpc::StatusCode::RESOURCE_EXHAUSTED: - return NBus::EMessageStatus::MESSAGE_BUSY; - default: - return NBus::MESSAGE_CONNECT_FAILED; - } - } +private: + static NBus::EMessageStatus MapGrpcStatus(int status) { + switch (status) { + case grpc::StatusCode::OK: + return NBus::EMessageStatus::MESSAGE_OK; + case grpc::StatusCode::DEADLINE_EXCEEDED: + return NBus::EMessageStatus::MESSAGE_TIMEOUT; + case grpc::StatusCode::RESOURCE_EXHAUSTED: + return NBus::EMessageStatus::MESSAGE_BUSY; + default: + return NBus::MESSAGE_CONNECT_FAILED; + } + } }; TKikimr::TRetryQueue::TRetryQueue(TKikimr::TImpl& kikimr) diff --git a/ydb/public/lib/deprecated/kicli/result.cpp b/ydb/public/lib/deprecated/kicli/result.cpp index 9558cb23e7..0f86ba4340 100644 --- a/ydb/public/lib/deprecated/kicli/result.cpp +++ b/ydb/public/lib/deprecated/kicli/result.cpp @@ -12,11 +12,11 @@ TResult::TResult(NBus::EMessageStatus transportStatus) : TransportStatus(transportStatus) {} -TResult::TResult(NBus::EMessageStatus transportStatus, const TString& message) - : TransportStatus(transportStatus) - , TransportErrorMessage(message) -{} - +TResult::TResult(NBus::EMessageStatus transportStatus, const TString& message) + : TransportStatus(transportStatus) + , TransportErrorMessage(message) +{} + TResult::TResult(TAutoPtr<NBus::TBusMessage> reply) : TransportStatus(NBus::MESSAGE_OK) , Reply(reply.Release()) @@ -101,35 +101,35 @@ const YdbOld::ResultSet &TReadTableResult::GetResultSet() const { template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Value &value, const YdbOld::DataType &type) { - switch (type.id()) { + switch (type.id()) { case NYql::NProto::Bool: - return value.bool_value() ? "true" : "false"; + return value.bool_value() ? "true" : "false"; case NYql::NProto::Uint64: - return ToString(value.uint64_value()); + return ToString(value.uint64_value()); case NScheme::NTypeIds::Int64: - return ToString(value.int64_value()); + return ToString(value.int64_value()); case NScheme::NTypeIds::Uint32: - return ToString(value.uint32_value()); + return ToString(value.uint32_value()); case NScheme::NTypeIds::Int32: - return ToString(value.int32_value()); + return ToString(value.int32_value()); case NScheme::NTypeIds::Uint16: - return ToString(static_cast<ui16>(value.uint32_value())); + return ToString(static_cast<ui16>(value.uint32_value())); case NScheme::NTypeIds::Int16: - return ToString(static_cast<i16>(value.int32_value())); + return ToString(static_cast<i16>(value.int32_value())); case NScheme::NTypeIds::Uint8: - return ToString(static_cast<ui8>(value.uint32_value())); + return ToString(static_cast<ui8>(value.uint32_value())); case NScheme::NTypeIds::Int8: - return ToString(static_cast<i8>(value.int32_value())); + return ToString(static_cast<i8>(value.int32_value())); case NScheme::NTypeIds::Double: - return ToString(value.double_value()); + return ToString(value.double_value()); case NScheme::NTypeIds::Float: - return ToString(value.float_value()); + return ToString(value.float_value()); case NScheme::NTypeIds::Utf8: case NScheme::NTypeIds::Json: case NScheme::NTypeIds::Yson: - return "\"" + TFormatCSV::EscapeString(value.text_value()) + "\""; + return "\"" + TFormatCSV::EscapeString(value.text_value()) + "\""; case NScheme::NTypeIds::String: - return value.bytes_value(); + return value.bytes_value(); case NScheme::NTypeIds::Date: { auto val = TInstant::Days(value.uint32_value()); @@ -157,7 +157,7 @@ template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Va NYql::NDecimal::TInt128 val; auto p = reinterpret_cast<char*>(&val); reinterpret_cast<ui64*>(p)[0] = value.low_128(); - reinterpret_cast<ui64*>(p)[1] = value.high_128(); + reinterpret_cast<ui64*>(p)[1] = value.high_128(); // In Kikimr the only decimal column type supported is Decimal(22,9). return NYql::NDecimal::ToString(val, NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE); } @@ -170,20 +170,20 @@ template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Va const YdbOld::Type &type) { switch (type.type_type_case()) { case YdbOld::Type::kDataType: - return ValueToString<TFormatCSV>(value, type.data_type()); + return ValueToString<TFormatCSV>(value, type.data_type()); case YdbOld::Type::kOptionalType: - if (value.Hasnull_flag_value()) + if (value.Hasnull_flag_value()) return "NULL"; // If we have Value field for optional type then it is always // should be followed (even for Optional<Variant<T>> case). - if (value.Hasnested_value()) - return ValueToString<TFormatCSV>(value.nested_value(), type.optional_type().item()); - return ValueToString<TFormatCSV>(value, type.optional_type().item()); + if (value.Hasnested_value()) + return ValueToString<TFormatCSV>(value.nested_value(), type.optional_type().item()); + return ValueToString<TFormatCSV>(value, type.optional_type().item()); case YdbOld::Type::kListType: { TString res = "["; - const auto &items = value.items(); - const auto &itemType = type.list_type().item(); + const auto &items = value.items(); + const auto &itemType = type.list_type().item(); for (auto it = items.begin(); it != items.end(); ++it) { if (it != items.begin()) res += ","; @@ -195,8 +195,8 @@ template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Va case YdbOld::Type::kTupleType: { TString res = "["; - const auto &items = value.items(); - const auto &types = type.tuple_type().elements(); + const auto &items = value.items(); + const auto &types = type.tuple_type().elements(); for (int i = 0; i < items.size(); ++i) { if (i) res += ","; @@ -208,12 +208,12 @@ template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Va case YdbOld::Type::kStructType: { TString res = "{"; - const auto &items = value.items(); - const auto &members = type.struct_type().members(); + const auto &items = value.items(); + const auto &members = type.struct_type().members(); for (int i = 0; i < members.size(); ++i) { if (i) res += ","; - res += ValueToString<TFormatCSV>(items[i], members[i].type()); + res += ValueToString<TFormatCSV>(items[i], members[i].type()); } res += "}"; return res; @@ -221,14 +221,14 @@ template <> TString TReadTableResult::ValueToString<TFormatCSV>(const YdbOld::Va case YdbOld::Type::kDictType: { TString res = "["; - const auto &pairs = value.pairs(); - const auto &dictType = type.dict_type(); + const auto &pairs = value.pairs(); + const auto &dictType = type.dict_type(); for (int i = 0; i < pairs.size(); ++i) { if (i) res += ","; - res += ValueToString<TFormatCSV>(pairs[i].key(), dictType.key()); + res += ValueToString<TFormatCSV>(pairs[i].key(), dictType.key()); res += ":"; - res += ValueToString<TFormatCSV>(pairs[i].payload(), dictType.payload()); + res += ValueToString<TFormatCSV>(pairs[i].payload(), dictType.payload()); } res += "]"; return res; @@ -244,11 +244,11 @@ template <> TString TReadTableResult::GetTypeText<TFormatCSV>(const TFormatCSV & auto &proto = GetResultSet(); TString res; bool first = true; - for (auto &meta : proto.column_meta()) { + for (auto &meta : proto.column_meta()) { if (!first) res += format.Delim; first = false; - res += meta.name(); + res += meta.name(); } return res; } @@ -262,16 +262,16 @@ template <> TString TReadTableResult::GetValueText<TFormatCSV>(const TFormatCSV res += "\n"; } - auto &colTypes = proto.column_meta(); + auto &colTypes = proto.column_meta(); bool first = true; - for (auto &row : proto.rows()) { + for (auto &row : proto.rows()) { if (!first) res += "\n"; first = false; for (int i = 0; i < colTypes.size(); ++i) { if (i) res += format.Delim; - res += ValueToString<TFormatCSV>(row.items(i), colTypes[i].type()); + res += ValueToString<TFormatCSV>(row.items(i), colTypes[i].type()); } } @@ -292,8 +292,8 @@ TPreparedQuery TPrepareResult::GetQuery() const { const auto& compileResult = response.GetMiniKQLCompileResults(); Y_VERIFY(compileResult.HasCompiledProgram(), "Compile error (%" PRIu64 "): %" PRIu32 ":%" PRIu32 " %s", compileResult.ProgramCompileErrorsSize(), - (compileResult.ProgramCompileErrorsSize() ? compileResult.GetProgramCompileErrors(0).position().row() : 0u), - (compileResult.ProgramCompileErrorsSize() ? compileResult.GetProgramCompileErrors(0).position().column() : 0u), + (compileResult.ProgramCompileErrorsSize() ? compileResult.GetProgramCompileErrors(0).position().row() : 0u), + (compileResult.ProgramCompileErrorsSize() ? compileResult.GetProgramCompileErrors(0).position().column() : 0u), (compileResult.ProgramCompileErrorsSize() ? compileResult.GetProgramCompileErrors(0).message().data() : "") ); return TPreparedQuery(*Query, compileResult.GetCompiledProgram()); diff --git a/ydb/public/lib/deprecated/kicli/schema.cpp b/ydb/public/lib/deprecated/kicli/schema.cpp index f575700d3a..d1ee0384f8 100644 --- a/ydb/public/lib/deprecated/kicli/schema.cpp +++ b/ydb/public/lib/deprecated/kicli/schema.cpp @@ -51,8 +51,8 @@ const TType TType::Utf8(NScheme::NTypeIds::Utf8); const TType TType::String(NScheme::NTypeIds::String); const TType TType::String4k(NScheme::NTypeIds::String4k); const TType TType::String2m(NScheme::NTypeIds::String2m); -const TType TType::Yson(NScheme::NTypeIds::Yson); -const TType TType::Json(NScheme::NTypeIds::Json); +const TType TType::Yson(NScheme::NTypeIds::Yson); +const TType TType::Json(NScheme::NTypeIds::Json); const TType TType::JsonDocument(NScheme::NTypeIds::JsonDocument); const TType TType::Timestamp(NScheme::NTypeIds::Timestamp); diff --git a/ydb/public/lib/deprecated/kicli/ut/ya.make b/ydb/public/lib/deprecated/kicli/ut/ya.make index 3ae512c9dd..ce45125f07 100644 --- a/ydb/public/lib/deprecated/kicli/ut/ya.make +++ b/ydb/public/lib/deprecated/kicli/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(ydb/public/lib/deprecated/kicli) -OWNER( - xenoxeno - g:kikimr -) +OWNER( + xenoxeno + g:kikimr +) TIMEOUT(600) diff --git a/ydb/public/lib/deprecated/kicli/ya.make b/ydb/public/lib/deprecated/kicli/ya.make index 1acf0cebab..d9aa54bc4c 100644 --- a/ydb/public/lib/deprecated/kicli/ya.make +++ b/ydb/public/lib/deprecated/kicli/ya.make @@ -20,7 +20,7 @@ SRCS( ) PEERDIR( - contrib/libs/grpc + contrib/libs/grpc library/cpp/actors/core library/cpp/threading/future ydb/core/protos diff --git a/ydb/public/lib/deprecated/ya.make b/ydb/public/lib/deprecated/ya.make index 4e37881249..84dab1af07 100644 --- a/ydb/public/lib/deprecated/ya.make +++ b/ydb/public/lib/deprecated/ya.make @@ -1,4 +1,4 @@ RECURSE( client - kicli + kicli ) diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp index 8588ec5c01..2984f680b7 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp @@ -64,7 +64,7 @@ std::pair<TString, bool> TScanResult::GetLastKey() const { class TScanClient::TImpl : public TClientImplCommon<TScanClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} + : TClientImplCommon(std::move(connections), settings) {} TAsyncScanResult Scan( const TString& table, const TVector<TString>& columns, @@ -88,8 +88,8 @@ public: auto promise = NThreading::NewPromise<TScanResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::ClickhouseInternal::ScanResult result; if (any) { any->UnpackTo(&result); @@ -97,7 +97,7 @@ public: // Cerr << result << Endl; TScanResult val(new TScanResult::TResultImpl(std::move(result)), - TStatus(std::move(status))); + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; @@ -107,7 +107,7 @@ public: extractor, &Ydb::ClickhouseInternal::V1::ClickhouseInternalService::Stub::AsyncScan, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_, settings.Endpoint_); @@ -343,7 +343,7 @@ class TMetaClient::TImpl : public TClientImplCommon<TMetaClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} + : TClientImplCommon(std::move(connections), settings) {} TAsyncGetShardLocationsResult GetShardLocations( const TVector<ui64>& tabletIds, @@ -356,15 +356,15 @@ public: auto promise = NThreading::NewPromise<TGetShardLocationsResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::ClickhouseInternal::GetShardLocationsResult result; if (any) { any->UnpackTo(&result); } TGetShardLocationsResult val(new TGetShardLocationsResult::TResultImpl(std::move(result)), - TStatus(std::move(status))); + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; @@ -374,7 +374,7 @@ public: extractor, &Ydb::ClickhouseInternal::V1::ClickhouseInternalService::Stub::AsyncGetShardLocations, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); @@ -392,7 +392,7 @@ public: auto promise = NThreading::NewPromise<TDescribeTableResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::ClickhouseInternal::DescribeTableResult result; if (any) { @@ -400,7 +400,7 @@ public: } TDescribeTableResult val(new TDescribeTableResult::TResultImpl(std::move(result)), - TStatus(std::move(status))); + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; @@ -419,7 +419,7 @@ public: template<class TProtoResult, class TResultWrapper> auto MakeResultExtractor(NThreading::TPromise<TResultWrapper> promise) { - return [promise = std::move(promise)] + return [promise = std::move(promise)] (google::protobuf::Any* any, TPlainStatus status) mutable { std::unique_ptr<TProtoResult> result; if (any) { @@ -429,7 +429,7 @@ public: promise.SetValue( TResultWrapper( - TStatus(std::move(status)), + TStatus(std::move(status)), std::move(result))); }; } diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp index f7c718ad3c..8765c0c750 100644 --- a/ydb/public/lib/experimental/ydb_experimental.cpp +++ b/ydb/public/lib/experimental/ydb_experimental.cpp @@ -32,10 +32,10 @@ public: using TGRpcStatus = NGrpc::TGrpcStatus; using TBatchReadResult = std::pair<TResponse, TGRpcStatus>; - TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) + TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) : StreamProcessor_(streamProcessor) , Finished_(false) - , Endpoint_(endpoint) + , Endpoint_(endpoint) {} ~TReaderImpl() { @@ -52,13 +52,13 @@ public: auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { if (!grpcStatus.Ok()) { self->Finished_ = true; - promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); + promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); } else { NYql::TIssues issues; NYql::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); - TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; - TStatus status{std::move(plainStatus)}; + TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; + TStatus status{std::move(plainStatus)}; if (self->Response_.result().has_result_set()) { promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), @@ -83,13 +83,13 @@ private: TStreamProcessorPtr StreamProcessor_; TResponse Response_; bool Finished_; - TString Endpoint_; + TString Endpoint_; }; TStreamPartIterator::TStreamPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status) - : TStatus(std::move(status)) + TPlainStatus&& status) + : TStatus(std::move(status)) , ReaderImpl_(impl) {} @@ -104,7 +104,7 @@ public: using TStreamProcessorPtr = TStreamPartIterator::TReaderImpl::TStreamProcessorPtr; TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} + : TClientImplCommon(std::move(connections), settings) {} TFuture<std::pair<TPlainStatus, TStreamProcessorPtr>> ExecuteStreamQueryInternal(const TString& query, const TParams* params, const TExecuteStreamQuerySettings& settings) @@ -153,16 +153,16 @@ public: { auto promise = NewPromise<TStreamPartIterator>(); - auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, + auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, TStreamQueryClient::TImpl::TStreamProcessorPtr>> future) mutable { Y_ASSERT(future.HasValue()); auto pair = future.ExtractValue(); promise.SetValue(TStreamPartIterator( pair.second - ? std::make_shared<TStreamPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) + ? std::make_shared<TStreamPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) : nullptr, - std::move(pair.first)) + std::move(pair.first)) ); }; @@ -175,7 +175,7 @@ TStreamQueryClient::TStreamQueryClient(const TDriver& driver, const TCommonClien : Impl_(new TImpl(CreateInternalInterface(driver), settings)) {} TParamsBuilder TStreamQueryClient::GetParamsBuilder() { - return TParamsBuilder(); + return TParamsBuilder(); } TAsyncStreamPartIterator TStreamQueryClient::ExecuteStreamQuery(const TString& query, diff --git a/ydb/public/lib/experimental/ydb_experimental.h b/ydb/public/lib/experimental/ydb_experimental.h index cd1a3a9439..3cdd8f6dc8 100644 --- a/ydb/public/lib/experimental/ydb_experimental.h +++ b/ydb/public/lib/experimental/ydb_experimental.h @@ -66,7 +66,7 @@ public: private: TStreamPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status + TPlainStatus&& status ); std::shared_ptr<TReaderImpl> ReaderImpl_; }; diff --git a/ydb/public/lib/experimental/ydb_s3_internal.cpp b/ydb/public/lib/experimental/ydb_s3_internal.cpp index 6323d44f30..ac1c5025ed 100644 --- a/ydb/public/lib/experimental/ydb_s3_internal.cpp +++ b/ydb/public/lib/experimental/ydb_s3_internal.cpp @@ -43,7 +43,7 @@ void SetProtoValue(Ydb::TypedValue& out, TValue&& in) { class TS3InternalClient::TImpl : public TClientImplCommon<TS3InternalClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} + : TClientImplCommon(std::move(connections), settings) {} TAsyncS3ListingResult S3Listing(const TString& tableName, TValue&& keyPrefix, @@ -67,8 +67,8 @@ public: auto promise = NThreading::NewPromise<TS3ListingResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::S3Internal::S3ListingResult result; if (any) { any->UnpackTo(&result); @@ -77,7 +77,7 @@ public: TResultSet contents(result.Getcontents()); TS3ListingResult val(std::move(commonPrefixes), std::move(contents), result.Getkey_suffix_size(), - TStatus(std::move(status))); + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; @@ -86,7 +86,7 @@ public: extractor, &Ydb::S3Internal::V1::S3InternalService::Stub::AsyncS3Listing, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); diff --git a/ydb/public/lib/idx_test/idx_test.h b/ydb/public/lib/idx_test/idx_test.h index fcda1df165..9444cf1463 100644 --- a/ydb/public/lib/idx_test/idx_test.h +++ b/ydb/public/lib/idx_test/idx_test.h @@ -1,90 +1,90 @@ -#pragma once - -#include <util/system/types.h> +#pragma once + +#include <util/system/types.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <memory> - + +#include <memory> + #include <library/cpp/json/json_value.h> - -namespace NIdxTest { - -class IDataProvider { -public: - using TPtr = std::unique_ptr<IDataProvider>; -public: - virtual ~IDataProvider() = default; - - virtual NYdb::NTable::TTableDescription GetTableSchema() = 0; - virtual TMaybe<TVector<NYdb::TValue>> GetBatch(ui32 upload_id) = 0; - virtual NYdb::TType GetRowType() = 0; -}; - -class IUploader { -public: - using TPtr = std::unique_ptr<IUploader>; -public: - virtual ~IUploader() = default; - - virtual void Run(IDataProvider::TPtr dataProvider) = 0; -}; - -class IWorkLoader { -public: - using TPtr = std::unique_ptr<IWorkLoader>; - enum ELoadCommand { - LC_UPSERT = 1, - LC_INSERT = 2, - LC_UPDATE = 4, - LC_UPDATE_ON = 8, - LC_REPLACE = 16, - LC_DELETE = 32, - LC_DELETE_ON = 64, - LC_SELECT = 128, - LC_UPSERT_IF_UNIQ = 256, - LC_ALTER_ADD_INDEX = 512, - LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN = 1024 - }; -public: - virtual ~IWorkLoader() = default; - - struct TRunSettings { - size_t RunLimit; - size_t Infly; - bool NewEngine; - }; - - virtual NJson::TJsonValue Run(const TString& tableName, ui32 loadCommands, const TRunSettings& settings) = 0; -}; - -class IProgressTracker { -public: - using TPtr = std::unique_ptr<IProgressTracker>; -public: - virtual ~IProgressTracker() = default; - virtual void Start(const TString& progressString, const TString& freeMessage) = 0; - virtual void Update(size_t progress) = 0; - virtual void Finish(const TString& freeMessage) = 0; -}; - -class IChecker { -public: - using TPtr = std::unique_ptr<IChecker>; -public: - virtual ~IChecker() = default; - - virtual void Run(const TString& tableName) = 0; -}; - -struct TUploaderParams { - ui32 ShardsCount; -}; - -IUploader::TPtr CreateUploader(NYdb::TDriver& driver, const TString& table, const TUploaderParams& params); -IDataProvider::TPtr CreateDataProvider(ui32 rowsCount, ui32 shardsCount, NYdb::NTable::TTableDescription tableDesc); -IWorkLoader::TPtr CreateWorkLoader(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker = IProgressTracker::TPtr(nullptr)); -IChecker::TPtr CreateChecker(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker = IProgressTracker::TPtr(nullptr)); - -IProgressTracker::TPtr CreateStderrProgressTracker(size_t subsampling, const TString& title = ""); - -} // namespace NIdxTest + +namespace NIdxTest { + +class IDataProvider { +public: + using TPtr = std::unique_ptr<IDataProvider>; +public: + virtual ~IDataProvider() = default; + + virtual NYdb::NTable::TTableDescription GetTableSchema() = 0; + virtual TMaybe<TVector<NYdb::TValue>> GetBatch(ui32 upload_id) = 0; + virtual NYdb::TType GetRowType() = 0; +}; + +class IUploader { +public: + using TPtr = std::unique_ptr<IUploader>; +public: + virtual ~IUploader() = default; + + virtual void Run(IDataProvider::TPtr dataProvider) = 0; +}; + +class IWorkLoader { +public: + using TPtr = std::unique_ptr<IWorkLoader>; + enum ELoadCommand { + LC_UPSERT = 1, + LC_INSERT = 2, + LC_UPDATE = 4, + LC_UPDATE_ON = 8, + LC_REPLACE = 16, + LC_DELETE = 32, + LC_DELETE_ON = 64, + LC_SELECT = 128, + LC_UPSERT_IF_UNIQ = 256, + LC_ALTER_ADD_INDEX = 512, + LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN = 1024 + }; +public: + virtual ~IWorkLoader() = default; + + struct TRunSettings { + size_t RunLimit; + size_t Infly; + bool NewEngine; + }; + + virtual NJson::TJsonValue Run(const TString& tableName, ui32 loadCommands, const TRunSettings& settings) = 0; +}; + +class IProgressTracker { +public: + using TPtr = std::unique_ptr<IProgressTracker>; +public: + virtual ~IProgressTracker() = default; + virtual void Start(const TString& progressString, const TString& freeMessage) = 0; + virtual void Update(size_t progress) = 0; + virtual void Finish(const TString& freeMessage) = 0; +}; + +class IChecker { +public: + using TPtr = std::unique_ptr<IChecker>; +public: + virtual ~IChecker() = default; + + virtual void Run(const TString& tableName) = 0; +}; + +struct TUploaderParams { + ui32 ShardsCount; +}; + +IUploader::TPtr CreateUploader(NYdb::TDriver& driver, const TString& table, const TUploaderParams& params); +IDataProvider::TPtr CreateDataProvider(ui32 rowsCount, ui32 shardsCount, NYdb::NTable::TTableDescription tableDesc); +IWorkLoader::TPtr CreateWorkLoader(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker = IProgressTracker::TPtr(nullptr)); +IChecker::TPtr CreateChecker(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker = IProgressTracker::TPtr(nullptr)); + +IProgressTracker::TPtr CreateStderrProgressTracker(size_t subsampling, const TString& title = ""); + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_checker.cpp b/ydb/public/lib/idx_test/idx_test_checker.cpp index 0cfa18cfe6..f87f580151 100644 --- a/ydb/public/lib/idx_test/idx_test_checker.cpp +++ b/ydb/public/lib/idx_test/idx_test_checker.cpp @@ -1,282 +1,282 @@ -#include "idx_test.h" -#include "idx_test_common.h" - +#include "idx_test.h" +#include "idx_test_common.h" + #include <ydb/public/lib/yson_value/ydb_yson_value.h> - -#include <util/generic/hash.h> - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace NIdxTest { - -class TChecker : public IChecker { -public: - TChecker(NYdb::NTable::TTableClient&& client, IProgressTracker::TPtr&& progressTracker) - : Client_(std::move(client)) - , ProgressTracker_(std::move(progressTracker)) - {} - - void Run(const TString& tableName) { - DescribeTable(tableName); - - if (!TableDescription_) { - throw yexception() << "Unable to load table description"; - } - if (!TableDescription_.GetRef().GetIndexDescriptions()) { - throw yexception() << "Index metadata was not found"; - } - CheckIndexes(tableName); - } - -private: - void CheckIndexData(const TString& tableName, - const TIndexDescription& indexDesc, - const THashMap<TString, size_t>& indexedColumnsMap, - const THashMap<TString, TVector<TValue>>& mainTable) { - TMaybe<TTablePartIterator> tableIterator; - - auto settings = TReadTableSettings(); - - // map indexTableColumn id -> dataTableColumnId - TVector<size_t> checkColumnsMap; - checkColumnsMap.resize(indexDesc.GetIndexColumns().size() + indexDesc.GetDataColumns().size()); - + +#include <util/generic/hash.h> + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace NIdxTest { + +class TChecker : public IChecker { +public: + TChecker(NYdb::NTable::TTableClient&& client, IProgressTracker::TPtr&& progressTracker) + : Client_(std::move(client)) + , ProgressTracker_(std::move(progressTracker)) + {} + + void Run(const TString& tableName) { + DescribeTable(tableName); + + if (!TableDescription_) { + throw yexception() << "Unable to load table description"; + } + if (!TableDescription_.GetRef().GetIndexDescriptions()) { + throw yexception() << "Index metadata was not found"; + } + CheckIndexes(tableName); + } + +private: + void CheckIndexData(const TString& tableName, + const TIndexDescription& indexDesc, + const THashMap<TString, size_t>& indexedColumnsMap, + const THashMap<TString, TVector<TValue>>& mainTable) { + TMaybe<TTablePartIterator> tableIterator; + + auto settings = TReadTableSettings(); + + // map indexTableColumn id -> dataTableColumnId + TVector<size_t> checkColumnsMap; + checkColumnsMap.resize(indexDesc.GetIndexColumns().size() + indexDesc.GetDataColumns().size()); + THashMap<TString, size_t> indexColumns; - { - size_t j = 0; - for (size_t i = 0; i < indexDesc.GetIndexColumns().size(); i++, j++) { - const auto& col = indexDesc.GetIndexColumns()[i]; - auto it = indexedColumnsMap.find(col); - Y_VERIFY(it != indexedColumnsMap.end()); - settings.AppendColumns(col); - checkColumnsMap[j] = it->second; - indexColumns[col] = i; - } - - for (size_t i = 0; i < indexDesc.GetDataColumns().size(); i++, j++) { - const auto& col = indexDesc.GetDataColumns()[i]; - auto it = indexedColumnsMap.find(col); - Y_VERIFY(it != indexedColumnsMap.end()); - settings.AppendColumns(col); - checkColumnsMap[j] = it->second; - } - } - - - TVector<size_t> pkColumnIdx; + { + size_t j = 0; + for (size_t i = 0; i < indexDesc.GetIndexColumns().size(); i++, j++) { + const auto& col = indexDesc.GetIndexColumns()[i]; + auto it = indexedColumnsMap.find(col); + Y_VERIFY(it != indexedColumnsMap.end()); + settings.AppendColumns(col); + checkColumnsMap[j] = it->second; + indexColumns[col] = i; + } + + for (size_t i = 0; i < indexDesc.GetDataColumns().size(); i++, j++) { + const auto& col = indexDesc.GetDataColumns()[i]; + auto it = indexedColumnsMap.find(col); + Y_VERIFY(it != indexedColumnsMap.end()); + settings.AppendColumns(col); + checkColumnsMap[j] = it->second; + } + } + + + TVector<size_t> pkColumnIdx; size_t includedColumns = 0; // count of PK columns included in the index (before i-th PK column) - for (size_t i = 0; i < TableDescription_.GetRef().GetPrimaryKeyColumns().size(); i++) { - const auto& col = TableDescription_.GetRef().GetPrimaryKeyColumns()[i]; + for (size_t i = 0; i < TableDescription_.GetRef().GetPrimaryKeyColumns().size(); i++) { + const auto& col = TableDescription_.GetRef().GetPrimaryKeyColumns()[i]; auto it = indexColumns.find(col); if (it != indexColumns.end()) { // PK column is included in the secondary index - pkColumnIdx.push_back(it->second); + pkColumnIdx.push_back(it->second); ++includedColumns; - } else { - settings.AppendColumns(col); + } else { + settings.AppendColumns(col); pkColumnIdx.push_back(checkColumnsMap.size() + i - includedColumns); - } - } - - - const TString indexTableName = tableName + "/" + indexDesc.GetIndexName() + "/indexImplTable"; - - if (ProgressTracker_) { - ProgressTracker_->Start("rows read", "Reading indexTable table " + indexTableName + "..."); - } - ThrowOnError(Client_.RetryOperationSync([indexTableName, settings, &tableIterator](TSession session) { - auto result = session.ReadTable(indexTableName, settings).GetValueSync(); - - if (result.IsSuccess()) { - tableIterator = result; - } - - return result; - })); - - size_t totalRows = 0; - for (;;) { - auto tablePart = tableIterator->ReadNext().GetValueSync(); - if (!tablePart.IsSuccess()) { - if (tablePart.EOS()) { - break; - } - - ThrowOnError(tablePart); - } - - auto rsParser = TResultSetParser(tablePart.ExtractPart()); - while (rsParser.TryNextRow()) { - if (ProgressTracker_) { - ProgressTracker_->Update(totalRows); - } - totalRows++; - TString key; - - for (const auto& id : pkColumnIdx) { - auto value = rsParser.GetValue(id); - key += NYdb::FormatValueYson(value); - } - - auto mainTableRow = mainTable.find(key); - if (mainTableRow == mainTable.end()) { - throw yexception() << "index table has unknown key: " << key; - } - - for (size_t id = 0; id < checkColumnsMap.size(); id++) { - auto value = rsParser.GetValue(id); - const auto mainTableColumnId = checkColumnsMap[id]; - - const auto& valueFromMain = mainTableRow->second[mainTableColumnId]; - - if (NYdb::FormatValueYson(value) != NYdb::FormatValueYson(valueFromMain)) { - throw yexception() << " value missmatch for row with key: " << key; - } - } - } - } - - if (totalRows != mainTable.size()) { - throw yexception() << "rows count missmatch, index table " - << indexTableName << " has " << totalRows - << " rows, but main table has " << mainTable.size() << " rows."; - } - if (ProgressTracker_) { - ProgressTracker_->Finish("Done."); - } - } - - void ReadMainTable( - const TString& tableName, - const TVector<TString>& columns, - const THashMap<TString, size_t>& columnMap, - THashMap<TString, TVector<TValue>>& buf) - { - TMaybe<TTablePartIterator> tableIterator; - - auto settings = TReadTableSettings(); - - for (const auto& col : columns) { - settings.AppendColumns(col); - } - - if (ProgressTracker_) { - ProgressTracker_->Start("rows read", "Reading main table " + tableName + "..."); - } - - // columnsMap.size() columns will be readed, but probably there are no pk here - // so append pk column in this case - size_t pkPos = columnMap.size(); - TVector<size_t> pkMap; - - for (const auto& col : TableDescription_.GetRef().GetPrimaryKeyColumns()) { - const auto it = columnMap.find(col); - if (it == columnMap.end()) { - settings.AppendColumns(col); - pkMap.push_back(pkPos); - pkPos++; - } else { - pkMap.push_back(it->second); - } - } - - ThrowOnError(Client_.RetryOperationSync([tableName, settings, &tableIterator](TSession session) { - auto result = session.ReadTable(tableName, settings).GetValueSync(); - - if (result.IsSuccess()) { - tableIterator = result; - } - - return result; - })); - - size_t rows = 0; - for (;;) { - auto tablePart = tableIterator->ReadNext().GetValueSync(); - if (!tablePart.IsSuccess()) { - if (tablePart.EOS()) { - break; - } - - ThrowOnError(tablePart); - } - - auto rsParser = TResultSetParser(tablePart.ExtractPart()); - - while (rsParser.TryNextRow()) { - TString key; - TVector<TValue> values; - - for (size_t i = 0; i < columns.size(); i++) { - values.push_back(rsParser.GetValue(i)); - } - - for (auto i : pkMap) { - auto value = rsParser.GetValue(i); - key += NYdb::FormatValueYson(value); - } - Y_VERIFY(buf.insert(std::make_pair(key, values)).second); - if (ProgressTracker_) { - ProgressTracker_->Update(rows++); - } - } - } - if (ProgressTracker_) { - ProgressTracker_->Finish("Done."); - } - } - - void CheckIndexes(const TString& tableName) { - - // Some magic for case of using same columns for multiple indexes - // or pk and indexes - - // Columns need to check index - TVector<TString> indexedColumns; - // Map column name -> position in result row in data table - THashMap<TString, size_t> indexedColumnsMap; - - size_t id = 0; - for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { - for (const auto& col : index.GetIndexColumns()) { - auto it = indexedColumnsMap.insert({col, id}); - if (it.second) { - indexedColumns.push_back(col); - id++; - } - } - - for (const auto& col : index.GetDataColumns()) { - auto it = indexedColumnsMap.insert({col, id}); - if (it.second) { - indexedColumns.push_back(col); - id++; - } - } - } - - // Data table - THashMap<TString, TVector<TValue>> mainTable; - - ReadMainTable(tableName, indexedColumns, indexedColumnsMap, mainTable); - for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { - CheckIndexData(tableName, index, indexedColumnsMap, mainTable); - } - } - - void DescribeTable(const TString& tableName) { - TableDescription_ = ::NIdxTest::DescribeTable(tableName, Client_); - } - - TTableClient Client_; - IProgressTracker::TPtr ProgressTracker_; - TMaybe<TTableDescription> TableDescription_; -}; - -IChecker::TPtr CreateChecker(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker) { - return std::make_unique<TChecker>(TTableClient(driver), std::move(progressTracker)); -} - -} // namespace NIdxTest + } + } + + + const TString indexTableName = tableName + "/" + indexDesc.GetIndexName() + "/indexImplTable"; + + if (ProgressTracker_) { + ProgressTracker_->Start("rows read", "Reading indexTable table " + indexTableName + "..."); + } + ThrowOnError(Client_.RetryOperationSync([indexTableName, settings, &tableIterator](TSession session) { + auto result = session.ReadTable(indexTableName, settings).GetValueSync(); + + if (result.IsSuccess()) { + tableIterator = result; + } + + return result; + })); + + size_t totalRows = 0; + for (;;) { + auto tablePart = tableIterator->ReadNext().GetValueSync(); + if (!tablePart.IsSuccess()) { + if (tablePart.EOS()) { + break; + } + + ThrowOnError(tablePart); + } + + auto rsParser = TResultSetParser(tablePart.ExtractPart()); + while (rsParser.TryNextRow()) { + if (ProgressTracker_) { + ProgressTracker_->Update(totalRows); + } + totalRows++; + TString key; + + for (const auto& id : pkColumnIdx) { + auto value = rsParser.GetValue(id); + key += NYdb::FormatValueYson(value); + } + + auto mainTableRow = mainTable.find(key); + if (mainTableRow == mainTable.end()) { + throw yexception() << "index table has unknown key: " << key; + } + + for (size_t id = 0; id < checkColumnsMap.size(); id++) { + auto value = rsParser.GetValue(id); + const auto mainTableColumnId = checkColumnsMap[id]; + + const auto& valueFromMain = mainTableRow->second[mainTableColumnId]; + + if (NYdb::FormatValueYson(value) != NYdb::FormatValueYson(valueFromMain)) { + throw yexception() << " value missmatch for row with key: " << key; + } + } + } + } + + if (totalRows != mainTable.size()) { + throw yexception() << "rows count missmatch, index table " + << indexTableName << " has " << totalRows + << " rows, but main table has " << mainTable.size() << " rows."; + } + if (ProgressTracker_) { + ProgressTracker_->Finish("Done."); + } + } + + void ReadMainTable( + const TString& tableName, + const TVector<TString>& columns, + const THashMap<TString, size_t>& columnMap, + THashMap<TString, TVector<TValue>>& buf) + { + TMaybe<TTablePartIterator> tableIterator; + + auto settings = TReadTableSettings(); + + for (const auto& col : columns) { + settings.AppendColumns(col); + } + + if (ProgressTracker_) { + ProgressTracker_->Start("rows read", "Reading main table " + tableName + "..."); + } + + // columnsMap.size() columns will be readed, but probably there are no pk here + // so append pk column in this case + size_t pkPos = columnMap.size(); + TVector<size_t> pkMap; + + for (const auto& col : TableDescription_.GetRef().GetPrimaryKeyColumns()) { + const auto it = columnMap.find(col); + if (it == columnMap.end()) { + settings.AppendColumns(col); + pkMap.push_back(pkPos); + pkPos++; + } else { + pkMap.push_back(it->second); + } + } + + ThrowOnError(Client_.RetryOperationSync([tableName, settings, &tableIterator](TSession session) { + auto result = session.ReadTable(tableName, settings).GetValueSync(); + + if (result.IsSuccess()) { + tableIterator = result; + } + + return result; + })); + + size_t rows = 0; + for (;;) { + auto tablePart = tableIterator->ReadNext().GetValueSync(); + if (!tablePart.IsSuccess()) { + if (tablePart.EOS()) { + break; + } + + ThrowOnError(tablePart); + } + + auto rsParser = TResultSetParser(tablePart.ExtractPart()); + + while (rsParser.TryNextRow()) { + TString key; + TVector<TValue> values; + + for (size_t i = 0; i < columns.size(); i++) { + values.push_back(rsParser.GetValue(i)); + } + + for (auto i : pkMap) { + auto value = rsParser.GetValue(i); + key += NYdb::FormatValueYson(value); + } + Y_VERIFY(buf.insert(std::make_pair(key, values)).second); + if (ProgressTracker_) { + ProgressTracker_->Update(rows++); + } + } + } + if (ProgressTracker_) { + ProgressTracker_->Finish("Done."); + } + } + + void CheckIndexes(const TString& tableName) { + + // Some magic for case of using same columns for multiple indexes + // or pk and indexes + + // Columns need to check index + TVector<TString> indexedColumns; + // Map column name -> position in result row in data table + THashMap<TString, size_t> indexedColumnsMap; + + size_t id = 0; + for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { + for (const auto& col : index.GetIndexColumns()) { + auto it = indexedColumnsMap.insert({col, id}); + if (it.second) { + indexedColumns.push_back(col); + id++; + } + } + + for (const auto& col : index.GetDataColumns()) { + auto it = indexedColumnsMap.insert({col, id}); + if (it.second) { + indexedColumns.push_back(col); + id++; + } + } + } + + // Data table + THashMap<TString, TVector<TValue>> mainTable; + + ReadMainTable(tableName, indexedColumns, indexedColumnsMap, mainTable); + for (const auto& index : TableDescription_.GetRef().GetIndexDescriptions()) { + CheckIndexData(tableName, index, indexedColumnsMap, mainTable); + } + } + + void DescribeTable(const TString& tableName) { + TableDescription_ = ::NIdxTest::DescribeTable(tableName, Client_); + } + + TTableClient Client_; + IProgressTracker::TPtr ProgressTracker_; + TMaybe<TTableDescription> TableDescription_; +}; + +IChecker::TPtr CreateChecker(NYdb::TDriver& driver, IProgressTracker::TPtr&& progressTracker) { + return std::make_unique<TChecker>(TTableClient(driver), std::move(progressTracker)); +} + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_common.cpp b/ydb/public/lib/idx_test/idx_test_common.cpp index a6c9bb9ea9..404fa8dfaf 100644 --- a/ydb/public/lib/idx_test/idx_test_common.cpp +++ b/ydb/public/lib/idx_test/idx_test_common.cpp @@ -1,29 +1,29 @@ -#include "idx_test_common.h" - +#include "idx_test_common.h" + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace NIdxTest { - -TMaybe<TTableDescription> DescribeTable(const TString& tableName, TTableClient client) { - TMaybe<TTableDescription> res; - auto describeFn = [&res, tableName](TSession session) mutable { - auto describeResult = session.DescribeTable( - tableName, - TDescribeTableSettings() - .WithKeyShardBoundary(true) - .ClientTimeout(TDuration::Seconds(5)) - ).GetValueSync(); - if (!describeResult.IsSuccess()) - return describeResult; - res = describeResult.GetTableDescription(); - return describeResult; - }; - auto status = client.RetryOperationSync(describeFn); - ThrowOnError(status); - return res; -} - -} // namespace NIdxTest + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace NIdxTest { + +TMaybe<TTableDescription> DescribeTable(const TString& tableName, TTableClient client) { + TMaybe<TTableDescription> res; + auto describeFn = [&res, tableName](TSession session) mutable { + auto describeResult = session.DescribeTable( + tableName, + TDescribeTableSettings() + .WithKeyShardBoundary(true) + .ClientTimeout(TDuration::Seconds(5)) + ).GetValueSync(); + if (!describeResult.IsSuccess()) + return describeResult; + res = describeResult.GetTableDescription(); + return describeResult; + }; + auto status = client.RetryOperationSync(describeFn); + ThrowOnError(status); + return res; +} + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_common.h b/ydb/public/lib/idx_test/idx_test_common.h index 845916e77d..7609c459ef 100644 --- a/ydb/public/lib/idx_test/idx_test_common.h +++ b/ydb/public/lib/idx_test/idx_test_common.h @@ -1,30 +1,30 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -namespace NYdb { -namespace NTable { - class TTableClient; - class TTableDescription; -} -} - -namespace NIdxTest { - -class TYdbErrorException : public yexception { -public: - TYdbErrorException(const NYdb::TStatus& status) - : Status(status) {} - - NYdb::TStatus Status; -}; - -inline void ThrowOnError(const NYdb::TStatus& status) { - if (!status.IsSuccess()) { - throw TYdbErrorException(status); - } -} - -TMaybe<NYdb::NTable::TTableDescription> DescribeTable(const TString& tableName, NYdb::NTable::TTableClient client); - -} + +namespace NYdb { +namespace NTable { + class TTableClient; + class TTableDescription; +} +} + +namespace NIdxTest { + +class TYdbErrorException : public yexception { +public: + TYdbErrorException(const NYdb::TStatus& status) + : Status(status) {} + + NYdb::TStatus Status; +}; + +inline void ThrowOnError(const NYdb::TStatus& status) { + if (!status.IsSuccess()) { + throw TYdbErrorException(status); + } +} + +TMaybe<NYdb::NTable::TTableDescription> DescribeTable(const TString& tableName, NYdb::NTable::TTableClient client); + +} diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.cpp b/ydb/public/lib/idx_test/idx_test_data_provider.cpp index 54ac805d0a..0006551f29 100644 --- a/ydb/public/lib/idx_test/idx_test_data_provider.cpp +++ b/ydb/public/lib/idx_test/idx_test_data_provider.cpp @@ -1,276 +1,276 @@ -#include "idx_test.h" -#include "idx_test_common.h" -#include "idx_test_data_provider.h" - +#include "idx_test.h" +#include "idx_test_common.h" +#include "idx_test_data_provider.h" + #include <library/cpp/string_utils/base64/base64.h> - + #include <util/string/builder.h> -using namespace NYdb; -using namespace NYdb::NTable; - -namespace NIdxTest { - -NYdb::TValue CreateOptionalValue(const TColumn& column, const TRandomValueProvider& rvp) { - NYdb::TValueBuilder value; - - const NYdb::TType& type = column.Type; - NYdb::TTypeParser typeParser(type); - typeParser.OpenOptional(); - switch (typeParser.GetPrimitive()) { - case EPrimitiveType::Bool: - value.OptionalBool(rvp.RandomBool()); - break; - case EPrimitiveType::Int8: - value.OptionalInt8(rvp.RandomUi8()); - break; - case EPrimitiveType::Uint8: - value.OptionalUint8(rvp.RandomI8()); - break; - case EPrimitiveType::Int16: - value.OptionalInt16(rvp.RandomI16()); - break; - case EPrimitiveType::Uint16: - value.OptionalUint16(rvp.RandomUi16()); - break; - case EPrimitiveType::Int32: - value.OptionalInt32(rvp.RandomI32()); - break; - case EPrimitiveType::Uint32: - value.OptionalUint32(rvp.RandomUi32()); - break; - case EPrimitiveType::Int64: - value.OptionalInt64(rvp.RandomI64()); - break; - case EPrimitiveType::Uint64: - value.OptionalUint64(rvp.RandomUi64()); - break; - case EPrimitiveType::Float: - value.OptionalFloat(rvp.RandomFloat()); - break; - case EPrimitiveType::Double: - value.OptionalDouble(rvp.RandomDouble()); - break; - case EPrimitiveType::String: - value.OptionalString(rvp.RandomString()); - break; - case EPrimitiveType::Utf8: - value.OptionalUtf8(Base64Encode(rvp.RandomString())); - break; - case EPrimitiveType::Json: - { - auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; - value.OptionalJson(TString(sb)); - } - break; - default: - Y_VERIFY(false, "unimplemented"); - } - return value.Build(); -} - -NYdb::TValue CreateValue(const TColumn& column, const TRandomValueProvider& rvp) { - NYdb::TValueBuilder value; - - const NYdb::TType& type = column.Type; - NYdb::TTypeParser typeParser(type); - typeParser.OpenOptional(); - switch (typeParser.GetPrimitive()) { - case EPrimitiveType::Bool: - value.Bool(rvp.RandomBool()); - break; - case EPrimitiveType::Int8: - value.Int8(rvp.RandomUi8()); - break; - case EPrimitiveType::Uint8: - value.Uint8(rvp.RandomI8()); - break; - case EPrimitiveType::Int16: - value.Int16(rvp.RandomI16()); - break; - case EPrimitiveType::Uint16: - value.Uint16(rvp.RandomUi16()); - break; - case EPrimitiveType::Int32: - value.Int32(rvp.RandomI32()); - break; - case EPrimitiveType::Uint32: - value.Uint32(rvp.RandomUi32()); - break; - case EPrimitiveType::Int64: - value.Int64(rvp.RandomI64()); - break; - case EPrimitiveType::Uint64: - value.Uint64(rvp.RandomUi64()); - break; - case EPrimitiveType::Float: - value.Float(rvp.RandomFloat()); - break; - case EPrimitiveType::Double: - value.Double(rvp.RandomDouble()); - break; - case EPrimitiveType::String: - value.String(rvp.RandomString()); - break; - case EPrimitiveType::Utf8: - value.Utf8(Base64Encode(rvp.RandomString())); - break; - case EPrimitiveType::Json: - { - auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; - value.Json(TString(sb)); - } - break; - default: - Y_VERIFY(false, "unimplemented"); - } - return value.Build(); -} - -NYdb::TValue CreateRow(const TVector<TColumn>& columns, const TRandomValueProvider& rvp) { - NYdb::TValueBuilder value; - value.BeginStruct(); - for (const NYdb::TColumn& col : columns) { - const NYdb::TType& type = col.Type; - NYdb::TTypeParser typeParser(type); - typeParser.OpenOptional(); - switch (typeParser.GetPrimitive()) { - case EPrimitiveType::Bool: - value.AddMember(col.Name).Bool(rvp.RandomBool()); - break; - case EPrimitiveType::Int8: - value.AddMember(col.Name).Int8(rvp.RandomUi8()); - break; - case EPrimitiveType::Uint8: - value.AddMember(col.Name).Uint8(rvp.RandomI8()); - break; - case EPrimitiveType::Int16: - value.AddMember(col.Name).Int16(rvp.RandomI16()); - break; - case EPrimitiveType::Uint16: - value.AddMember(col.Name).Uint16(rvp.RandomUi16()); - break; - case EPrimitiveType::Int32: - value.AddMember(col.Name).Int32(rvp.RandomI32()); - break; - case EPrimitiveType::Uint32: - value.AddMember(col.Name).Uint32(rvp.RandomUi32()); - break; - case EPrimitiveType::Int64: - value.AddMember(col.Name).Int64(rvp.RandomI64()); - break; - case EPrimitiveType::Uint64: - value.AddMember(col.Name).Uint64(rvp.RandomUi64()); - break; - case EPrimitiveType::Float: - value.AddMember(col.Name).Float(rvp.RandomFloat()); - break; - case EPrimitiveType::Double: - value.AddMember(col.Name).Double(rvp.RandomDouble()); - break; - case EPrimitiveType::String: - value.AddMember(col.Name).String(rvp.RandomString()); - break; - case EPrimitiveType::Utf8: - value.AddMember(col.Name).Utf8(Base64Encode(rvp.RandomString())); - break; - case EPrimitiveType::Json: - { - auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; - value.AddMember(col.Name).Json(TString(sb)); - } - break; - - default: - Y_VERIFY(false, "unimplemented"); - } - } - value.EndStruct(); - return value.Build(); -} - -NYdb::TParams CreateParamsAsItems(const TVector<TValue>& values, const TVector<TString>& paramNames) { - TParamsBuilder paramsBuilder; - if (values.size() != paramNames.size()) { - ythrow yexception() << "values and params size missmatch"; - } - - for (size_t i = 0; i < values.size(); i++) { - paramsBuilder.AddParam(paramNames[i], values[i]); - } - - return paramsBuilder.Build(); -} - -NYdb::TParams CreateParamsAsList(const TVector<NYdb::TValue>& batch, const TString& paramName) { - TValueBuilder builder; - builder.BeginList(); - - for (const NYdb::TValue& item: batch) { - builder.AddListItem(item); - } - builder.EndList(); - NYdb::TParamsBuilder paramsBuilder; - paramsBuilder.AddParam(paramName, builder.Build()); - return paramsBuilder.Build(); -} - -class TDataProvider - : public IDataProvider - , public TRandomValueProvider { -public: - TDataProvider(ui32 rowsCount, ui32 shardsCount, TTableDescription tableDesc) - : RowsCount_(rowsCount) - , TableDesc_(tableDesc) - , BatchSz_((rowsCount > 10) ? 10 : rowsCount) - { - Positions_.resize(shardsCount); - } - - NYdb::NTable::TTableDescription GetTableSchema() override { - return TableDesc_; - } - - NYdb::TType GetRowType() override { - return CreateRow().GetType(); - } - - TMaybe<TVector<NYdb::TValue>> GetBatch(ui32 streamId) override { - auto& pos = Positions_[streamId]; - if (pos >= RowsCount_) - return {}; - - pos += BatchSz_; - return CreateBatch(); - } -private: - TVector<NYdb::TValue> CreateBatch() { - TVector<NYdb::TValue> batch; - size_t batchSz = BatchSz_; - while (batchSz--) { - batch.push_back(CreateRow()); - } - return batch; - } - - NYdb::TValue CreateRow() { - return ::NIdxTest::CreateRow(TableDesc_.GetColumns(), *this); - } - - const ui32 RowsCount_; - const TTableDescription TableDesc_; - const ui32 BatchSz_; - - TVector<size_t> Positions_; -}; - -TRandomValueProvider::TRandomValueProvider(ui8 ranges, ui8 limit) - : RangesBits_(ranges) - , LimitMask_((limit == 0) ? Max<ui64>() : (1ull << limit) - 1) -{} - -IDataProvider::TPtr CreateDataProvider(ui32 rowsCount, ui32 shardsCount, TTableDescription tableDesc) { - return std::make_unique<TDataProvider>(rowsCount, shardsCount, tableDesc); -} - -} // namespace NIdxTest +using namespace NYdb; +using namespace NYdb::NTable; + +namespace NIdxTest { + +NYdb::TValue CreateOptionalValue(const TColumn& column, const TRandomValueProvider& rvp) { + NYdb::TValueBuilder value; + + const NYdb::TType& type = column.Type; + NYdb::TTypeParser typeParser(type); + typeParser.OpenOptional(); + switch (typeParser.GetPrimitive()) { + case EPrimitiveType::Bool: + value.OptionalBool(rvp.RandomBool()); + break; + case EPrimitiveType::Int8: + value.OptionalInt8(rvp.RandomUi8()); + break; + case EPrimitiveType::Uint8: + value.OptionalUint8(rvp.RandomI8()); + break; + case EPrimitiveType::Int16: + value.OptionalInt16(rvp.RandomI16()); + break; + case EPrimitiveType::Uint16: + value.OptionalUint16(rvp.RandomUi16()); + break; + case EPrimitiveType::Int32: + value.OptionalInt32(rvp.RandomI32()); + break; + case EPrimitiveType::Uint32: + value.OptionalUint32(rvp.RandomUi32()); + break; + case EPrimitiveType::Int64: + value.OptionalInt64(rvp.RandomI64()); + break; + case EPrimitiveType::Uint64: + value.OptionalUint64(rvp.RandomUi64()); + break; + case EPrimitiveType::Float: + value.OptionalFloat(rvp.RandomFloat()); + break; + case EPrimitiveType::Double: + value.OptionalDouble(rvp.RandomDouble()); + break; + case EPrimitiveType::String: + value.OptionalString(rvp.RandomString()); + break; + case EPrimitiveType::Utf8: + value.OptionalUtf8(Base64Encode(rvp.RandomString())); + break; + case EPrimitiveType::Json: + { + auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; + value.OptionalJson(TString(sb)); + } + break; + default: + Y_VERIFY(false, "unimplemented"); + } + return value.Build(); +} + +NYdb::TValue CreateValue(const TColumn& column, const TRandomValueProvider& rvp) { + NYdb::TValueBuilder value; + + const NYdb::TType& type = column.Type; + NYdb::TTypeParser typeParser(type); + typeParser.OpenOptional(); + switch (typeParser.GetPrimitive()) { + case EPrimitiveType::Bool: + value.Bool(rvp.RandomBool()); + break; + case EPrimitiveType::Int8: + value.Int8(rvp.RandomUi8()); + break; + case EPrimitiveType::Uint8: + value.Uint8(rvp.RandomI8()); + break; + case EPrimitiveType::Int16: + value.Int16(rvp.RandomI16()); + break; + case EPrimitiveType::Uint16: + value.Uint16(rvp.RandomUi16()); + break; + case EPrimitiveType::Int32: + value.Int32(rvp.RandomI32()); + break; + case EPrimitiveType::Uint32: + value.Uint32(rvp.RandomUi32()); + break; + case EPrimitiveType::Int64: + value.Int64(rvp.RandomI64()); + break; + case EPrimitiveType::Uint64: + value.Uint64(rvp.RandomUi64()); + break; + case EPrimitiveType::Float: + value.Float(rvp.RandomFloat()); + break; + case EPrimitiveType::Double: + value.Double(rvp.RandomDouble()); + break; + case EPrimitiveType::String: + value.String(rvp.RandomString()); + break; + case EPrimitiveType::Utf8: + value.Utf8(Base64Encode(rvp.RandomString())); + break; + case EPrimitiveType::Json: + { + auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; + value.Json(TString(sb)); + } + break; + default: + Y_VERIFY(false, "unimplemented"); + } + return value.Build(); +} + +NYdb::TValue CreateRow(const TVector<TColumn>& columns, const TRandomValueProvider& rvp) { + NYdb::TValueBuilder value; + value.BeginStruct(); + for (const NYdb::TColumn& col : columns) { + const NYdb::TType& type = col.Type; + NYdb::TTypeParser typeParser(type); + typeParser.OpenOptional(); + switch (typeParser.GetPrimitive()) { + case EPrimitiveType::Bool: + value.AddMember(col.Name).Bool(rvp.RandomBool()); + break; + case EPrimitiveType::Int8: + value.AddMember(col.Name).Int8(rvp.RandomUi8()); + break; + case EPrimitiveType::Uint8: + value.AddMember(col.Name).Uint8(rvp.RandomI8()); + break; + case EPrimitiveType::Int16: + value.AddMember(col.Name).Int16(rvp.RandomI16()); + break; + case EPrimitiveType::Uint16: + value.AddMember(col.Name).Uint16(rvp.RandomUi16()); + break; + case EPrimitiveType::Int32: + value.AddMember(col.Name).Int32(rvp.RandomI32()); + break; + case EPrimitiveType::Uint32: + value.AddMember(col.Name).Uint32(rvp.RandomUi32()); + break; + case EPrimitiveType::Int64: + value.AddMember(col.Name).Int64(rvp.RandomI64()); + break; + case EPrimitiveType::Uint64: + value.AddMember(col.Name).Uint64(rvp.RandomUi64()); + break; + case EPrimitiveType::Float: + value.AddMember(col.Name).Float(rvp.RandomFloat()); + break; + case EPrimitiveType::Double: + value.AddMember(col.Name).Double(rvp.RandomDouble()); + break; + case EPrimitiveType::String: + value.AddMember(col.Name).String(rvp.RandomString()); + break; + case EPrimitiveType::Utf8: + value.AddMember(col.Name).Utf8(Base64Encode(rvp.RandomString())); + break; + case EPrimitiveType::Json: + { + auto sb = TStringBuilder() << "[\"" << Base64Encode(rvp.RandomString()) << "\"]"; + value.AddMember(col.Name).Json(TString(sb)); + } + break; + + default: + Y_VERIFY(false, "unimplemented"); + } + } + value.EndStruct(); + return value.Build(); +} + +NYdb::TParams CreateParamsAsItems(const TVector<TValue>& values, const TVector<TString>& paramNames) { + TParamsBuilder paramsBuilder; + if (values.size() != paramNames.size()) { + ythrow yexception() << "values and params size missmatch"; + } + + for (size_t i = 0; i < values.size(); i++) { + paramsBuilder.AddParam(paramNames[i], values[i]); + } + + return paramsBuilder.Build(); +} + +NYdb::TParams CreateParamsAsList(const TVector<NYdb::TValue>& batch, const TString& paramName) { + TValueBuilder builder; + builder.BeginList(); + + for (const NYdb::TValue& item: batch) { + builder.AddListItem(item); + } + builder.EndList(); + NYdb::TParamsBuilder paramsBuilder; + paramsBuilder.AddParam(paramName, builder.Build()); + return paramsBuilder.Build(); +} + +class TDataProvider + : public IDataProvider + , public TRandomValueProvider { +public: + TDataProvider(ui32 rowsCount, ui32 shardsCount, TTableDescription tableDesc) + : RowsCount_(rowsCount) + , TableDesc_(tableDesc) + , BatchSz_((rowsCount > 10) ? 10 : rowsCount) + { + Positions_.resize(shardsCount); + } + + NYdb::NTable::TTableDescription GetTableSchema() override { + return TableDesc_; + } + + NYdb::TType GetRowType() override { + return CreateRow().GetType(); + } + + TMaybe<TVector<NYdb::TValue>> GetBatch(ui32 streamId) override { + auto& pos = Positions_[streamId]; + if (pos >= RowsCount_) + return {}; + + pos += BatchSz_; + return CreateBatch(); + } +private: + TVector<NYdb::TValue> CreateBatch() { + TVector<NYdb::TValue> batch; + size_t batchSz = BatchSz_; + while (batchSz--) { + batch.push_back(CreateRow()); + } + return batch; + } + + NYdb::TValue CreateRow() { + return ::NIdxTest::CreateRow(TableDesc_.GetColumns(), *this); + } + + const ui32 RowsCount_; + const TTableDescription TableDesc_; + const ui32 BatchSz_; + + TVector<size_t> Positions_; +}; + +TRandomValueProvider::TRandomValueProvider(ui8 ranges, ui8 limit) + : RangesBits_(ranges) + , LimitMask_((limit == 0) ? Max<ui64>() : (1ull << limit) - 1) +{} + +IDataProvider::TPtr CreateDataProvider(ui32 rowsCount, ui32 shardsCount, TTableDescription tableDesc) { + return std::make_unique<TDataProvider>(rowsCount, shardsCount, tableDesc); +} + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.h b/ydb/public/lib/idx_test/idx_test_data_provider.h index d0a1857d45..e6a9bd4aab 100644 --- a/ydb/public/lib/idx_test/idx_test_data_provider.h +++ b/ydb/public/lib/idx_test/idx_test_data_provider.h @@ -1,101 +1,101 @@ -#pragma once +#pragma once #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> -#include <util/system/types.h> -#include <util/generic/string.h> - -#include <util/random/random.h> - -namespace NIdxTest { - -class TRandomValueProvider { -public: - TRandomValueProvider(ui8 ranges = 0, ui8 limit = 0); - bool RandomBool() const { - return RandomNumber<ui8>(2); - } - - i8 RandomI8() const { - return RandomUi8() - 0x80; - } - - i16 RandomI16() const { - return RandomUi16() - 0x8000; - } - - i32 RandomI32() const { - return RandomUi32() - 0x80000000; - } - - i64 RandomI64() const { - return RandomUi64() - 0x8000000000000000; - } - - ui8 RandomUi8() const { - auto rnd = RandomNumber<ui8>(); - if (!RangesBits_) - return rnd & LimitMask_; - auto shard = RandomNumber<ui8>(Min(1 << RangesBits_, 128)); - return rnd & LimitMask_ | (shard << 8 - RangesBits_); - } - - ui16 RandomUi16() const { - auto rnd = RandomNumber<ui16>(); - if (!RangesBits_) - return rnd & LimitMask_; - auto shard = RandomNumber<ui16>(1 << RangesBits_); - return rnd & LimitMask_ | (shard << 16 - RangesBits_); - } - - ui32 RandomUi32() const { - auto rnd = RandomNumber<ui32>(); - if (!RangesBits_) - return rnd & LimitMask_; - auto shard = RandomNumber<ui32>(1 << RangesBits_); - return rnd & LimitMask_ | (shard << 32 - RangesBits_); - } - - ui64 RandomUi64() const { - auto rnd = RandomNumber<ui64>(); - if (!RangesBits_) - return rnd & LimitMask_; - auto shard = RandomNumber<ui64>(1 << RangesBits_); - return rnd & LimitMask_ | (shard << 64 - RangesBits_); - } - - float RandomFloat() const { - return RandomDouble(); - } - - double RandomDouble() const { - return RandomI32() / (double)(1ull << 32); - } - - // Random binary data - TString RandomString() const { - ui64 random = RandomUi64(); - size_t sz = (random & 0xf) * 8; - if (!sz) - return {}; - auto buf = std::unique_ptr<char[]>(new char[sz]); - - for (size_t i = 0; i < sz; i += 8) { - memcpy(buf.get() + i, (char*)(&random), 8); - } - buf.get()[sz - 1] = 0; - - return TString(buf.get(), sz - 1); - } -private: - const ui16 RangesBits_; - const ui64 LimitMask_; - -}; - -NYdb::TValue CreateOptionalValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); -NYdb::TValue CreateValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); -NYdb::TValue CreateRow(const TVector<NYdb::TColumn>& columns, const TRandomValueProvider& rvp); -NYdb::TParams CreateParamsAsItems(const TVector<NYdb::TValue>& values, const TVector<TString>& paramNames); -NYdb::TParams CreateParamsAsList(const TVector<NYdb::TValue>& batch, const TString& paramName); - -} +#include <util/system/types.h> +#include <util/generic/string.h> + +#include <util/random/random.h> + +namespace NIdxTest { + +class TRandomValueProvider { +public: + TRandomValueProvider(ui8 ranges = 0, ui8 limit = 0); + bool RandomBool() const { + return RandomNumber<ui8>(2); + } + + i8 RandomI8() const { + return RandomUi8() - 0x80; + } + + i16 RandomI16() const { + return RandomUi16() - 0x8000; + } + + i32 RandomI32() const { + return RandomUi32() - 0x80000000; + } + + i64 RandomI64() const { + return RandomUi64() - 0x8000000000000000; + } + + ui8 RandomUi8() const { + auto rnd = RandomNumber<ui8>(); + if (!RangesBits_) + return rnd & LimitMask_; + auto shard = RandomNumber<ui8>(Min(1 << RangesBits_, 128)); + return rnd & LimitMask_ | (shard << 8 - RangesBits_); + } + + ui16 RandomUi16() const { + auto rnd = RandomNumber<ui16>(); + if (!RangesBits_) + return rnd & LimitMask_; + auto shard = RandomNumber<ui16>(1 << RangesBits_); + return rnd & LimitMask_ | (shard << 16 - RangesBits_); + } + + ui32 RandomUi32() const { + auto rnd = RandomNumber<ui32>(); + if (!RangesBits_) + return rnd & LimitMask_; + auto shard = RandomNumber<ui32>(1 << RangesBits_); + return rnd & LimitMask_ | (shard << 32 - RangesBits_); + } + + ui64 RandomUi64() const { + auto rnd = RandomNumber<ui64>(); + if (!RangesBits_) + return rnd & LimitMask_; + auto shard = RandomNumber<ui64>(1 << RangesBits_); + return rnd & LimitMask_ | (shard << 64 - RangesBits_); + } + + float RandomFloat() const { + return RandomDouble(); + } + + double RandomDouble() const { + return RandomI32() / (double)(1ull << 32); + } + + // Random binary data + TString RandomString() const { + ui64 random = RandomUi64(); + size_t sz = (random & 0xf) * 8; + if (!sz) + return {}; + auto buf = std::unique_ptr<char[]>(new char[sz]); + + for (size_t i = 0; i < sz; i += 8) { + memcpy(buf.get() + i, (char*)(&random), 8); + } + buf.get()[sz - 1] = 0; + + return TString(buf.get(), sz - 1); + } +private: + const ui16 RangesBits_; + const ui64 LimitMask_; + +}; + +NYdb::TValue CreateOptionalValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); +NYdb::TValue CreateValue(const NYdb::TColumn& column, const TRandomValueProvider& rvp); +NYdb::TValue CreateRow(const TVector<NYdb::TColumn>& columns, const TRandomValueProvider& rvp); +NYdb::TParams CreateParamsAsItems(const TVector<NYdb::TValue>& values, const TVector<TString>& paramNames); +NYdb::TParams CreateParamsAsList(const TVector<NYdb::TValue>& batch, const TString& paramName); + +} diff --git a/ydb/public/lib/idx_test/idx_test_loader.cpp b/ydb/public/lib/idx_test/idx_test_loader.cpp index a6e10ce7c4..392604288e 100644 --- a/ydb/public/lib/idx_test/idx_test_loader.cpp +++ b/ydb/public/lib/idx_test/idx_test_loader.cpp @@ -1,1271 +1,1271 @@ -#include "idx_test.h" -#include "idx_test_common.h" -#include "idx_test_data_provider.h" - +#include "idx_test.h" +#include "idx_test_common.h" +#include "idx_test_data_provider.h" + #include <ydb/public/lib/yson_value/ydb_yson_value.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> - -#include <util/generic/map.h> + +#include <util/generic/map.h> #include <util/string/printf.h> #include <util/generic/hash_set.h> #include <util/generic/list.h> #include <util/system/mutex.h> - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace { - -NThreading::TFuture<TStatus> FinishTxAsync(const TDataQueryResult& in) { - auto cb = [](TAsyncCommitTransactionResult future) { - auto result = future.ExtractValue(); - return NThreading::MakeFuture<TStatus>(result); - }; - if (in.GetTransaction()->IsActive()) { - return in.GetTransaction()->Commit().Apply(cb); - } else { - return NThreading::MakeFuture<TStatus>(in); - } -} - -TString SetPragmas(const TString& in, bool useNewEngine) { - if (useNewEngine) { - return "--!syntax_v1\nPRAGMA Kikimr.UseNewEngine = \"true\";\n" + in; - } - return "--!syntax_v1\n" + in; -} - -const static TRetryOperationSettings retryOperationSettings = TRetryOperationSettings() - .MaxRetries(10) - .FastBackoffSettings(NYdb::NTable::TBackoffSettings().SlotDuration(TDuration::MilliSeconds(5)).Ceiling(6)) - .SlowBackoffSettings(NYdb::NTable::TBackoffSettings().SlotDuration(TDuration::MilliSeconds(1000)).Ceiling(6)); -} - -namespace NIdxTest { - -class IWorkTask { -public: - using TPtr = std::unique_ptr<IWorkTask>; -public: - virtual TAsyncStatus Run() = 0; - virtual IWorkLoader::ELoadCommand GetTaskId() const = 0; - virtual ~IWorkTask() = default; -}; - -static TAsyncStatus RunOperation(TTableClient client, const TString& programText, NYdb::TParams params) -{ - return client.RetryOperation([programText, params](TSession session) { - return session.PrepareDataQuery(programText).Apply( - [params, programText](TAsyncPrepareQueryResult result) -> NYdb::TAsyncStatus { - auto prepareResult = result.ExtractValue(); - if (!prepareResult.IsSuccess()) { - Cerr << "Prepare failed" << Endl; - return NThreading::MakeFuture<TStatus>(prepareResult); - } - - auto dataQuery = prepareResult.GetQuery(); - static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); - return dataQuery.Execute(txControl, params).Apply( - [](TAsyncDataQueryResult asyncResult) { - auto result = asyncResult.ExtractValue(); - if (!result.IsSuccess()) { - Cerr << "Execute err: " << result.GetStatus() << " " << result.GetIssues().ToString() << Endl; - } - return NThreading::MakeFuture<TStatus>(result); - }); - }); - }, retryOperationSettings); -} - -static const ui8 KEYRANGELIMIT = 20; - -static ui8 TableDescriptionToShardsPower(const TTableDescription& tableDescription) { - ui32 ranges = tableDescription.GetKeyRanges().size(); - ui8 result = 0; - while (ranges >>= 1) { - result++; - } - - return result; -} - -static bool IsIndexedType(const TType& type) { - NYdb::TTypeParser parser(type); - if (parser.GetKind() == TTypeParser::ETypeKind::Optional) { - parser.OpenOptional(); - } - - if (parser.GetKind() == TTypeParser::ETypeKind::Primitive) { - switch (parser.GetPrimitive()) { - case EPrimitiveType::Bool: - case EPrimitiveType::Int8: - case EPrimitiveType::Uint8: - case EPrimitiveType::Int32: - case EPrimitiveType::Uint32: - case EPrimitiveType::Int64: - case EPrimitiveType::Uint64: - case EPrimitiveType::String: - case EPrimitiveType::Utf8: - return true; - break; - default: - return false; - break; - } - } - - return false; -} - -template<typename T> -static bool HasColumn(const T& container, const TString& col) { - return Find(container.begin(), container.end(), col) != container.end(); -} - -class TAlterTableAddIndexTask - : public IWorkTask - , public TRandomValueProvider { -public: - TAlterTableAddIndexTask( - TTableDescription tableDescription, - const TString& tableName, - TTableClient& client, - bool withDataColumn) - : TableDescription_(tableDescription) - , TableName_(tableName) - , Client_(client) - , WithDataColumn_(withDataColumn) - { + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace { + +NThreading::TFuture<TStatus> FinishTxAsync(const TDataQueryResult& in) { + auto cb = [](TAsyncCommitTransactionResult future) { + auto result = future.ExtractValue(); + return NThreading::MakeFuture<TStatus>(result); + }; + if (in.GetTransaction()->IsActive()) { + return in.GetTransaction()->Commit().Apply(cb); + } else { + return NThreading::MakeFuture<TStatus>(in); + } +} + +TString SetPragmas(const TString& in, bool useNewEngine) { + if (useNewEngine) { + return "--!syntax_v1\nPRAGMA Kikimr.UseNewEngine = \"true\";\n" + in; + } + return "--!syntax_v1\n" + in; +} + +const static TRetryOperationSettings retryOperationSettings = TRetryOperationSettings() + .MaxRetries(10) + .FastBackoffSettings(NYdb::NTable::TBackoffSettings().SlotDuration(TDuration::MilliSeconds(5)).Ceiling(6)) + .SlowBackoffSettings(NYdb::NTable::TBackoffSettings().SlotDuration(TDuration::MilliSeconds(1000)).Ceiling(6)); +} + +namespace NIdxTest { + +class IWorkTask { +public: + using TPtr = std::unique_ptr<IWorkTask>; +public: + virtual TAsyncStatus Run() = 0; + virtual IWorkLoader::ELoadCommand GetTaskId() const = 0; + virtual ~IWorkTask() = default; +}; + +static TAsyncStatus RunOperation(TTableClient client, const TString& programText, NYdb::TParams params) +{ + return client.RetryOperation([programText, params](TSession session) { + return session.PrepareDataQuery(programText).Apply( + [params, programText](TAsyncPrepareQueryResult result) -> NYdb::TAsyncStatus { + auto prepareResult = result.ExtractValue(); + if (!prepareResult.IsSuccess()) { + Cerr << "Prepare failed" << Endl; + return NThreading::MakeFuture<TStatus>(prepareResult); + } + + auto dataQuery = prepareResult.GetQuery(); + static const TTxControl txControl = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); + return dataQuery.Execute(txControl, params).Apply( + [](TAsyncDataQueryResult asyncResult) { + auto result = asyncResult.ExtractValue(); + if (!result.IsSuccess()) { + Cerr << "Execute err: " << result.GetStatus() << " " << result.GetIssues().ToString() << Endl; + } + return NThreading::MakeFuture<TStatus>(result); + }); + }); + }, retryOperationSettings); +} + +static const ui8 KEYRANGELIMIT = 20; + +static ui8 TableDescriptionToShardsPower(const TTableDescription& tableDescription) { + ui32 ranges = tableDescription.GetKeyRanges().size(); + ui8 result = 0; + while (ranges >>= 1) { + result++; + } + + return result; +} + +static bool IsIndexedType(const TType& type) { + NYdb::TTypeParser parser(type); + if (parser.GetKind() == TTypeParser::ETypeKind::Optional) { + parser.OpenOptional(); + } + + if (parser.GetKind() == TTypeParser::ETypeKind::Primitive) { + switch (parser.GetPrimitive()) { + case EPrimitiveType::Bool: + case EPrimitiveType::Int8: + case EPrimitiveType::Uint8: + case EPrimitiveType::Int32: + case EPrimitiveType::Uint32: + case EPrimitiveType::Int64: + case EPrimitiveType::Uint64: + case EPrimitiveType::String: + case EPrimitiveType::Utf8: + return true; + break; + default: + return false; + break; + } + } + + return false; +} + +template<typename T> +static bool HasColumn(const T& container, const TString& col) { + return Find(container.begin(), container.end(), col) != container.end(); +} + +class TAlterTableAddIndexTask + : public IWorkTask + , public TRandomValueProvider { +public: + TAlterTableAddIndexTask( + TTableDescription tableDescription, + const TString& tableName, + TTableClient& client, + bool withDataColumn) + : TableDescription_(tableDescription) + , TableName_(tableName) + , Client_(client) + , WithDataColumn_(withDataColumn) + { ChooseColumnsForIndex(); - } - - TAsyncStatus Run() override { - if (!Description_) { - // Nothing to add, skip - return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - } - return Client_.RetryOperation([this](TSession session) { - auto settings = TAlterTableSettings() - .AppendAddIndexes(Description_.GetRef()) - .ClientTimeout(TDuration::Seconds(5)); - return session.AlterTable(TableName_, settings) - .Apply([](TAsyncStatus future) { - const auto& status = future.GetValue(); - // status BAD_REQUEST - index already exists (realy?), treat it as success - // TODO: improve this check - if (status.GetStatus() == EStatus::BAD_REQUEST) { - return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - } - return future; - }); - }); - return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - } - - IWorkLoader::ELoadCommand GetTaskId() const override { - return WithDataColumn_ ? IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN : IWorkLoader::LC_ALTER_ADD_INDEX; - } -private: + } + + TAsyncStatus Run() override { + if (!Description_) { + // Nothing to add, skip + return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + } + return Client_.RetryOperation([this](TSession session) { + auto settings = TAlterTableSettings() + .AppendAddIndexes(Description_.GetRef()) + .ClientTimeout(TDuration::Seconds(5)); + return session.AlterTable(TableName_, settings) + .Apply([](TAsyncStatus future) { + const auto& status = future.GetValue(); + // status BAD_REQUEST - index already exists (realy?), treat it as success + // TODO: improve this check + if (status.GetStatus() == EStatus::BAD_REQUEST) { + return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + } + return future; + }); + }); + return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + } + + IWorkLoader::ELoadCommand GetTaskId() const override { + return WithDataColumn_ ? IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN : IWorkLoader::LC_ALTER_ADD_INDEX; + } +private: void ChooseColumnsForIndex() { - TVector<TString> columns; - - bool hasIndexedType = false; - - do { - for (const TTableColumn& col : TableDescription_.GetTableColumns()) { - if (IsIndexedType(col.Type)) { - hasIndexedType = true; - if (RandomBool()) { - columns.emplace_back(col.Name); - } - } - } - // repeat if we have something to build index but for all columns random returns false - } while (hasIndexedType && columns.empty()); - - if (columns.empty()) { - return; - } - - TVector<TString> dataColumn; - if (WithDataColumn_) { - for (const TTableColumn& col : TableDescription_.GetTableColumns()) { - if (HasColumn(TableDescription_.GetPrimaryKeyColumns(), col.Name)) { - continue; - } - if (HasColumn(columns, col.Name)) { - continue; - } - dataColumn.emplace_back(col.Name); - } - } - - TString indexName; - ui64 i = 0; - for (const auto& name : columns) { - indexName += name; - if (++i == columns.size()) { - indexName += "_index"; - } indexName += "_"; - } - - Description_ = NYdb::NTable::TIndexDescription(indexName, columns, dataColumn); - } - TMaybe<NYdb::NTable::TIndexDescription> Description_; - TTableDescription TableDescription_; - TString TableName_; - TTableClient Client_; - const bool WithDataColumn_; -}; - -class TSelectAndUpsertIfUniqTask - : public IWorkTask - , public TRandomValueProvider { -public: - TSelectAndUpsertIfUniqTask( - TTableDescription tableDescription, - const TString& tableName, - TTableClient& client, - bool useNewEngine) - : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) - , TableDescription_(tableDescription) - , TableName_(tableName) - , Client_(client) - , UseNewEngine_(useNewEngine) - { - CreateProgram(); - } - - TAsyncStatus Run() override { - return Client_.RetryOperation([this](TSession session) mutable { - return GetCheckIndexUniq()(1, session, TMaybe<TTransaction>(), THashMap<TString, NYdb::TValue>()) - .Apply([this](NThreading::TFuture<IndexValues> future) { - auto result = future.ExtractValue(); - const auto& status = result.Status; - if (status.GetStatus() == EStatus::PRECONDITION_FAILED) { - // Client emulated status + TVector<TString> columns; + + bool hasIndexedType = false; + + do { + for (const TTableColumn& col : TableDescription_.GetTableColumns()) { + if (IsIndexedType(col.Type)) { + hasIndexedType = true; + if (RandomBool()) { + columns.emplace_back(col.Name); + } + } + } + // repeat if we have something to build index but for all columns random returns false + } while (hasIndexedType && columns.empty()); + + if (columns.empty()) { + return; + } + + TVector<TString> dataColumn; + if (WithDataColumn_) { + for (const TTableColumn& col : TableDescription_.GetTableColumns()) { + if (HasColumn(TableDescription_.GetPrimaryKeyColumns(), col.Name)) { + continue; + } + if (HasColumn(columns, col.Name)) { + continue; + } + dataColumn.emplace_back(col.Name); + } + } + + TString indexName; + ui64 i = 0; + for (const auto& name : columns) { + indexName += name; + if (++i == columns.size()) { + indexName += "_index"; + } indexName += "_"; + } + + Description_ = NYdb::NTable::TIndexDescription(indexName, columns, dataColumn); + } + TMaybe<NYdb::NTable::TIndexDescription> Description_; + TTableDescription TableDescription_; + TString TableName_; + TTableClient Client_; + const bool WithDataColumn_; +}; + +class TSelectAndUpsertIfUniqTask + : public IWorkTask + , public TRandomValueProvider { +public: + TSelectAndUpsertIfUniqTask( + TTableDescription tableDescription, + const TString& tableName, + TTableClient& client, + bool useNewEngine) + : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) + , TableDescription_(tableDescription) + , TableName_(tableName) + , Client_(client) + , UseNewEngine_(useNewEngine) + { + CreateProgram(); + } + + TAsyncStatus Run() override { + return Client_.RetryOperation([this](TSession session) mutable { + return GetCheckIndexUniq()(1, session, TMaybe<TTransaction>(), THashMap<TString, NYdb::TValue>()) + .Apply([this](NThreading::TFuture<IndexValues> future) { + auto result = future.ExtractValue(); + const auto& status = result.Status; + if (status.GetStatus() == EStatus::PRECONDITION_FAILED) { + // Client emulated status return result.Tx->Commit().Apply([](const auto& future) { return NThreading::MakeFuture<TStatus>(future.GetValue()); }); - } else if (status.GetStatus() == EStatus::SUCCESS) { - // start upsert - const auto& program = Programs_[0]; - const auto& programText = program.first; - - NYdb::TValueBuilder value; - value.BeginStruct(); - for (const auto& x : program.second) { - auto it = result.Values.find(x.first.Name); - if (it != result.Values.end()) { - value.AddMember(x.first.Name, it->second); - } else { - value.AddMember(x.first.Name, ::NIdxTest::CreateValue(x.first, *this)); - } - } - value.EndStruct(); - TVector<NYdb::TValue> batch; - batch.push_back(value.Build()); - - Y_VERIFY(result.Tx); - const auto params = ::NIdxTest::CreateParamsAsList(batch, ParamName_); - - return result.Tx->GetSession().ExecuteDataQuery( - programText, TTxControl::Tx(result.Tx.GetRef()), std::move(params), - TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) - .Apply([](TAsyncDataQueryResult future){ - auto result = future.ExtractValue(); - if (result.IsSuccess()) { - return FinishTxAsync(result); - } else { - return NThreading::MakeFuture<TStatus>(result); - } - - }); - } else { - return NThreading::MakeFuture<TStatus>(status); - } - }); - }, retryOperationSettings); - - } - - IWorkLoader::ELoadCommand GetTaskId() const override { - return IWorkLoader::LC_UPSERT_IF_UNIQ; - } -private: - struct IndexValues { - TStatus Status; - THashMap<TString, NYdb::TValue> Values; - TMaybe<TTransaction> Tx; - }; - using TCheckIndexCb = std::function<NThreading::TFuture<IndexValues>( - size_t i, - TSession session, - TMaybe<TTransaction>, - THashMap<TString, NYdb::TValue>)>; - - TCheckIndexCb GetCheckIndexUniq() { - return [this] ( - size_t i, - TSession session, - TMaybe<TTransaction> tx, - THashMap<TString, NYdb::TValue>&& checked) mutable - { - if (i == Programs_.size()) { - return NThreading::MakeFuture<IndexValues>( - { - TStatus(EStatus::SUCCESS, NYql::TIssues()), - checked, - tx - } - ); - } - - const auto& p = Programs_[i]; - - TVector<NYdb::TValue> values; - TVector<TString> paramNames; - for (const auto col : p.second) { - const auto val = ::NIdxTest::CreateValue(col.first, *this); - - checked.insert({col.first.Name, val}); - values.push_back(val); - paramNames.push_back(col.second); - } - - const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); - - TTxControl txctrl = tx ? TTxControl::Tx(tx.GetRef()) : TTxControl::BeginTx(TTxSettings::SerializableRW()); - - return session.ExecuteDataQuery( - p.first, - txctrl, - params, - TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) - .Apply([this, i, session, checked{std::move(checked)}](TAsyncDataQueryResult future) mutable { - auto result = future.ExtractValue(); - // non success call - return status - if (!result.IsSuccess()) { - return NThreading::MakeFuture<IndexValues>( - { - result, - THashMap<TString, NYdb::TValue>(), - TMaybe<TTransaction>() - } - ); - } - - // success call and no record found (index uniq) - check next one - if (result.GetResultSet(0).RowsCount() == 0) { - return GetCheckIndexUniq()(i + 1, session, result.GetTransaction(), std::move(checked)); - } else { - return NThreading::MakeFuture<IndexValues>( - { - TStatus(EStatus::PRECONDITION_FAILED, NYql::TIssues()), - THashMap<TString, NYdb::TValue>(), - result.GetTransaction() - } - ); - } - }); - }; - } - - - void CreateProgram() { - const auto columns = TableDescription_.GetColumns(); - const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); - - // Used to find column description by column Name - THashMap<TString, std::pair<TColumn, TString>> colHash; - - TVector<std::pair<TColumn, TString>> upsertInput; - - for (const auto& col : columns) { - auto pkType = NIdxTest::CreateValue(col, *this); - auto typeString = NYdb::FormatType(pkType.GetType()); - colHash.insert({col.Name, {col, typeString}}); - upsertInput.push_back({col, ""}); - } - - { - // Create UPSERT program - auto rowType = NIdxTest::CreateRow(columns, *this); - auto rowsTypeString = NYdb::FormatType(rowType.GetType()); - - TString sqlTemplate = "DECLARE %s AS List<%s>; UPSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; - auto program = Sprintf(sqlTemplate.c_str(), - ParamName_.c_str(), - rowsTypeString.c_str(), - TableName_.c_str(), - ParamName_.c_str()); - - Programs_.push_back({SetPragmas(program, UseNewEngine_), upsertInput}); - } - - TString pkColumns; - { - size_t id = 0; - for (const auto& col : pkColNames) { - pkColumns += col; - if (++id != pkColNames.size()) - pkColumns += ", "; - } - } - - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - // Create select pk program for each index - size_t id = 0; - TString indexPredicate; - TString declare; - TVector<std::pair<TColumn, TString>> predicates; - for (const auto& col : indexDesc.GetIndexColumns()) { - const TString paramName = Sprintf("$items_%zu", id); - declare += Sprintf("DECLARE %s AS %s;\n", - paramName.c_str(), colHash.find(col)->second.second.c_str()); - predicates.push_back({colHash.find(col)->second.first, paramName}); - indexPredicate += Sprintf(" %s = %s ", col.c_str(), paramName.c_str()); - if (++id != pkColNames.size()) { - indexPredicate += " AND "; - } - } - - TString sql; - sql += declare; - sql += Sprintf("SELECT %s FROM `%s` VIEW %s WHERE %s", pkColumns.c_str(), - TableName_.c_str(), indexDesc.GetIndexName().c_str(), indexPredicate.c_str()); - - Programs_.push_back({SetPragmas(sql, UseNewEngine_), predicates}); - } - } - - const TString ParamName_ = "$items"; - TTableDescription TableDescription_; - TString TableName_; - TTableClient Client_; - bool UseNewEngine_; - - TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; - - mutable TString Err_; -}; - - -class TSelectAndCompareTask - : public IWorkTask - , public TRandomValueProvider { -public: - TSelectAndCompareTask( - TTableDescription tableDescription, - const TString& tableName, - TTableClient& client, - bool useNewEngine) - : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) - , TableDescription_(tableDescription) - , TableName_(tableName) - , Client_(client) - , UseNewEngine_(useNewEngine) - { - CreateProgram(); - } - - TAsyncStatus Run() override { - if (Programs_.empty()) - throw yexception() << "no program to run"; - - TString err; - with_lock(Mtx_) { - if (Err_) - err = Err_; - } - if (err) - throw yexception() << err; - - const auto& program = Programs_[0]; - - TVector<NYdb::TValue> values; - TVector<TString> paramNames; - for (const auto col : program.second) { - values.push_back(::NIdxTest::CreateOptionalValue(col.first, *this)); - paramNames.push_back(col.second); - } - - const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); - - return Client_.RetryOperation([this, params](TSession session) mutable { - const auto& mainTableQuery = Programs_[0].first; - return session.ExecuteDataQuery( - mainTableQuery, - TTxControl::BeginTx(TTxSettings::SerializableRW()), - params, - TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) - .Apply([this, session](TAsyncDataQueryResult future) mutable { - auto result = future.ExtractValue(); - if (result.GetResultSets().empty()) { - if (result.IsSuccess()) { - return FinishTxAsync(result); - } else { - return NThreading::MakeFuture<TStatus>(result); - } - } - - const auto& mainResultSet = NYdb::FormatResultSetYson(result.GetResultSet(0));; - return GetCheckIndexOp()(1, session, result, mainResultSet, ""); - }); - }).Apply([this](TAsyncStatus future) { - TString err; - with_lock(Mtx_) { - if (Err_) - err = Err_; - } - if (err) - throw yexception() << err; - return future; - }); - } - - IWorkLoader::ELoadCommand GetTaskId() const override { - return IWorkLoader::LC_SELECT; - } - -private: - using TCheckIndexCb = std::function<TAsyncStatus( - size_t i, - TSession session, - TDataQueryResult in, - const TString& mainResultSet, - const TString& err)>; - - TCheckIndexCb GetCheckIndexOp() { - return [this] ( - size_t i, - TSession session, - TDataQueryResult in, - const TString& mainResultSet, - const TString& err) mutable - { - TResultSetParser rsParser(in.GetResultSet(0)); - if (!rsParser.TryNextRow()) { - return FinishTxAsync(in); - } - - - if (i == Programs_.size()) { - return in.GetTransaction()->Commit() + } else if (status.GetStatus() == EStatus::SUCCESS) { + // start upsert + const auto& program = Programs_[0]; + const auto& programText = program.first; + + NYdb::TValueBuilder value; + value.BeginStruct(); + for (const auto& x : program.second) { + auto it = result.Values.find(x.first.Name); + if (it != result.Values.end()) { + value.AddMember(x.first.Name, it->second); + } else { + value.AddMember(x.first.Name, ::NIdxTest::CreateValue(x.first, *this)); + } + } + value.EndStruct(); + TVector<NYdb::TValue> batch; + batch.push_back(value.Build()); + + Y_VERIFY(result.Tx); + const auto params = ::NIdxTest::CreateParamsAsList(batch, ParamName_); + + return result.Tx->GetSession().ExecuteDataQuery( + programText, TTxControl::Tx(result.Tx.GetRef()), std::move(params), + TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) + .Apply([](TAsyncDataQueryResult future){ + auto result = future.ExtractValue(); + if (result.IsSuccess()) { + return FinishTxAsync(result); + } else { + return NThreading::MakeFuture<TStatus>(result); + } + + }); + } else { + return NThreading::MakeFuture<TStatus>(status); + } + }); + }, retryOperationSettings); + + } + + IWorkLoader::ELoadCommand GetTaskId() const override { + return IWorkLoader::LC_UPSERT_IF_UNIQ; + } +private: + struct IndexValues { + TStatus Status; + THashMap<TString, NYdb::TValue> Values; + TMaybe<TTransaction> Tx; + }; + using TCheckIndexCb = std::function<NThreading::TFuture<IndexValues>( + size_t i, + TSession session, + TMaybe<TTransaction>, + THashMap<TString, NYdb::TValue>)>; + + TCheckIndexCb GetCheckIndexUniq() { + return [this] ( + size_t i, + TSession session, + TMaybe<TTransaction> tx, + THashMap<TString, NYdb::TValue>&& checked) mutable + { + if (i == Programs_.size()) { + return NThreading::MakeFuture<IndexValues>( + { + TStatus(EStatus::SUCCESS, NYql::TIssues()), + checked, + tx + } + ); + } + + const auto& p = Programs_[i]; + + TVector<NYdb::TValue> values; + TVector<TString> paramNames; + for (const auto col : p.second) { + const auto val = ::NIdxTest::CreateValue(col.first, *this); + + checked.insert({col.first.Name, val}); + values.push_back(val); + paramNames.push_back(col.second); + } + + const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); + + TTxControl txctrl = tx ? TTxControl::Tx(tx.GetRef()) : TTxControl::BeginTx(TTxSettings::SerializableRW()); + + return session.ExecuteDataQuery( + p.first, + txctrl, + params, + TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) + .Apply([this, i, session, checked{std::move(checked)}](TAsyncDataQueryResult future) mutable { + auto result = future.ExtractValue(); + // non success call - return status + if (!result.IsSuccess()) { + return NThreading::MakeFuture<IndexValues>( + { + result, + THashMap<TString, NYdb::TValue>(), + TMaybe<TTransaction>() + } + ); + } + + // success call and no record found (index uniq) - check next one + if (result.GetResultSet(0).RowsCount() == 0) { + return GetCheckIndexUniq()(i + 1, session, result.GetTransaction(), std::move(checked)); + } else { + return NThreading::MakeFuture<IndexValues>( + { + TStatus(EStatus::PRECONDITION_FAILED, NYql::TIssues()), + THashMap<TString, NYdb::TValue>(), + result.GetTransaction() + } + ); + } + }); + }; + } + + + void CreateProgram() { + const auto columns = TableDescription_.GetColumns(); + const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); + + // Used to find column description by column Name + THashMap<TString, std::pair<TColumn, TString>> colHash; + + TVector<std::pair<TColumn, TString>> upsertInput; + + for (const auto& col : columns) { + auto pkType = NIdxTest::CreateValue(col, *this); + auto typeString = NYdb::FormatType(pkType.GetType()); + colHash.insert({col.Name, {col, typeString}}); + upsertInput.push_back({col, ""}); + } + + { + // Create UPSERT program + auto rowType = NIdxTest::CreateRow(columns, *this); + auto rowsTypeString = NYdb::FormatType(rowType.GetType()); + + TString sqlTemplate = "DECLARE %s AS List<%s>; UPSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; + auto program = Sprintf(sqlTemplate.c_str(), + ParamName_.c_str(), + rowsTypeString.c_str(), + TableName_.c_str(), + ParamName_.c_str()); + + Programs_.push_back({SetPragmas(program, UseNewEngine_), upsertInput}); + } + + TString pkColumns; + { + size_t id = 0; + for (const auto& col : pkColNames) { + pkColumns += col; + if (++id != pkColNames.size()) + pkColumns += ", "; + } + } + + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + // Create select pk program for each index + size_t id = 0; + TString indexPredicate; + TString declare; + TVector<std::pair<TColumn, TString>> predicates; + for (const auto& col : indexDesc.GetIndexColumns()) { + const TString paramName = Sprintf("$items_%zu", id); + declare += Sprintf("DECLARE %s AS %s;\n", + paramName.c_str(), colHash.find(col)->second.second.c_str()); + predicates.push_back({colHash.find(col)->second.first, paramName}); + indexPredicate += Sprintf(" %s = %s ", col.c_str(), paramName.c_str()); + if (++id != pkColNames.size()) { + indexPredicate += " AND "; + } + } + + TString sql; + sql += declare; + sql += Sprintf("SELECT %s FROM `%s` VIEW %s WHERE %s", pkColumns.c_str(), + TableName_.c_str(), indexDesc.GetIndexName().c_str(), indexPredicate.c_str()); + + Programs_.push_back({SetPragmas(sql, UseNewEngine_), predicates}); + } + } + + const TString ParamName_ = "$items"; + TTableDescription TableDescription_; + TString TableName_; + TTableClient Client_; + bool UseNewEngine_; + + TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; + + mutable TString Err_; +}; + + +class TSelectAndCompareTask + : public IWorkTask + , public TRandomValueProvider { +public: + TSelectAndCompareTask( + TTableDescription tableDescription, + const TString& tableName, + TTableClient& client, + bool useNewEngine) + : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) + , TableDescription_(tableDescription) + , TableName_(tableName) + , Client_(client) + , UseNewEngine_(useNewEngine) + { + CreateProgram(); + } + + TAsyncStatus Run() override { + if (Programs_.empty()) + throw yexception() << "no program to run"; + + TString err; + with_lock(Mtx_) { + if (Err_) + err = Err_; + } + if (err) + throw yexception() << err; + + const auto& program = Programs_[0]; + + TVector<NYdb::TValue> values; + TVector<TString> paramNames; + for (const auto col : program.second) { + values.push_back(::NIdxTest::CreateOptionalValue(col.first, *this)); + paramNames.push_back(col.second); + } + + const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); + + return Client_.RetryOperation([this, params](TSession session) mutable { + const auto& mainTableQuery = Programs_[0].first; + return session.ExecuteDataQuery( + mainTableQuery, + TTxControl::BeginTx(TTxSettings::SerializableRW()), + params, + TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) + .Apply([this, session](TAsyncDataQueryResult future) mutable { + auto result = future.ExtractValue(); + if (result.GetResultSets().empty()) { + if (result.IsSuccess()) { + return FinishTxAsync(result); + } else { + return NThreading::MakeFuture<TStatus>(result); + } + } + + const auto& mainResultSet = NYdb::FormatResultSetYson(result.GetResultSet(0));; + return GetCheckIndexOp()(1, session, result, mainResultSet, ""); + }); + }).Apply([this](TAsyncStatus future) { + TString err; + with_lock(Mtx_) { + if (Err_) + err = Err_; + } + if (err) + throw yexception() << err; + return future; + }); + } + + IWorkLoader::ELoadCommand GetTaskId() const override { + return IWorkLoader::LC_SELECT; + } + +private: + using TCheckIndexCb = std::function<TAsyncStatus( + size_t i, + TSession session, + TDataQueryResult in, + const TString& mainResultSet, + const TString& err)>; + + TCheckIndexCb GetCheckIndexOp() { + return [this] ( + size_t i, + TSession session, + TDataQueryResult in, + const TString& mainResultSet, + const TString& err) mutable + { + TResultSetParser rsParser(in.GetResultSet(0)); + if (!rsParser.TryNextRow()) { + return FinishTxAsync(in); + } + + + if (i == Programs_.size()) { + return in.GetTransaction()->Commit() .Apply([this, err](TAsyncCommitTransactionResult future) { - auto result = future.ExtractValue(); - if (err && (result.GetStatus() != EStatus::ABORTED)) { - with_lock(Mtx_) { - Err_ += err; - } - } - return NThreading::MakeFuture<TStatus>(result); - }); - } - - const auto& p = Programs_[i]; - - TVector<NYdb::TValue> val; - TVector<TString> parNames; - for (const auto col : p.second) { - val.push_back(rsParser.GetValue(col.first.Name)); - auto vp = TValueParser(val.back()); - vp.OpenOptional(); - if (vp.IsNull()) { - //Cerr << "Null value found, skip check.." << Endl; - return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - } - parNames.push_back(col.second); - } - const auto params = ::NIdxTest::CreateParamsAsItems(val, parNames); - - return session.ExecuteDataQuery( - p.first, - TTxControl::Tx(*in.GetTransaction()), - params, - TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) - .Apply([this, i, session, in, mainResultSet, val](TAsyncDataQueryResult future) mutable { - auto result = future.ExtractValue(); - with_lock(Mtx_) { - if (Err_) { - return FinishTxAsync(result); - } - } - - if (!result.IsSuccess()) { - return NThreading::MakeFuture<TStatus>(result); - } - - try { - auto fromIndexReq = NYdb::FormatResultSetYson(result.GetResultSet(0));; - TString err; - if (fromIndexReq != mainResultSet) { - TStringStream ss; - ss << "result set missmatch, index: " << fromIndexReq << " and main: " << mainResultSet << Endl; - err = ss.Str(); - } - - return GetCheckIndexOp()(i + 1, session, in, mainResultSet, err); - } catch (...) { - Y_FAIL("Unexpected exception"); - return FinishTxAsync(result); - } - }); - }; - } - - - void CreateProgram() { - const auto columns = TableDescription_.GetColumns(); - const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); - - THashMap<TString, std::pair<TColumn, TString>> colHash; - size_t id = 0; - - TString allColumns; - for (const auto& col : columns) { - auto pkType = NIdxTest::CreateOptionalValue(col, *this); - auto typeString = NYdb::FormatType(pkType.GetType()); - colHash.insert({col.Name, {col, typeString}}); - allColumns += col.Name; - if (++id != columns.size()) - allColumns += ", "; - } - - id = 0; - TString pkPredicate; - TString select1; - TVector<std::pair<TColumn, TString>> pkPredicates; - for (const TString& str : pkColNames) { - const TString paramName = Sprintf("$items_%zu", id); - select1 += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), colHash.find(str)->second.second.c_str()); - pkPredicates.push_back({colHash.find(str)->second.first, paramName}); - pkPredicate += Sprintf(" %s = %s ", str.c_str(), paramName.c_str()); - if (++id != pkColNames.size()) { - pkPredicate += " AND "; - } - } - - select1 += Sprintf("SELECT %s FROM `%s` WHERE %s", allColumns.c_str(), TableName_.c_str(), pkPredicate.c_str()); - Programs_.push_back({SetPragmas(select1, UseNewEngine_), pkPredicates}); - - THashMap<TString, std::pair<TString, - TVector<std::pair<TColumn, - TString>>>> selectUsingIndexSql; - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - id = 0; - TString indexPredicate; - TString declare; - TVector<std::pair<TColumn, TString>> predicates; - for (const auto& col : indexDesc.GetIndexColumns()) { - const TString paramName = Sprintf("$items_%zu", id); - declare += Sprintf("DECLARE %s AS %s;\n", - paramName.c_str(), colHash.find(col)->second.second.c_str()); - predicates.push_back({colHash.find(col)->second.first, paramName}); - indexPredicate += Sprintf(" %s = %s ", col.c_str(), paramName.c_str()); - indexPredicate += " AND "; - id++; - } - // Add key column to request to handle non uniq index - for (const TString& str : pkColNames) { - const TString paramName = Sprintf("$items_%zu", id); - declare += Sprintf("DECLARE %s AS %s;\n", - paramName.c_str(), colHash.find(str)->second.second.c_str()); - predicates.push_back({colHash.find(str)->second.first, paramName}); - indexPredicate += Sprintf(" %s = %s ", str.c_str(), paramName.c_str()); - if (++id != pkColNames.size() + indexDesc.GetIndexColumns().size()) { - pkPredicate += " AND "; - } - } - - auto& sql = selectUsingIndexSql[indexDesc.GetIndexName()]; - sql.first += declare; - sql.first += Sprintf("SELECT %s FROM `%s` VIEW %s WHERE %s", allColumns.c_str(), - TableName_.c_str(), indexDesc.GetIndexName().c_str(), indexPredicate.c_str()); - - Programs_.push_back({SetPragmas(sql.first, UseNewEngine_), predicates}); - } - } - - TTableDescription TableDescription_; - TString TableName_; - TTableClient Client_; - const bool UseNewEngine_; - - TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; - - mutable TString Err_; - TMutex Mtx_; -}; - -class TUpdateViaParamItemsTask - : public IWorkTask - , public TRandomValueProvider { -public: - TUpdateViaParamItemsTask( - TTableDescription tableDescription, - const TString& tableName, - TTableClient& client, - IWorkLoader::ELoadCommand stmt, - bool useNewEngine) - : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) - , TableDescription_(tableDescription) - , TableName_(tableName) - , Client_(client) - , Stmt_(stmt) - , UseNewEngine_(useNewEngine) - { - CreatePrograms(); - } - - TAsyncStatus Run() override { - auto rnd = RandomNumber<ui32>(Programs_.size()); - const auto& program = Programs_[rnd]; - const auto& programText = program.first; - - TVector<NYdb::TValue> values; - TVector<TString> paramNames; - for (const auto col : program.second) { - values.push_back(::NIdxTest::CreateOptionalValue(col.first, *this)); - paramNames.push_back(col.second); - } - - const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); - return RunOperation(Client_, programText, params); - } - - std::pair<TString, TVector<std::pair<TColumn, TString>>> CreateProgram(const TString& indexName, bool allColumns = false) { - const auto columns = TableDescription_.GetColumns(); - const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); - - TVector<TColumn> pkCol; - THashSet<TString> pkColHash; - for (const auto& pk : pkColNames) { - pkColHash.insert(pk); - } - for (const auto& col : columns) { - if (pkColHash.contains(col.Name)) { - pkCol.push_back(col); - } - } - - TVector<TColumn> valueCol; - THashSet<TString> indexColumns; - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - if (indexName.empty() || indexDesc.GetIndexName() == indexName) { - for (const auto& col : indexDesc.GetIndexColumns()) { - indexColumns.insert(col); - } - } - } - - for (const auto& col : columns) { - if (indexName.empty()) { - // only columns withount indexes or all columns - if (allColumns || !indexColumns.contains(col.Name)) { - if (!pkColHash.contains(col.Name)) { - valueCol.push_back(col); - } - } - } else { - // only columns with given index - if (indexColumns.contains(col.Name)) { - if (!pkColHash.contains(col.Name)) { - valueCol.push_back(col); - } - } - } - } - - TVector<TString> pkTypeStrings; - for (const auto& pk : pkCol) { - auto pkType = NIdxTest::CreateOptionalValue(pk, *this); - pkTypeStrings.push_back(NYdb::FormatType(pkType.GetType())); - } - - TVector<TString> valueTypeStrings; - for (const auto& v : valueCol) { - auto val = NIdxTest::CreateOptionalValue(v, *this); - valueTypeStrings.push_back(NYdb::FormatType(val.GetType())); - } - - TString sql; - TVector<std::pair<TColumn, TString>> resultColumns; - Y_VERIFY(Stmt_ == IWorkLoader::LC_DELETE || Stmt_ == IWorkLoader::LC_UPDATE); - - size_t id = 0; - TString predicate; - for (const TString& str : pkTypeStrings) { - const TString paramName = Sprintf("$items_%zu", id); - sql += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), str.c_str()); - resultColumns.push_back({pkCol[id], paramName}); - predicate += Sprintf(" %s = %s ", pkCol[id].Name.c_str(), paramName.c_str()); - id++; - if (id != pkTypeStrings.size()) { - predicate += " AND "; - } - } - if (Stmt_ == IWorkLoader::LC_DELETE) { - sql += Sprintf("DELETE FROM `%s` WHERE ", TableName_.c_str()); - sql += predicate; - } else { - TString update = " SET "; - size_t dataId = 0; - for (const TString& str : valueTypeStrings) { - const TString paramName = Sprintf("$items_%zu", id); - sql += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), str.c_str()); - resultColumns.push_back({valueCol[dataId], paramName}); - update += Sprintf(" %s = %s ", valueCol[dataId].Name.c_str(), paramName.c_str()); - id++; - dataId++; - if (dataId != valueTypeStrings.size()) { - update += ", "; - } - } - - sql += Sprintf("UPDATE `%s` ", TableName_.c_str()); - sql += update; - sql += " WHERE "; - sql += predicate; - } - - - return {SetPragmas(sql, UseNewEngine_), resultColumns}; - } - - IWorkLoader::ELoadCommand GetTaskId() const override { - return Stmt_; - } - -private: - void CreatePrograms() { - if (Stmt_ == IWorkLoader::LC_DELETE) { - Programs_.emplace_back(CreateProgram("", false)); - } else { - // All columns - Programs_.emplace_back(CreateProgram("", true)); - // Columns with given index (only if more then 1 index) - if (TableDescription_.GetIndexDescriptions().size() > 1) { - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - Programs_.emplace_back(CreateProgram(indexDesc.GetIndexName())); - } - } - } - } - - TTableDescription TableDescription_; - TString TableName_; - TTableClient Client_; - const IWorkLoader::ELoadCommand Stmt_; - const bool UseNewEngine_; - // program, columns - TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; -}; - -class TUpdateViaParamListTask - : public IWorkTask - , public TRandomValueProvider { -public: - TUpdateViaParamListTask( - TTableDescription tableDescription, - const TString& tableName, - TTableClient& client, - IWorkLoader::ELoadCommand stmt, - bool useNewEngine) - : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) - , TableDescription_(tableDescription) - , TableName_(tableName) - , Client_(client) - , Stmt_(stmt) - , UseNewEngine_(useNewEngine) - { - CreatePrograms(); - } - - TAsyncStatus Run() override { - auto rnd = RandomNumber<ui32>(Programs_.size()); - const auto& program = Programs_[rnd]; - const auto& programText = program.first; - TVector<NYdb::TValue> batch; - batch.emplace_back(::NIdxTest::CreateRow(program.second, *this)); - - const auto params = ::NIdxTest::CreateParamsAsList(batch, ParamName_); - return RunOperation(Client_, programText, params); - } - - IWorkLoader::ELoadCommand GetTaskId() const override { - return Stmt_; - } -private: - - // indexName - create program to touch this indexName only - // indexName empty && !allColumns - touch pk avd value only - // indexName empty && allColumns - touch all columns - std::pair<TString, TVector<TColumn>> CreateUpsertProgram(const TString& indexName, bool allColumns = false) { - const auto columns = TableDescription_.GetColumns(); - TVector<TColumn> resCol; - - THashSet<TString> indexColumns; - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - for (const auto& col : indexDesc.GetIndexColumns()) { - indexColumns.insert(col); - } - } - - THashSet<TString> pks; - for (const auto& pk : TableDescription_.GetPrimaryKeyColumns()) { - pks.insert(pk); - indexColumns.insert(pk); - } - - for (const auto& col : columns) { - if (indexName.empty()) { - // only columns withount indexes or all columns - if (allColumns) { - resCol.push_back(col); - } else { - if (pks.contains(col.Name)) { - resCol.push_back(col); - } - } - } else { - // only columns with given index - if (indexColumns.contains(col.Name)) { - resCol.push_back(col); - } - } - } - - auto rowType = NIdxTest::CreateRow(resCol, *this); - auto rowsTypeString = NYdb::FormatType(rowType.GetType()); - - TString sql; - switch (Stmt_) { - case IWorkLoader::LC_UPSERT: - sql = "DECLARE %s AS List<%s>; UPSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; - break; - case IWorkLoader::LC_REPLACE: - sql = "DECLARE %s AS List<%s>; REPLACE INTO `%s` SELECT * FROM AS_TABLE(%s);"; - break; - case IWorkLoader::LC_INSERT: - sql = "DECLARE %s AS List<%s>; INSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; - break; - case IWorkLoader::LC_UPDATE_ON: - sql = "DECLARE %s AS List<%s>; UPDATE `%s` ON SELECT * FROM AS_TABLE(%s);"; - break; - case IWorkLoader::LC_DELETE_ON: - sql = "DECLARE %s AS List<%s>; DELETE FROM `%s` ON SELECT * FROM AS_TABLE(%s);"; - break; - default: - ythrow yexception() << "unsupported statement"; - break; - } - - auto program = Sprintf(sql.c_str(), - ParamName_.c_str(), - rowsTypeString.c_str(), - TableName_.c_str(), - ParamName_.c_str()); - - return {SetPragmas(program, UseNewEngine_), resCol}; - } - - void CreatePrograms() { - // Only columns without indes - //Programs_.emplace_back(CreateUpsertProgram("", false)); - // All columns - Programs_.emplace_back(CreateUpsertProgram("", true)); - // Columns with given index (only if more then 1 index) - if (TableDescription_.GetIndexDescriptions().size() > 1) { - for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { - Programs_.emplace_back(CreateUpsertProgram(indexDesc.GetIndexName())); - } - } - } - TTableDescription TableDescription_; - TString TableName_; - TTableClient Client_; - const IWorkLoader::ELoadCommand Stmt_; - const bool UseNewEngine_; - // program, columns - TVector<std::pair<TString, TVector<TColumn>>> Programs_; - const TString ParamName_ = "$items"; -}; - -class TTimingTracker { -public: - struct TStat { - TDuration Min; - TDuration Max; - TDuration Total; - ui64 Count = 0; - ui64 Errors = 0; - }; - - void Start(IWorkLoader::ELoadCommand cmd) { - CurStat_ = &Stats_[cmd]; - CurTime_ = TInstant::Now(); - } - - void Finish(bool success) { - auto time = TInstant::Now() - CurTime_; - CurStat_->Total += time; - - if (CurStat_->Count) { - CurStat_->Min = Min(CurStat_->Min, time); - } else { - CurStat_->Min = time; - } - CurStat_->Count++; - CurStat_->Max = Max(CurStat_->Max, time); - if (!success) { - CurStat_->Errors++; - } - } - - const THashMap<IWorkLoader::ELoadCommand, TStat>& GetStat() const { - return Stats_; - } - -private: - THashMap<IWorkLoader::ELoadCommand, TStat> Stats_; - TStat* CurStat_ = nullptr; - TInstant CurTime_; -}; - - -NJson::TJsonValue GetQueryLabels(IWorkLoader::ELoadCommand queryId) { - NJson::TJsonValue labels(NJson::JSON_MAP); - TStringStream ss; - ss << queryId; - labels.InsertValue("query", ss.Str()); - return labels; -} - -NJson::TJsonValue GetSensorValue(TStringBuf sensor, const TDuration& value, IWorkLoader::ELoadCommand queryId) { - NJson::TJsonValue sensorValue(NJson::JSON_MAP); - sensorValue.InsertValue("sensor", sensor); - sensorValue.InsertValue("value", value.MilliSeconds()); - sensorValue.InsertValue("labels", GetQueryLabels(queryId)); - return sensorValue; -} - -NJson::TJsonValue GetSensorValue(TStringBuf sensor, double value, IWorkLoader::ELoadCommand queryId) { - NJson::TJsonValue sensorValue(NJson::JSON_MAP); - sensorValue.InsertValue("sensor", sensor); - sensorValue.InsertValue("value", value); - sensorValue.InsertValue("labels", GetQueryLabels(queryId)); - return sensorValue; -} - - -class TWorkLoader : public IWorkLoader { - using TWorkTaskEntry = std::pair<IWorkTask::TPtr, ui32>; -public: - TWorkLoader(TTableClient&& client, IProgressTracker::TPtr&& progressTracker) - : Client_(std::move(client)) - , ProgressTracker_(std::move(progressTracker)) - {} - - NJson::TJsonValue Run(const TString& tableName, ui32 loadCommands, const TRunSettings& settings) override { - if (!loadCommands) - return {}; - - const auto runLimit = settings.RunLimit; - - if (ProgressTracker_) { - ProgressTracker_->Start("reqests processed", "Run work loader on table " + tableName); - } - - auto tableDescription = DescribeTable(tableName, Client_); - - if (!tableDescription) - throw yexception() << "Unable to describe table " << tableName << Endl; - - TVector<TWorkTaskEntry> tasks; - for (ui64 i = 1; i < Max<ui32>(); i <<= 1) { - switch (i & loadCommands) { - case IWorkLoader::LC_UPSERT: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPSERT, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_INSERT: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_INSERT, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_UPDATE: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamItemsTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPDATE, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_UPDATE_ON: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPDATE_ON, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_REPLACE: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_REPLACE, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_DELETE: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamItemsTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_DELETE, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_DELETE_ON: - tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( - tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_DELETE_ON, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_SELECT: - tasks.emplace_back(std::make_pair(std::make_unique<TSelectAndCompareTask>( - tableDescription.GetRef(), tableName, Client_, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_UPSERT_IF_UNIQ: - tasks.emplace_back(std::make_pair(std::make_unique<TSelectAndUpsertIfUniqTask>( - tableDescription.GetRef(), tableName, Client_, settings.NewEngine), 1000)); - break; - case IWorkLoader::LC_ALTER_ADD_INDEX: - tasks.emplace_back(std::make_pair(std::make_unique<TAlterTableAddIndexTask>( - tableDescription.GetRef(), tableName, Client_, false), 1)); - break; - case IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN: - tasks.emplace_back(std::make_pair(std::make_unique<TAlterTableAddIndexTask>( - tableDescription.GetRef(), tableName, Client_, true), 1)); - break; - } - } - - WeightPrecalc(tasks); - - TVector<TAsyncStatus> results; - TAtomic curAtomicRunLimit = runLimit - settings.Infly; //infly already "counted" by initiall call - auto pt = ProgressTracker_.get(); - - std::function<TAsyncStatus(TTimingTracker*)> runAsync; - runAsync = [&tasks, &curAtomicRunLimit, &runAsync, pt, runLimit] (TTimingTracker* tt) -> TAsyncStatus { - auto task = GetRandomTask(tasks); - if (pt) { - pt->Update(runLimit - AtomicGet(curAtomicRunLimit)); - } - tt->Start(task->GetTaskId()); - return task->Run().Apply([&curAtomicRunLimit, &runAsync, tt](TAsyncStatus status) { - int newValue = AtomicDecrement(curAtomicRunLimit); - if (!status.HasException() && status.GetValue().IsSuccess()) { - tt->Finish(true); - } else { - tt->Finish(false); - } - if (status.HasException() || newValue < 0 || - (!status.GetValue().IsSuccess() && status.GetValue().GetStatus() != EStatus::PRECONDITION_FAILED)) { - Cerr << "finished with status: " << status.GetValue().GetStatus() << Endl; - return status; - } else { - if (!status.GetValue().IsSuccess()) - Cerr << "cont with status: " << status.GetValue().GetStatus() << Endl; - return runAsync(tt); - } - }); - }; - - TVector<TTimingTracker> timing; - timing.resize(settings.Infly); - for (size_t i = 0; i < settings.Infly; i++) { - results.push_back(runAsync(&timing[i])); - } - - TString err; - for (size_t i = 0; i < settings.Infly; i++) { - try { - if (!results[i].GetValueSync().IsSuccess()) { - TStringStream ss; - ss << "non success status: "; - ss << results[i].GetValueSync().GetStatus(); - ss << " issues: " << results[i].GetValueSync().GetIssues().ToString(); - ss << "\n"; - err += ss.Str(); - } - } catch (const std::exception& ex) { - err += ex.what(); - } - } - - if (ProgressTracker_) { - ProgressTracker_->Finish("Done."); - } - - auto result = AggregateTiming(timing); - - NJson::TJsonValue jsonReport(NJson::JSON_ARRAY); - - for (const auto& queryRes : result) { - jsonReport.AppendValue(GetSensorValue("MinTime", queryRes.second.Min, queryRes.first)); - jsonReport.AppendValue(GetSensorValue("MaxTime", queryRes.second.Max, queryRes.first)); - jsonReport.AppendValue(GetSensorValue("MeanTime", queryRes.second.Total / (double)queryRes.second.Count, queryRes.first)); - jsonReport.AppendValue(GetSensorValue("VisiableErrorsCount", queryRes.second.Errors, queryRes.first)); - } - - if (err) - throw yexception() << err; - - return jsonReport; - } - -private: - static THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat> AggregateTiming(const TVector<TTimingTracker>& timings) { - THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat> result; - bool first = true; - - for (const auto& x : timings) { - const THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat>& t = x.GetStat(); - for (const auto& pair : t) { - TTimingTracker::TStat& stat = result[pair.first]; - - stat.Max = Max(stat.Max, pair.second.Max); - if (first) { - stat.Min = pair.second.Min; - } else { - stat.Min = Min(stat.Min, pair.second.Min); - } - stat.Count += pair.second.Count; - stat.Total += pair.second.Total; - stat.Errors += pair.second.Errors; - } - } - return result; - } - - static bool CompareWorkTaskEntry(const TWorkTaskEntry& a, const TWorkTaskEntry& b) { - return a.second < b.second; - } - - static bool CompareValueWorkTaskEntry(ui32 a, const TWorkTaskEntry& b) { - return a < b.second; - } - - static void WeightPrecalc(TVector<TWorkTaskEntry>& tasks) { - ui32 s = 0; - for (auto& t : tasks) { - s += t.second; - t.second = s; - } - - std::sort(tasks.begin(), tasks.end(), &CompareWorkTaskEntry); - } - - static IWorkTask* GetRandomTask(TVector<TWorkTaskEntry>& tasks) { - auto rnd = RandomNumber<ui32>(tasks.back().second); - // First element that is greater than rnd - const auto upper = std::upper_bound(tasks.begin(), tasks.end(), rnd, &CompareValueWorkTaskEntry); - return upper->first.get(); - } - - TTableClient Client_; - IProgressTracker::TPtr ProgressTracker_; -}; - -IWorkLoader::TPtr CreateWorkLoader(TDriver& driver, IProgressTracker::TPtr&& progressTracker) { - auto settings = TClientSettings() - .SessionPoolSettings( - TSessionPoolSettings() - .MaxActiveSessions(2000)); - return std::make_unique<TWorkLoader>(TTableClient(driver, settings), std::move(progressTracker)); -} - -} // namespace NIdxTest + auto result = future.ExtractValue(); + if (err && (result.GetStatus() != EStatus::ABORTED)) { + with_lock(Mtx_) { + Err_ += err; + } + } + return NThreading::MakeFuture<TStatus>(result); + }); + } + + const auto& p = Programs_[i]; + + TVector<NYdb::TValue> val; + TVector<TString> parNames; + for (const auto col : p.second) { + val.push_back(rsParser.GetValue(col.first.Name)); + auto vp = TValueParser(val.back()); + vp.OpenOptional(); + if (vp.IsNull()) { + //Cerr << "Null value found, skip check.." << Endl; + return NThreading::MakeFuture<TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + } + parNames.push_back(col.second); + } + const auto params = ::NIdxTest::CreateParamsAsItems(val, parNames); + + return session.ExecuteDataQuery( + p.first, + TTxControl::Tx(*in.GetTransaction()), + params, + TExecDataQuerySettings().KeepInQueryCache(true).ClientTimeout(TDuration::Seconds(5))) + .Apply([this, i, session, in, mainResultSet, val](TAsyncDataQueryResult future) mutable { + auto result = future.ExtractValue(); + with_lock(Mtx_) { + if (Err_) { + return FinishTxAsync(result); + } + } + + if (!result.IsSuccess()) { + return NThreading::MakeFuture<TStatus>(result); + } + + try { + auto fromIndexReq = NYdb::FormatResultSetYson(result.GetResultSet(0));; + TString err; + if (fromIndexReq != mainResultSet) { + TStringStream ss; + ss << "result set missmatch, index: " << fromIndexReq << " and main: " << mainResultSet << Endl; + err = ss.Str(); + } + + return GetCheckIndexOp()(i + 1, session, in, mainResultSet, err); + } catch (...) { + Y_FAIL("Unexpected exception"); + return FinishTxAsync(result); + } + }); + }; + } + + + void CreateProgram() { + const auto columns = TableDescription_.GetColumns(); + const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); + + THashMap<TString, std::pair<TColumn, TString>> colHash; + size_t id = 0; + + TString allColumns; + for (const auto& col : columns) { + auto pkType = NIdxTest::CreateOptionalValue(col, *this); + auto typeString = NYdb::FormatType(pkType.GetType()); + colHash.insert({col.Name, {col, typeString}}); + allColumns += col.Name; + if (++id != columns.size()) + allColumns += ", "; + } + + id = 0; + TString pkPredicate; + TString select1; + TVector<std::pair<TColumn, TString>> pkPredicates; + for (const TString& str : pkColNames) { + const TString paramName = Sprintf("$items_%zu", id); + select1 += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), colHash.find(str)->second.second.c_str()); + pkPredicates.push_back({colHash.find(str)->second.first, paramName}); + pkPredicate += Sprintf(" %s = %s ", str.c_str(), paramName.c_str()); + if (++id != pkColNames.size()) { + pkPredicate += " AND "; + } + } + + select1 += Sprintf("SELECT %s FROM `%s` WHERE %s", allColumns.c_str(), TableName_.c_str(), pkPredicate.c_str()); + Programs_.push_back({SetPragmas(select1, UseNewEngine_), pkPredicates}); + + THashMap<TString, std::pair<TString, + TVector<std::pair<TColumn, + TString>>>> selectUsingIndexSql; + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + id = 0; + TString indexPredicate; + TString declare; + TVector<std::pair<TColumn, TString>> predicates; + for (const auto& col : indexDesc.GetIndexColumns()) { + const TString paramName = Sprintf("$items_%zu", id); + declare += Sprintf("DECLARE %s AS %s;\n", + paramName.c_str(), colHash.find(col)->second.second.c_str()); + predicates.push_back({colHash.find(col)->second.first, paramName}); + indexPredicate += Sprintf(" %s = %s ", col.c_str(), paramName.c_str()); + indexPredicate += " AND "; + id++; + } + // Add key column to request to handle non uniq index + for (const TString& str : pkColNames) { + const TString paramName = Sprintf("$items_%zu", id); + declare += Sprintf("DECLARE %s AS %s;\n", + paramName.c_str(), colHash.find(str)->second.second.c_str()); + predicates.push_back({colHash.find(str)->second.first, paramName}); + indexPredicate += Sprintf(" %s = %s ", str.c_str(), paramName.c_str()); + if (++id != pkColNames.size() + indexDesc.GetIndexColumns().size()) { + pkPredicate += " AND "; + } + } + + auto& sql = selectUsingIndexSql[indexDesc.GetIndexName()]; + sql.first += declare; + sql.first += Sprintf("SELECT %s FROM `%s` VIEW %s WHERE %s", allColumns.c_str(), + TableName_.c_str(), indexDesc.GetIndexName().c_str(), indexPredicate.c_str()); + + Programs_.push_back({SetPragmas(sql.first, UseNewEngine_), predicates}); + } + } + + TTableDescription TableDescription_; + TString TableName_; + TTableClient Client_; + const bool UseNewEngine_; + + TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; + + mutable TString Err_; + TMutex Mtx_; +}; + +class TUpdateViaParamItemsTask + : public IWorkTask + , public TRandomValueProvider { +public: + TUpdateViaParamItemsTask( + TTableDescription tableDescription, + const TString& tableName, + TTableClient& client, + IWorkLoader::ELoadCommand stmt, + bool useNewEngine) + : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) + , TableDescription_(tableDescription) + , TableName_(tableName) + , Client_(client) + , Stmt_(stmt) + , UseNewEngine_(useNewEngine) + { + CreatePrograms(); + } + + TAsyncStatus Run() override { + auto rnd = RandomNumber<ui32>(Programs_.size()); + const auto& program = Programs_[rnd]; + const auto& programText = program.first; + + TVector<NYdb::TValue> values; + TVector<TString> paramNames; + for (const auto col : program.second) { + values.push_back(::NIdxTest::CreateOptionalValue(col.first, *this)); + paramNames.push_back(col.second); + } + + const auto params = ::NIdxTest::CreateParamsAsItems(values, paramNames); + return RunOperation(Client_, programText, params); + } + + std::pair<TString, TVector<std::pair<TColumn, TString>>> CreateProgram(const TString& indexName, bool allColumns = false) { + const auto columns = TableDescription_.GetColumns(); + const auto pkColNames = TableDescription_.GetPrimaryKeyColumns(); + + TVector<TColumn> pkCol; + THashSet<TString> pkColHash; + for (const auto& pk : pkColNames) { + pkColHash.insert(pk); + } + for (const auto& col : columns) { + if (pkColHash.contains(col.Name)) { + pkCol.push_back(col); + } + } + + TVector<TColumn> valueCol; + THashSet<TString> indexColumns; + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + if (indexName.empty() || indexDesc.GetIndexName() == indexName) { + for (const auto& col : indexDesc.GetIndexColumns()) { + indexColumns.insert(col); + } + } + } + + for (const auto& col : columns) { + if (indexName.empty()) { + // only columns withount indexes or all columns + if (allColumns || !indexColumns.contains(col.Name)) { + if (!pkColHash.contains(col.Name)) { + valueCol.push_back(col); + } + } + } else { + // only columns with given index + if (indexColumns.contains(col.Name)) { + if (!pkColHash.contains(col.Name)) { + valueCol.push_back(col); + } + } + } + } + + TVector<TString> pkTypeStrings; + for (const auto& pk : pkCol) { + auto pkType = NIdxTest::CreateOptionalValue(pk, *this); + pkTypeStrings.push_back(NYdb::FormatType(pkType.GetType())); + } + + TVector<TString> valueTypeStrings; + for (const auto& v : valueCol) { + auto val = NIdxTest::CreateOptionalValue(v, *this); + valueTypeStrings.push_back(NYdb::FormatType(val.GetType())); + } + + TString sql; + TVector<std::pair<TColumn, TString>> resultColumns; + Y_VERIFY(Stmt_ == IWorkLoader::LC_DELETE || Stmt_ == IWorkLoader::LC_UPDATE); + + size_t id = 0; + TString predicate; + for (const TString& str : pkTypeStrings) { + const TString paramName = Sprintf("$items_%zu", id); + sql += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), str.c_str()); + resultColumns.push_back({pkCol[id], paramName}); + predicate += Sprintf(" %s = %s ", pkCol[id].Name.c_str(), paramName.c_str()); + id++; + if (id != pkTypeStrings.size()) { + predicate += " AND "; + } + } + if (Stmt_ == IWorkLoader::LC_DELETE) { + sql += Sprintf("DELETE FROM `%s` WHERE ", TableName_.c_str()); + sql += predicate; + } else { + TString update = " SET "; + size_t dataId = 0; + for (const TString& str : valueTypeStrings) { + const TString paramName = Sprintf("$items_%zu", id); + sql += Sprintf("DECLARE %s AS %s;\n", paramName.c_str(), str.c_str()); + resultColumns.push_back({valueCol[dataId], paramName}); + update += Sprintf(" %s = %s ", valueCol[dataId].Name.c_str(), paramName.c_str()); + id++; + dataId++; + if (dataId != valueTypeStrings.size()) { + update += ", "; + } + } + + sql += Sprintf("UPDATE `%s` ", TableName_.c_str()); + sql += update; + sql += " WHERE "; + sql += predicate; + } + + + return {SetPragmas(sql, UseNewEngine_), resultColumns}; + } + + IWorkLoader::ELoadCommand GetTaskId() const override { + return Stmt_; + } + +private: + void CreatePrograms() { + if (Stmt_ == IWorkLoader::LC_DELETE) { + Programs_.emplace_back(CreateProgram("", false)); + } else { + // All columns + Programs_.emplace_back(CreateProgram("", true)); + // Columns with given index (only if more then 1 index) + if (TableDescription_.GetIndexDescriptions().size() > 1) { + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + Programs_.emplace_back(CreateProgram(indexDesc.GetIndexName())); + } + } + } + } + + TTableDescription TableDescription_; + TString TableName_; + TTableClient Client_; + const IWorkLoader::ELoadCommand Stmt_; + const bool UseNewEngine_; + // program, columns + TVector<std::pair<TString, TVector<std::pair<TColumn, TString>>>> Programs_; +}; + +class TUpdateViaParamListTask + : public IWorkTask + , public TRandomValueProvider { +public: + TUpdateViaParamListTask( + TTableDescription tableDescription, + const TString& tableName, + TTableClient& client, + IWorkLoader::ELoadCommand stmt, + bool useNewEngine) + : TRandomValueProvider(TableDescriptionToShardsPower(tableDescription), KEYRANGELIMIT) + , TableDescription_(tableDescription) + , TableName_(tableName) + , Client_(client) + , Stmt_(stmt) + , UseNewEngine_(useNewEngine) + { + CreatePrograms(); + } + + TAsyncStatus Run() override { + auto rnd = RandomNumber<ui32>(Programs_.size()); + const auto& program = Programs_[rnd]; + const auto& programText = program.first; + TVector<NYdb::TValue> batch; + batch.emplace_back(::NIdxTest::CreateRow(program.second, *this)); + + const auto params = ::NIdxTest::CreateParamsAsList(batch, ParamName_); + return RunOperation(Client_, programText, params); + } + + IWorkLoader::ELoadCommand GetTaskId() const override { + return Stmt_; + } +private: + + // indexName - create program to touch this indexName only + // indexName empty && !allColumns - touch pk avd value only + // indexName empty && allColumns - touch all columns + std::pair<TString, TVector<TColumn>> CreateUpsertProgram(const TString& indexName, bool allColumns = false) { + const auto columns = TableDescription_.GetColumns(); + TVector<TColumn> resCol; + + THashSet<TString> indexColumns; + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + for (const auto& col : indexDesc.GetIndexColumns()) { + indexColumns.insert(col); + } + } + + THashSet<TString> pks; + for (const auto& pk : TableDescription_.GetPrimaryKeyColumns()) { + pks.insert(pk); + indexColumns.insert(pk); + } + + for (const auto& col : columns) { + if (indexName.empty()) { + // only columns withount indexes or all columns + if (allColumns) { + resCol.push_back(col); + } else { + if (pks.contains(col.Name)) { + resCol.push_back(col); + } + } + } else { + // only columns with given index + if (indexColumns.contains(col.Name)) { + resCol.push_back(col); + } + } + } + + auto rowType = NIdxTest::CreateRow(resCol, *this); + auto rowsTypeString = NYdb::FormatType(rowType.GetType()); + + TString sql; + switch (Stmt_) { + case IWorkLoader::LC_UPSERT: + sql = "DECLARE %s AS List<%s>; UPSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; + break; + case IWorkLoader::LC_REPLACE: + sql = "DECLARE %s AS List<%s>; REPLACE INTO `%s` SELECT * FROM AS_TABLE(%s);"; + break; + case IWorkLoader::LC_INSERT: + sql = "DECLARE %s AS List<%s>; INSERT INTO `%s` SELECT * FROM AS_TABLE(%s);"; + break; + case IWorkLoader::LC_UPDATE_ON: + sql = "DECLARE %s AS List<%s>; UPDATE `%s` ON SELECT * FROM AS_TABLE(%s);"; + break; + case IWorkLoader::LC_DELETE_ON: + sql = "DECLARE %s AS List<%s>; DELETE FROM `%s` ON SELECT * FROM AS_TABLE(%s);"; + break; + default: + ythrow yexception() << "unsupported statement"; + break; + } + + auto program = Sprintf(sql.c_str(), + ParamName_.c_str(), + rowsTypeString.c_str(), + TableName_.c_str(), + ParamName_.c_str()); + + return {SetPragmas(program, UseNewEngine_), resCol}; + } + + void CreatePrograms() { + // Only columns without indes + //Programs_.emplace_back(CreateUpsertProgram("", false)); + // All columns + Programs_.emplace_back(CreateUpsertProgram("", true)); + // Columns with given index (only if more then 1 index) + if (TableDescription_.GetIndexDescriptions().size() > 1) { + for (const auto& indexDesc : TableDescription_.GetIndexDescriptions()) { + Programs_.emplace_back(CreateUpsertProgram(indexDesc.GetIndexName())); + } + } + } + TTableDescription TableDescription_; + TString TableName_; + TTableClient Client_; + const IWorkLoader::ELoadCommand Stmt_; + const bool UseNewEngine_; + // program, columns + TVector<std::pair<TString, TVector<TColumn>>> Programs_; + const TString ParamName_ = "$items"; +}; + +class TTimingTracker { +public: + struct TStat { + TDuration Min; + TDuration Max; + TDuration Total; + ui64 Count = 0; + ui64 Errors = 0; + }; + + void Start(IWorkLoader::ELoadCommand cmd) { + CurStat_ = &Stats_[cmd]; + CurTime_ = TInstant::Now(); + } + + void Finish(bool success) { + auto time = TInstant::Now() - CurTime_; + CurStat_->Total += time; + + if (CurStat_->Count) { + CurStat_->Min = Min(CurStat_->Min, time); + } else { + CurStat_->Min = time; + } + CurStat_->Count++; + CurStat_->Max = Max(CurStat_->Max, time); + if (!success) { + CurStat_->Errors++; + } + } + + const THashMap<IWorkLoader::ELoadCommand, TStat>& GetStat() const { + return Stats_; + } + +private: + THashMap<IWorkLoader::ELoadCommand, TStat> Stats_; + TStat* CurStat_ = nullptr; + TInstant CurTime_; +}; + + +NJson::TJsonValue GetQueryLabels(IWorkLoader::ELoadCommand queryId) { + NJson::TJsonValue labels(NJson::JSON_MAP); + TStringStream ss; + ss << queryId; + labels.InsertValue("query", ss.Str()); + return labels; +} + +NJson::TJsonValue GetSensorValue(TStringBuf sensor, const TDuration& value, IWorkLoader::ELoadCommand queryId) { + NJson::TJsonValue sensorValue(NJson::JSON_MAP); + sensorValue.InsertValue("sensor", sensor); + sensorValue.InsertValue("value", value.MilliSeconds()); + sensorValue.InsertValue("labels", GetQueryLabels(queryId)); + return sensorValue; +} + +NJson::TJsonValue GetSensorValue(TStringBuf sensor, double value, IWorkLoader::ELoadCommand queryId) { + NJson::TJsonValue sensorValue(NJson::JSON_MAP); + sensorValue.InsertValue("sensor", sensor); + sensorValue.InsertValue("value", value); + sensorValue.InsertValue("labels", GetQueryLabels(queryId)); + return sensorValue; +} + + +class TWorkLoader : public IWorkLoader { + using TWorkTaskEntry = std::pair<IWorkTask::TPtr, ui32>; +public: + TWorkLoader(TTableClient&& client, IProgressTracker::TPtr&& progressTracker) + : Client_(std::move(client)) + , ProgressTracker_(std::move(progressTracker)) + {} + + NJson::TJsonValue Run(const TString& tableName, ui32 loadCommands, const TRunSettings& settings) override { + if (!loadCommands) + return {}; + + const auto runLimit = settings.RunLimit; + + if (ProgressTracker_) { + ProgressTracker_->Start("reqests processed", "Run work loader on table " + tableName); + } + + auto tableDescription = DescribeTable(tableName, Client_); + + if (!tableDescription) + throw yexception() << "Unable to describe table " << tableName << Endl; + + TVector<TWorkTaskEntry> tasks; + for (ui64 i = 1; i < Max<ui32>(); i <<= 1) { + switch (i & loadCommands) { + case IWorkLoader::LC_UPSERT: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPSERT, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_INSERT: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_INSERT, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_UPDATE: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamItemsTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPDATE, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_UPDATE_ON: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_UPDATE_ON, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_REPLACE: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_REPLACE, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_DELETE: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamItemsTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_DELETE, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_DELETE_ON: + tasks.emplace_back(std::make_pair(std::make_unique<TUpdateViaParamListTask>( + tableDescription.GetRef(), tableName, Client_, IWorkLoader::LC_DELETE_ON, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_SELECT: + tasks.emplace_back(std::make_pair(std::make_unique<TSelectAndCompareTask>( + tableDescription.GetRef(), tableName, Client_, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_UPSERT_IF_UNIQ: + tasks.emplace_back(std::make_pair(std::make_unique<TSelectAndUpsertIfUniqTask>( + tableDescription.GetRef(), tableName, Client_, settings.NewEngine), 1000)); + break; + case IWorkLoader::LC_ALTER_ADD_INDEX: + tasks.emplace_back(std::make_pair(std::make_unique<TAlterTableAddIndexTask>( + tableDescription.GetRef(), tableName, Client_, false), 1)); + break; + case IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN: + tasks.emplace_back(std::make_pair(std::make_unique<TAlterTableAddIndexTask>( + tableDescription.GetRef(), tableName, Client_, true), 1)); + break; + } + } + + WeightPrecalc(tasks); + + TVector<TAsyncStatus> results; + TAtomic curAtomicRunLimit = runLimit - settings.Infly; //infly already "counted" by initiall call + auto pt = ProgressTracker_.get(); + + std::function<TAsyncStatus(TTimingTracker*)> runAsync; + runAsync = [&tasks, &curAtomicRunLimit, &runAsync, pt, runLimit] (TTimingTracker* tt) -> TAsyncStatus { + auto task = GetRandomTask(tasks); + if (pt) { + pt->Update(runLimit - AtomicGet(curAtomicRunLimit)); + } + tt->Start(task->GetTaskId()); + return task->Run().Apply([&curAtomicRunLimit, &runAsync, tt](TAsyncStatus status) { + int newValue = AtomicDecrement(curAtomicRunLimit); + if (!status.HasException() && status.GetValue().IsSuccess()) { + tt->Finish(true); + } else { + tt->Finish(false); + } + if (status.HasException() || newValue < 0 || + (!status.GetValue().IsSuccess() && status.GetValue().GetStatus() != EStatus::PRECONDITION_FAILED)) { + Cerr << "finished with status: " << status.GetValue().GetStatus() << Endl; + return status; + } else { + if (!status.GetValue().IsSuccess()) + Cerr << "cont with status: " << status.GetValue().GetStatus() << Endl; + return runAsync(tt); + } + }); + }; + + TVector<TTimingTracker> timing; + timing.resize(settings.Infly); + for (size_t i = 0; i < settings.Infly; i++) { + results.push_back(runAsync(&timing[i])); + } + + TString err; + for (size_t i = 0; i < settings.Infly; i++) { + try { + if (!results[i].GetValueSync().IsSuccess()) { + TStringStream ss; + ss << "non success status: "; + ss << results[i].GetValueSync().GetStatus(); + ss << " issues: " << results[i].GetValueSync().GetIssues().ToString(); + ss << "\n"; + err += ss.Str(); + } + } catch (const std::exception& ex) { + err += ex.what(); + } + } + + if (ProgressTracker_) { + ProgressTracker_->Finish("Done."); + } + + auto result = AggregateTiming(timing); + + NJson::TJsonValue jsonReport(NJson::JSON_ARRAY); + + for (const auto& queryRes : result) { + jsonReport.AppendValue(GetSensorValue("MinTime", queryRes.second.Min, queryRes.first)); + jsonReport.AppendValue(GetSensorValue("MaxTime", queryRes.second.Max, queryRes.first)); + jsonReport.AppendValue(GetSensorValue("MeanTime", queryRes.second.Total / (double)queryRes.second.Count, queryRes.first)); + jsonReport.AppendValue(GetSensorValue("VisiableErrorsCount", queryRes.second.Errors, queryRes.first)); + } + + if (err) + throw yexception() << err; + + return jsonReport; + } + +private: + static THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat> AggregateTiming(const TVector<TTimingTracker>& timings) { + THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat> result; + bool first = true; + + for (const auto& x : timings) { + const THashMap<IWorkLoader::ELoadCommand, TTimingTracker::TStat>& t = x.GetStat(); + for (const auto& pair : t) { + TTimingTracker::TStat& stat = result[pair.first]; + + stat.Max = Max(stat.Max, pair.second.Max); + if (first) { + stat.Min = pair.second.Min; + } else { + stat.Min = Min(stat.Min, pair.second.Min); + } + stat.Count += pair.second.Count; + stat.Total += pair.second.Total; + stat.Errors += pair.second.Errors; + } + } + return result; + } + + static bool CompareWorkTaskEntry(const TWorkTaskEntry& a, const TWorkTaskEntry& b) { + return a.second < b.second; + } + + static bool CompareValueWorkTaskEntry(ui32 a, const TWorkTaskEntry& b) { + return a < b.second; + } + + static void WeightPrecalc(TVector<TWorkTaskEntry>& tasks) { + ui32 s = 0; + for (auto& t : tasks) { + s += t.second; + t.second = s; + } + + std::sort(tasks.begin(), tasks.end(), &CompareWorkTaskEntry); + } + + static IWorkTask* GetRandomTask(TVector<TWorkTaskEntry>& tasks) { + auto rnd = RandomNumber<ui32>(tasks.back().second); + // First element that is greater than rnd + const auto upper = std::upper_bound(tasks.begin(), tasks.end(), rnd, &CompareValueWorkTaskEntry); + return upper->first.get(); + } + + TTableClient Client_; + IProgressTracker::TPtr ProgressTracker_; +}; + +IWorkLoader::TPtr CreateWorkLoader(TDriver& driver, IProgressTracker::TPtr&& progressTracker) { + auto settings = TClientSettings() + .SessionPoolSettings( + TSessionPoolSettings() + .MaxActiveSessions(2000)); + return std::make_unique<TWorkLoader>(TTableClient(driver, settings), std::move(progressTracker)); +} + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_stderr_progress_tracker.cpp b/ydb/public/lib/idx_test/idx_test_stderr_progress_tracker.cpp index f227bd0a80..42bb67ffb8 100644 --- a/ydb/public/lib/idx_test/idx_test_stderr_progress_tracker.cpp +++ b/ydb/public/lib/idx_test/idx_test_stderr_progress_tracker.cpp @@ -1,104 +1,104 @@ -#include "idx_test.h" - -#include <util/string/printf.h> -#include <util/system/mutex.h> - -namespace NIdxTest { - -class TStderrProgressTracker : public IProgressTracker { -public: - TStderrProgressTracker(size_t subsampling, const TString& title) - : Subsampling_(subsampling) - { - LastSampledUpdateTime_ = TInstant::Now(); - LastSampledUpdateValue_ = 0; - Cerr << title << Endl; - } - void Start(const TString& progressString, const TString& freeMessage) override { - ProgressString_ = " %lu " + progressString + " (cur: %.2f tps)"; - if (!freeMessage) - return; - auto msg = ShiftText(freeMessage, 1); - Cerr << msg << Endl; - } - - void Update(size_t pos) override { - DoUpdate(pos, false); - } - +#include "idx_test.h" + +#include <util/string/printf.h> +#include <util/system/mutex.h> + +namespace NIdxTest { + +class TStderrProgressTracker : public IProgressTracker { +public: + TStderrProgressTracker(size_t subsampling, const TString& title) + : Subsampling_(subsampling) + { + LastSampledUpdateTime_ = TInstant::Now(); + LastSampledUpdateValue_ = 0; + Cerr << title << Endl; + } + void Start(const TString& progressString, const TString& freeMessage) override { + ProgressString_ = " %lu " + progressString + " (cur: %.2f tps)"; + if (!freeMessage) + return; + auto msg = ShiftText(freeMessage, 1); + Cerr << msg << Endl; + } + + void Update(size_t pos) override { + DoUpdate(pos, false); + } + void Finish(const TString& freeMessage) override { - if (LastUpdateValue_) { - DoUpdate(LastUpdateValue_.GetRef(), true); - Cerr << Endl; - } - auto msg = ShiftText(freeMessage, 1); - Cerr << msg << Endl; - } - -private: - void DoUpdate(size_t pos, bool force) { - TInstant curTime; - TGuard lock(Mtx_); - if (!CurUpdate_ || force) { - curTime = TInstant::Now(); - static ui8 counter; - counter++; - const TString symbols[4] = {" -", " \\", " |", " /"}; - const TDuration delta = - TDuration::MicroSeconds(curTime.MicroSeconds() - LastSampledUpdateTime_.MicroSeconds()); - float tps = 0; - if (delta) { - tps = (pos - LastSampledUpdateValue_) / delta.SecondsFloat(); - } - auto msg = Sprintf((symbols[counter % 4] + ProgressString_).c_str(), pos, tps); - UpdateSz_ = Max(UpdateSz_, msg.size()); - msg.resize(UpdateSz_); - - Cerr << msg << "\r"; - fflush(stderr); - LastSampledUpdateTime_ = curTime; - LastSampledUpdateValue_ = pos; - } - - LastUpdateValue_ = pos; - if (CurUpdate_ == Subsampling_) { - CurUpdate_ = 0; - } else { - CurUpdate_++; - } - } - - TString ShiftText(const TString& in, size_t n) { - TString shift; - shift.resize(n); - TString msg = " " + in; - size_t pos = 0; - while(true) { - pos = msg.find('\n', pos); - if (pos != msg.npos) { - if (pos < msg.size() - 1) { - pos++; - msg.insert(pos, shift); - } else { - break; - } - } else { - break; - } - } - return msg; - } - const size_t Subsampling_; - size_t CurUpdate_ = 0; - TString ProgressString_; - size_t UpdateSz_ = 0; - TMaybe<size_t> LastUpdateValue_; - TInstant LastSampledUpdateTime_; - size_t LastSampledUpdateValue_; - TMutex Mtx_; -}; - -IProgressTracker::TPtr CreateStderrProgressTracker(size_t subsampling, const TString& title) { - return std::make_unique<TStderrProgressTracker>(subsampling, title); -} -} // namespace NIdxTest + if (LastUpdateValue_) { + DoUpdate(LastUpdateValue_.GetRef(), true); + Cerr << Endl; + } + auto msg = ShiftText(freeMessage, 1); + Cerr << msg << Endl; + } + +private: + void DoUpdate(size_t pos, bool force) { + TInstant curTime; + TGuard lock(Mtx_); + if (!CurUpdate_ || force) { + curTime = TInstant::Now(); + static ui8 counter; + counter++; + const TString symbols[4] = {" -", " \\", " |", " /"}; + const TDuration delta = + TDuration::MicroSeconds(curTime.MicroSeconds() - LastSampledUpdateTime_.MicroSeconds()); + float tps = 0; + if (delta) { + tps = (pos - LastSampledUpdateValue_) / delta.SecondsFloat(); + } + auto msg = Sprintf((symbols[counter % 4] + ProgressString_).c_str(), pos, tps); + UpdateSz_ = Max(UpdateSz_, msg.size()); + msg.resize(UpdateSz_); + + Cerr << msg << "\r"; + fflush(stderr); + LastSampledUpdateTime_ = curTime; + LastSampledUpdateValue_ = pos; + } + + LastUpdateValue_ = pos; + if (CurUpdate_ == Subsampling_) { + CurUpdate_ = 0; + } else { + CurUpdate_++; + } + } + + TString ShiftText(const TString& in, size_t n) { + TString shift; + shift.resize(n); + TString msg = " " + in; + size_t pos = 0; + while(true) { + pos = msg.find('\n', pos); + if (pos != msg.npos) { + if (pos < msg.size() - 1) { + pos++; + msg.insert(pos, shift); + } else { + break; + } + } else { + break; + } + } + return msg; + } + const size_t Subsampling_; + size_t CurUpdate_ = 0; + TString ProgressString_; + size_t UpdateSz_ = 0; + TMaybe<size_t> LastUpdateValue_; + TInstant LastSampledUpdateTime_; + size_t LastSampledUpdateValue_; + TMutex Mtx_; +}; + +IProgressTracker::TPtr CreateStderrProgressTracker(size_t subsampling, const TString& title) { + return std::make_unique<TStderrProgressTracker>(subsampling, title); +} +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/idx_test_upload.cpp b/ydb/public/lib/idx_test/idx_test_upload.cpp index 97cc883641..7c7252e34d 100644 --- a/ydb/public/lib/idx_test/idx_test_upload.cpp +++ b/ydb/public/lib/idx_test/idx_test_upload.cpp @@ -1,152 +1,152 @@ -#include "idx_test.h" -#include "idx_test_common.h" -#include "idx_test_data_provider.h" - -#include <util/string/printf.h> -#include <util/system/mutex.h> +#include "idx_test.h" +#include "idx_test_common.h" +#include "idx_test_data_provider.h" + +#include <util/string/printf.h> +#include <util/system/mutex.h> #include <library/cpp/threading/future/future.h> - + #include <ydb/public/lib/yson_value/ydb_yson_value.h> - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace NIdxTest { - -class TUploader : public IUploader { -public: - TUploader(TTableClient&& client, const TString& table, const TUploaderParams& params) - : Client_(std::move(client)) - , Table_(table) - , Params_(params) - { - } - - void Run(IDataProvider::TPtr dataProvider) override { - CreateTable(dataProvider->GetTableSchema()); - CreateQuery(dataProvider->GetRowType()); - UploadData(dataProvider.get()); - } - -private: - enum class EUploadStatus { - InProgress, - Done - }; - - void CreateTable(TTableDescription tableDesc) { - ThrowOnError(Client_.RetryOperationSync([this, tableDesc](TSession session) { - TTableDescription desc(tableDesc); - return session.CreateTable( - Table_, - std::move(desc), - TCreateTableSettings() - .PartitioningPolicy( - TPartitioningPolicy() - .UniformPartitions(Params_.ShardsCount) - ) - .ClientTimeout(TDuration::Seconds(5))).GetValueSync(); - })); - } - - void CreateQuery(NYdb::TType rowType) { - auto rowsTypeString = NYdb::FormatType(rowType); - - Query_ = Sprintf("DECLARE %s AS \"List<%s>\"; REPLACE INTO [%s] SELECT * FROM AS_TABLE(%s);", - ParamName_.c_str(), - rowsTypeString.c_str(), - Table_.c_str(), - ParamName_.c_str()); - } - - TMaybe<NYdb::TParams> CreateParams(IDataProvider* dataProvider, ui32 id) { - const auto& mayBeItems = dataProvider->GetBatch(id); - if (!mayBeItems) { - return {}; - } - return ::NIdxTest::CreateParamsAsList(mayBeItems.GetRef(), ParamName_); - } - - std::function<NYdb::TAsyncStatus(TSession session)> CreateUploadFn(IDataProvider* dataProvider, ui32 id) { - return [this, dataProvider, id](TSession session) -> TAsyncStatus { - - if (FinishedByError) - return NThreading::MakeFuture<NYdb::TStatus>(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues())); - - auto maybeParamsList = CreateParams(dataProvider, id); - if (!maybeParamsList) { - UploadStatuses_[id] = EUploadStatus::Done; - return NThreading::MakeFuture<NYdb::TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); - } - - auto paramsList = maybeParamsList.GetRef(); - return session.PrepareDataQuery(Query_).Apply( - [paramsList](TAsyncPrepareQueryResult result) -> NYdb::TAsyncStatus { - auto prepareResult = result.ExtractValue(); - if (!prepareResult.IsSuccess()) - return NThreading::MakeFuture<NYdb::TStatus>(prepareResult); - - auto dataQuery = prepareResult.GetQuery(); - return dataQuery.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - paramsList).Apply( - [](TAsyncDataQueryResult asyncResult) { - auto result = asyncResult.ExtractValue(); - return NThreading::MakeFuture<NYdb::TStatus>(result); - }); - }); - }; - } - - TAsyncStatus RunUploadTask(IDataProvider* dataProvider, ui32 id, std::function<TAsyncStatus(TSession session)> uploadFn) { - return Client_.RetryOperation(uploadFn).Apply( - [dataProvider, id, uploadFn, this](TAsyncStatus asyncStatus) { - auto status = asyncStatus.GetValue(); - if (!status.IsSuccess() || UploadStatuses_[id] == EUploadStatus::Done) { - return NThreading::MakeFuture<TStatus>(status); - } - return RunUploadTask(dataProvider, id, uploadFn); - }); - } - - void UploadData(IDataProvider* dataProvider) { - for (ui32 shard = 0; shard < Params_.ShardsCount; shard++) { - UploadStatuses_.push_back(EUploadStatus::InProgress); - } - - TVector<TAsyncStatus> resultFutures; - - TMaybe<NYdb::TStatus> errStatus; - - auto checker = [&errStatus, this](NYdb::TAsyncStatus asyncStatus) mutable { - auto status = asyncStatus.ExtractValue(); - if (!status.IsSuccess()) { - FinishedByError = true; - errStatus = status; - } - return NThreading::MakeFuture(status); - }; - - for (ui32 shard = 0; shard < Params_.ShardsCount; shard++) { - auto uploadFn = CreateUploadFn(dataProvider, shard); - resultFutures.push_back(RunUploadTask(dataProvider, shard, uploadFn).Apply(checker)); - } - + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace NIdxTest { + +class TUploader : public IUploader { +public: + TUploader(TTableClient&& client, const TString& table, const TUploaderParams& params) + : Client_(std::move(client)) + , Table_(table) + , Params_(params) + { + } + + void Run(IDataProvider::TPtr dataProvider) override { + CreateTable(dataProvider->GetTableSchema()); + CreateQuery(dataProvider->GetRowType()); + UploadData(dataProvider.get()); + } + +private: + enum class EUploadStatus { + InProgress, + Done + }; + + void CreateTable(TTableDescription tableDesc) { + ThrowOnError(Client_.RetryOperationSync([this, tableDesc](TSession session) { + TTableDescription desc(tableDesc); + return session.CreateTable( + Table_, + std::move(desc), + TCreateTableSettings() + .PartitioningPolicy( + TPartitioningPolicy() + .UniformPartitions(Params_.ShardsCount) + ) + .ClientTimeout(TDuration::Seconds(5))).GetValueSync(); + })); + } + + void CreateQuery(NYdb::TType rowType) { + auto rowsTypeString = NYdb::FormatType(rowType); + + Query_ = Sprintf("DECLARE %s AS \"List<%s>\"; REPLACE INTO [%s] SELECT * FROM AS_TABLE(%s);", + ParamName_.c_str(), + rowsTypeString.c_str(), + Table_.c_str(), + ParamName_.c_str()); + } + + TMaybe<NYdb::TParams> CreateParams(IDataProvider* dataProvider, ui32 id) { + const auto& mayBeItems = dataProvider->GetBatch(id); + if (!mayBeItems) { + return {}; + } + return ::NIdxTest::CreateParamsAsList(mayBeItems.GetRef(), ParamName_); + } + + std::function<NYdb::TAsyncStatus(TSession session)> CreateUploadFn(IDataProvider* dataProvider, ui32 id) { + return [this, dataProvider, id](TSession session) -> TAsyncStatus { + + if (FinishedByError) + return NThreading::MakeFuture<NYdb::TStatus>(TStatus(EStatus::INTERNAL_ERROR, NYql::TIssues())); + + auto maybeParamsList = CreateParams(dataProvider, id); + if (!maybeParamsList) { + UploadStatuses_[id] = EUploadStatus::Done; + return NThreading::MakeFuture<NYdb::TStatus>(TStatus(EStatus::SUCCESS, NYql::TIssues())); + } + + auto paramsList = maybeParamsList.GetRef(); + return session.PrepareDataQuery(Query_).Apply( + [paramsList](TAsyncPrepareQueryResult result) -> NYdb::TAsyncStatus { + auto prepareResult = result.ExtractValue(); + if (!prepareResult.IsSuccess()) + return NThreading::MakeFuture<NYdb::TStatus>(prepareResult); + + auto dataQuery = prepareResult.GetQuery(); + return dataQuery.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + paramsList).Apply( + [](TAsyncDataQueryResult asyncResult) { + auto result = asyncResult.ExtractValue(); + return NThreading::MakeFuture<NYdb::TStatus>(result); + }); + }); + }; + } + + TAsyncStatus RunUploadTask(IDataProvider* dataProvider, ui32 id, std::function<TAsyncStatus(TSession session)> uploadFn) { + return Client_.RetryOperation(uploadFn).Apply( + [dataProvider, id, uploadFn, this](TAsyncStatus asyncStatus) { + auto status = asyncStatus.GetValue(); + if (!status.IsSuccess() || UploadStatuses_[id] == EUploadStatus::Done) { + return NThreading::MakeFuture<TStatus>(status); + } + return RunUploadTask(dataProvider, id, uploadFn); + }); + } + + void UploadData(IDataProvider* dataProvider) { + for (ui32 shard = 0; shard < Params_.ShardsCount; shard++) { + UploadStatuses_.push_back(EUploadStatus::InProgress); + } + + TVector<TAsyncStatus> resultFutures; + + TMaybe<NYdb::TStatus> errStatus; + + auto checker = [&errStatus, this](NYdb::TAsyncStatus asyncStatus) mutable { + auto status = asyncStatus.ExtractValue(); + if (!status.IsSuccess()) { + FinishedByError = true; + errStatus = status; + } + return NThreading::MakeFuture(status); + }; + + for (ui32 shard = 0; shard < Params_.ShardsCount; shard++) { + auto uploadFn = CreateUploadFn(dataProvider, shard); + resultFutures.push_back(RunUploadTask(dataProvider, shard, uploadFn).Apply(checker)); + } + NThreading::WaitExceptionOrAll(resultFutures).Wait(); - if (errStatus) - ThrowOnError(errStatus.GetRef()); - } - - TTableClient Client_; - const TString Table_; - const TUploaderParams Params_; - const TString ParamName_ = "$items"; - TString Query_; - bool FinishedByError = false; - TVector<EUploadStatus> UploadStatuses_; -}; - -IUploader::TPtr CreateUploader(TDriver& driver, const TString& table, const TUploaderParams& params) { - return std::make_unique<TUploader>(TTableClient(driver), table, params); -} - -} // namespace NIdxTest + if (errStatus) + ThrowOnError(errStatus.GetRef()); + } + + TTableClient Client_; + const TString Table_; + const TUploaderParams Params_; + const TString ParamName_ = "$items"; + TString Query_; + bool FinishedByError = false; + TVector<EUploadStatus> UploadStatuses_; +}; + +IUploader::TPtr CreateUploader(TDriver& driver, const TString& table, const TUploaderParams& params) { + return std::make_unique<TUploader>(TTableClient(driver), table, params); +} + +} // namespace NIdxTest diff --git a/ydb/public/lib/idx_test/ut/idx_test_data_provider_ut.cpp b/ydb/public/lib/idx_test/ut/idx_test_data_provider_ut.cpp index 17af5a06a7..2ffe6d636e 100644 --- a/ydb/public/lib/idx_test/ut/idx_test_data_provider_ut.cpp +++ b/ydb/public/lib/idx_test/ut/idx_test_data_provider_ut.cpp @@ -1,65 +1,65 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - + #include <ydb/public/lib/idx_test/idx_test_data_provider.h> - -using namespace NIdxTest; - -Y_UNIT_TEST_SUITE(IdxTestDataProvider) { - Y_UNIT_TEST(1ShardLimit6bitFromRandomUi8) { - TRandomValueProvider provider(0, 6); - ui64 val = Max<ui64>(); - while (val) { - auto random = provider.RandomUi8(); - UNIT_ASSERT(random <= 63); - val &= ~(1ull << random); - } - } - - Y_UNIT_TEST(1ShardLimit6bitFromRandomUi16) { - TRandomValueProvider provider(0, 6); - ui64 val = Max<ui64>(); - while (val) { - auto random = provider.RandomUi16(); - UNIT_ASSERT(random <= 63); - val &= ~(1ull << random); - } - } - - Y_UNIT_TEST(1ShardLimit6bitFromRandomUi32) { - TRandomValueProvider provider(0, 6); - ui64 val = Max<ui64>(); - while (val) { - auto random = provider.RandomUi32(); - UNIT_ASSERT(random <= 63); - val &= ~(1ull << random); - } - } - - Y_UNIT_TEST(1ShardLimit6bitFromRandomUi64) { - TRandomValueProvider provider(0, 6); - ui64 val = Max<ui64>(); - while (val) { - auto random = provider.RandomUi64(); - UNIT_ASSERT(random <= 63); - val &= ~(1ull << random); - } - } - - Y_UNIT_TEST(4ShardsLimit20bitFromRandomUi64) { - TRandomValueProvider provider(2, 20); - size_t count = 10000; - const auto shardRange = Max<ui64>() / 4 + 1; - const auto bound0 = 1ull << 20; - const auto bound1 = bound0 + shardRange; - const auto bound2 = bound1 + shardRange; - const auto bound3 = bound2 + shardRange; - while (count--) { - auto rnd = provider.RandomUi64(); - UNIT_ASSERT( (rnd >= 0 * shardRange && rnd < bound0) || - (rnd >= 1 * shardRange && rnd < bound1) || - (rnd >= 2 * shardRange && rnd < bound2) || - (rnd >= 3 * shardRange && rnd < bound3) ); - } - } -} + +using namespace NIdxTest; + +Y_UNIT_TEST_SUITE(IdxTestDataProvider) { + Y_UNIT_TEST(1ShardLimit6bitFromRandomUi8) { + TRandomValueProvider provider(0, 6); + ui64 val = Max<ui64>(); + while (val) { + auto random = provider.RandomUi8(); + UNIT_ASSERT(random <= 63); + val &= ~(1ull << random); + } + } + + Y_UNIT_TEST(1ShardLimit6bitFromRandomUi16) { + TRandomValueProvider provider(0, 6); + ui64 val = Max<ui64>(); + while (val) { + auto random = provider.RandomUi16(); + UNIT_ASSERT(random <= 63); + val &= ~(1ull << random); + } + } + + Y_UNIT_TEST(1ShardLimit6bitFromRandomUi32) { + TRandomValueProvider provider(0, 6); + ui64 val = Max<ui64>(); + while (val) { + auto random = provider.RandomUi32(); + UNIT_ASSERT(random <= 63); + val &= ~(1ull << random); + } + } + + Y_UNIT_TEST(1ShardLimit6bitFromRandomUi64) { + TRandomValueProvider provider(0, 6); + ui64 val = Max<ui64>(); + while (val) { + auto random = provider.RandomUi64(); + UNIT_ASSERT(random <= 63); + val &= ~(1ull << random); + } + } + + Y_UNIT_TEST(4ShardsLimit20bitFromRandomUi64) { + TRandomValueProvider provider(2, 20); + size_t count = 10000; + const auto shardRange = Max<ui64>() / 4 + 1; + const auto bound0 = 1ull << 20; + const auto bound1 = bound0 + shardRange; + const auto bound2 = bound1 + shardRange; + const auto bound3 = bound2 + shardRange; + while (count--) { + auto rnd = provider.RandomUi64(); + UNIT_ASSERT( (rnd >= 0 * shardRange && rnd < bound0) || + (rnd >= 1 * shardRange && rnd < bound1) || + (rnd >= 2 * shardRange && rnd < bound2) || + (rnd >= 3 * shardRange && rnd < bound3) ); + } + } +} diff --git a/ydb/public/lib/idx_test/ut/ya.make b/ydb/public/lib/idx_test/ut/ya.make index 7e5f8446a5..1f4711c2f5 100644 --- a/ydb/public/lib/idx_test/ut/ya.make +++ b/ydb/public/lib/idx_test/ut/ya.make @@ -1,22 +1,22 @@ UNITTEST_FOR(ydb/public/lib/idx_test) - -OWNER( - dcherednik - g:kikimr -) - -TIMEOUT(600) -SIZE(MEDIUM) - -FORK_SUBTESTS() - -PEERDIR( +OWNER( + dcherednik + g:kikimr +) + +TIMEOUT(600) + +SIZE(MEDIUM) + +FORK_SUBTESTS() + +PEERDIR( ydb/public/lib/idx_test -) - -SRCS( - idx_test_data_provider_ut.cpp -) - -END() +) + +SRCS( + idx_test_data_provider_ut.cpp +) + +END() diff --git a/ydb/public/lib/idx_test/ya.make b/ydb/public/lib/idx_test/ya.make index 3a578efb8a..6593eee071 100644 --- a/ydb/public/lib/idx_test/ya.make +++ b/ydb/public/lib/idx_test/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - idx_test_checker.cpp - idx_test_common.cpp - idx_test_data_provider.cpp - idx_test_stderr_progress_tracker.cpp - idx_test_loader.cpp - idx_test_upload.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + idx_test_checker.cpp + idx_test_common.cpp + idx_test_data_provider.cpp + idx_test_stderr_progress_tracker.cpp + idx_test_loader.cpp + idx_test_upload.cpp +) + +PEERDIR( library/cpp/string_utils/base64 ydb/public/sdk/cpp/client/ydb_table -) - -GENERATE_ENUM_SERIALIZATION(idx_test.h) - -END() +) + +GENERATE_ENUM_SERIALIZATION(idx_test.h) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/lib/json_value/ydb_json_value.cpp b/ydb/public/lib/json_value/ydb_json_value.cpp index b54567f4ab..af126501a7 100644 --- a/ydb/public/lib/json_value/ydb_json_value.cpp +++ b/ydb/public/lib/json_value/ydb_json_value.cpp @@ -12,7 +12,7 @@ namespace NYdb { class TUtf8Transcoder { public: - explicit TUtf8Transcoder() + explicit TUtf8Transcoder() { }; @@ -82,7 +82,7 @@ namespace NYdb { Buffer.push_back(((c & '\x03') << 6) | (str[i + 1] & '\x3F')); i += 1; } else { - ThrowFatalError("Unicode symbols with codes greater than 255 are not supported."); + ThrowFatalError("Unicode symbols with codes greater than 255 are not supported."); } } if (IsAscii) { @@ -109,7 +109,7 @@ namespace NYdb { class TYdbToJsonConverter { public: - TYdbToJsonConverter(TValueParser& parser, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) + TYdbToJsonConverter(TValueParser& parser, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) : Parser(parser) , Writer(writer) , Encoding(encoding) @@ -196,7 +196,7 @@ namespace NYdb { Writer.WriteString(Parser.GetDyNumber()); break; default: - ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); } } @@ -274,7 +274,7 @@ namespace NYdb { break; default: - ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << Parser.GetKind()); + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << Parser.GetKind()); } } @@ -289,7 +289,7 @@ namespace NYdb { str << Base64Encode(s); break; default: - ThrowFatalError(TStringBuilder() << "Unknown binary string encode mode: " + ThrowFatalError(TStringBuilder() << "Unknown binary string encode mode: " << static_cast<size_t>(Encoding)); } str << "\""; @@ -305,47 +305,47 @@ namespace NYdb { } void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, - EBinaryStringEncoding encoding) + EBinaryStringEncoding encoding) { TValueParser typeParser(value); - TYdbToJsonConverter converter(typeParser, writer, encoding); + TYdbToJsonConverter converter(typeParser, writer, encoding); converter.Convert(); } -TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding) +TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding) { TStringStream out; NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); - FormatValueJson(value, writer, encoding); + FormatValueJson(value, writer, encoding); return out.Str(); } void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, - EBinaryStringEncoding encoding) + EBinaryStringEncoding encoding) { writer.BeginObject(); for (ui32 i = 0; i < columns.size(); ++i) { writer.WriteKey(columns[i].Name); - TYdbToJsonConverter converter(parser.ColumnParser(i), writer, encoding); + TYdbToJsonConverter converter(parser.ColumnParser(i), writer, encoding); converter.Convert(); } writer.EndObject(); } TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, - EBinaryStringEncoding encoding) + EBinaryStringEncoding encoding) { TStringStream out; NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); - FormatResultRowJson(parser, columns, writer, encoding); + FormatResultRowJson(parser, columns, writer, encoding); return out.Str(); } -void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding) +void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding) { auto columns = result.GetColumnsMeta(); @@ -353,16 +353,16 @@ void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinarySt while (parser.TryNextRow()) { NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, out); - FormatResultRowJson(parser, columns, writer, encoding); + FormatResultRowJson(parser, columns, writer, encoding); *out << Endl; } } -TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding) +TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding) { TStringStream out; - FormatResultSetJson(result, &out, encoding); + FormatResultSetJson(result, &out, encoding); return out.Str(); } @@ -371,7 +371,7 @@ namespace { class TJsonToYdbConverter { public: TJsonToYdbConverter(TValueBuilder& valueBuilder, const NJson::TJsonValue& jsonValue, TTypeParser& typeParser, - EBinaryStringEncoding encoding) + EBinaryStringEncoding encoding) : ValueBuilder(valueBuilder) , JsonValue(jsonValue) , TypeParser(typeParser) @@ -395,7 +395,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_INTEGER); long long intValue = jsonValue.GetInteger(); if (intValue > std::numeric_limits<i8>::max() || intValue < std::numeric_limits<i8>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int8 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int8 type"); } ValueBuilder.Int8(intValue); break; @@ -405,7 +405,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_UINTEGER); unsigned long long intValue = jsonValue.GetUInteger(); if (intValue > std::numeric_limits<ui8>::max() || intValue < std::numeric_limits<ui8>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt8 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt8 type"); } ValueBuilder.Uint8(intValue); break; @@ -415,7 +415,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_INTEGER); long long intValue = jsonValue.GetInteger(); if (intValue > std::numeric_limits<i16>::max() || intValue < std::numeric_limits<i16>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int16 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int16 type"); } ValueBuilder.Int16(intValue); break; @@ -425,7 +425,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_UINTEGER); unsigned long long intValue = jsonValue.GetUInteger(); if (intValue > std::numeric_limits<ui16>::max() || intValue < std::numeric_limits<ui16>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt16 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt16 type"); } ValueBuilder.Uint16(intValue); break; @@ -435,7 +435,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_INTEGER); long long intValue = jsonValue.GetInteger(); if (intValue > std::numeric_limits<i32>::max() || intValue < std::numeric_limits<i32>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int32 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int32 type"); } ValueBuilder.Int32(intValue); break; @@ -445,7 +445,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_UINTEGER); unsigned long long intValue = jsonValue.GetUInteger(); if (intValue > std::numeric_limits<ui32>::max() || intValue < std::numeric_limits<ui32>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt32 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt32 type"); } ValueBuilder.Uint32(intValue); break; @@ -455,7 +455,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_INTEGER); long long intValue = jsonValue.GetInteger(); if (intValue > std::numeric_limits<i64>::max() || intValue < std::numeric_limits<i64>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int64 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int64 type"); } ValueBuilder.Int64(intValue); break; @@ -465,7 +465,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_UINTEGER); unsigned long long intValue = jsonValue.GetUInteger(); if (intValue > std::numeric_limits<ui64>::max() || intValue < std::numeric_limits<ui64>::min()) { - ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt64 type"); + ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt64 type"); } ValueBuilder.Uint64(intValue); break; @@ -483,7 +483,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_STRING); TInstant date; if (!TInstant::TryParseIso8601(jsonValue.GetString(), date)) { - ThrowFatalError(TStringBuilder() << "Can't parse date from string \"" << jsonValue.GetString() << "\""); + ThrowFatalError(TStringBuilder() << "Can't parse date from string \"" << jsonValue.GetString() << "\""); } ValueBuilder.Date(date); break; @@ -493,7 +493,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_STRING); TInstant dateTime; if (!TInstant::TryParseIso8601(jsonValue.GetString(), dateTime)) { - ThrowFatalError(TStringBuilder() << "Can't parse dateTime from string \"" << jsonValue.GetString() << "\""); + ThrowFatalError(TStringBuilder() << "Can't parse dateTime from string \"" << jsonValue.GetString() << "\""); } ValueBuilder.Datetime(dateTime); break; @@ -503,7 +503,7 @@ namespace { EnsureType(jsonValue, NJson::JSON_STRING); TInstant timestamp; if (!TInstant::TryParseIso8601(jsonValue.GetString(), timestamp)) { - ThrowFatalError(TStringBuilder() << "Can't parse timestamp from string \"" << jsonValue.GetString() << "\""); + ThrowFatalError(TStringBuilder() << "Can't parse timestamp from string \"" << jsonValue.GetString() << "\""); } ValueBuilder.Timestamp(timestamp); break; @@ -549,7 +549,7 @@ namespace { ValueBuilder.DyNumber(jsonValue.GetString()); break; default: - ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); } } @@ -608,7 +608,7 @@ namespace { typeBuilder.EndDict(); break; default: - ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); } } @@ -667,7 +667,7 @@ namespace { const TString& memberName = TypeParser.GetMemberName(); const auto it = jsonMap.find(memberName); if (it == jsonMap.end()) { - ThrowFatalError(TStringBuilder() << "No member \"" << memberName + ThrowFatalError(TStringBuilder() << "No member \"" << memberName << "\" in the map in json string for YDB struct type"); } ValueBuilder.AddMember(memberName); @@ -675,7 +675,7 @@ namespace { ++counter; } if (counter != jsonMap.size()) { - ThrowFatalError("Map in json string contains more members than YDB struct does"); + ThrowFatalError("Map in json string contains more members than YDB struct does"); } ValueBuilder.EndStruct(); @@ -690,13 +690,13 @@ namespace { for (const auto& element : jsonValue.GetArray()) { if (!TypeParser.TryNextElement()) { - ThrowFatalError("Tuple in json string should contain less elements than provided"); + ThrowFatalError("Tuple in json string should contain less elements than provided"); } ValueBuilder.AddElement(); ParseValue(element); } if (TypeParser.TryNextElement()) { - ThrowFatalError("Tuple in json string should contain more elements than provided"); + ThrowFatalError("Tuple in json string should contain more elements than provided"); } ValueBuilder.EndTuple(); @@ -713,7 +713,7 @@ namespace { EnsureType(keyValueElement, NJson::JSON_ARRAY); const auto& keyValueArray = keyValueElement.GetArray(); if (keyValueArray.size() != 2) { - ThrowFatalError("Each element of a dict type in YDB must be represented with " + ThrowFatalError("Each element of a dict type in YDB must be represented with " "exactly 2 elements in array in json string"); } auto it = keyValueArray.begin(); @@ -741,7 +741,7 @@ namespace { break; default: - ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); } } @@ -756,7 +756,7 @@ namespace { str << Base64Decode(s); break; default: - ThrowFatalError("Unknown binary string encode mode"); + ThrowFatalError("Unknown binary string encode mode"); break; } return str.Str(); @@ -771,7 +771,7 @@ namespace { TStringStream str; NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &str); writer.WriteJsonValue(&value); - ThrowFatalError(TStringBuilder() << "Wrong type for json value \"" << str.Str() + ThrowFatalError(TStringBuilder() << "Wrong type for json value \"" << str.Str() << "\". Expected type: " << type << ", received type: " << value.GetType() << ". "); } } @@ -785,26 +785,26 @@ namespace { }; } -TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding) { +TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding) { NJson::TJsonValue jsonValue; try { if (!NJson::ReadJsonTree(jsonString, &jsonValue, true)) { - ThrowFatalError(TStringBuilder() << "Can't parse string \"" << jsonString << "\" as json."); + ThrowFatalError(TStringBuilder() << "Can't parse string \"" << jsonString << "\" as json."); } } catch (std::exception& e) { - ThrowFatalError( + ThrowFatalError( TStringBuilder() << "Exception while parsing string \"" << jsonString << "\" as json: " << e.what()); } - return JsonToYdbValue(jsonValue, type, encoding); + return JsonToYdbValue(jsonValue, type, encoding); } -TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding) { +TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding) { TValueBuilder builder; TTypeParser typeParser(type); - TJsonToYdbConverter converter(builder, jsonValue, typeParser, encoding); + TJsonToYdbConverter converter(builder, jsonValue, typeParser, encoding); converter.Convert(); return builder.Build(); diff --git a/ydb/public/lib/json_value/ydb_json_value.h b/ydb/public/lib/json_value/ydb_json_value.h index b16f0b50b2..24fa92d02e 100644 --- a/ydb/public/lib/json_value/ydb_json_value.h +++ b/ydb/public/lib/json_value/ydb_json_value.h @@ -21,22 +21,22 @@ enum class EBinaryStringEncoding { }; // ====== YDB to json ====== -void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); +void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); -TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding); +TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding); void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, - EBinaryStringEncoding encoding); + EBinaryStringEncoding encoding); TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, - EBinaryStringEncoding encoding); + EBinaryStringEncoding encoding); -void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding); +void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding); -TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding); +TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding); // ====== json to YDB ====== -TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding); -TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding); +TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding); +TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding); } // namespace NYdb diff --git a/ydb/public/lib/operation_id/operation_id.cpp b/ydb/public/lib/operation_id/operation_id.cpp index 7cda80edb1..725da690f4 100644 --- a/ydb/public/lib/operation_id/operation_id.cpp +++ b/ydb/public/lib/operation_id/operation_id.cpp @@ -1,35 +1,35 @@ -#include "operation_id.h" - +#include "operation_id.h" + #include <google/protobuf/message.h> #include <library/cpp/cgiparam/cgiparam.h> #include <library/cpp/uri/uri.h> -#include <util/generic/yexception.h> -#include <util/string/cast.h> - -namespace NKikimr { +#include <util/generic/yexception.h> +#include <util/string/cast.h> + +namespace NKikimr { namespace NOperationId { - -using namespace NUri; - + +using namespace NUri; + TString ProtoToString(const Ydb::TOperationId& proto) { - using namespace ::google::protobuf; - const Reflection& reflection = *proto.GetReflection(); - std::vector<const FieldDescriptor*> fields; - reflection.ListFields(proto, &fields); - TStringStream res; - switch (proto.GetKind()) { + using namespace ::google::protobuf; + const Reflection& reflection = *proto.GetReflection(); + std::vector<const FieldDescriptor*> fields; + reflection.ListFields(proto, &fields); + TStringStream res; + switch (proto.GetKind()) { case Ydb::TOperationId::OPERATION_DDL: case Ydb::TOperationId::OPERATION_DML: - res << "ydb://operation"; - break; + res << "ydb://operation"; + break; case Ydb::TOperationId::SESSION_YQL: - res << "ydb://session"; - break; + res << "ydb://session"; + break; case Ydb::TOperationId::PREPARED_QUERY_ID: - res << "ydb://preparedqueryid"; - break; + res << "ydb://preparedqueryid"; + break; case Ydb::TOperationId::CMS_REQUEST: res << "ydb://cmsrequest"; break; @@ -39,50 +39,50 @@ TString ProtoToString(const Ydb::TOperationId& proto) { case Ydb::TOperationId::IMPORT: res << "ydb://import"; break; - case Ydb::TOperationId::BUILD_INDEX: - res << "ydb://buildindex"; - break; - default: - Y_VERIFY(false, "unexpected kind"); - } - // According to protobuf documentation: - // Fields (both normal fields and extension fields) will be listed ordered by field number, - // so we can rely on it to build url string - for (const FieldDescriptor* field : fields) { - Y_ASSERT(field != nullptr); - if (field) { - if (field->is_repeated()) { - int size = reflection.FieldSize(proto, field); - if (size) { - res << "?"; - } - for (int i = 0; i < size; i++) { - const auto& message = reflection.GetRepeatedMessage(proto, field, i); + case Ydb::TOperationId::BUILD_INDEX: + res << "ydb://buildindex"; + break; + default: + Y_VERIFY(false, "unexpected kind"); + } + // According to protobuf documentation: + // Fields (both normal fields and extension fields) will be listed ordered by field number, + // so we can rely on it to build url string + for (const FieldDescriptor* field : fields) { + Y_ASSERT(field != nullptr); + if (field) { + if (field->is_repeated()) { + int size = reflection.FieldSize(proto, field); + if (size) { + res << "?"; + } + for (int i = 0; i < size; i++) { + const auto& message = reflection.GetRepeatedMessage(proto, field, i); const auto& data = dynamic_cast<const Ydb::TOperationId::TData&>(message); - TUri::ReEncode(res, data.GetKey()); - res << "="; - TUri::ReEncode(res, data.GetValue()); - if (i < size - 1) { - res << "&"; - } - } - } else { - res << "/"; - const FieldDescriptor::CppType type = field->cpp_type(); - switch (type) { - case FieldDescriptor::CPPTYPE_ENUM: - res << reflection.GetEnumValue(proto, field); - break; - default: - Y_VERIFY(false, "unexpected protobuf field type"); - break; - } - } - } - } - return res.Str(); -} - + TUri::ReEncode(res, data.GetKey()); + res << "="; + TUri::ReEncode(res, data.GetValue()); + if (i < size - 1) { + res << "&"; + } + } + } else { + res << "/"; + const FieldDescriptor::CppType type = field->cpp_type(); + switch (type) { + case FieldDescriptor::CPPTYPE_ENUM: + res << reflection.GetEnumValue(proto, field); + break; + default: + Y_VERIFY(false, "unexpected protobuf field type"); + break; + } + } + } + } + return res.Str(); +} + TOperationId::TOperationId() { SetKind(Ydb::TOperationId::UNUSED); } @@ -93,49 +93,49 @@ TOperationId::TOperationId(const TString &string, bool allowEmpty) { return; } - TUri uri; - TState::EParsed er = uri.Parse(string, TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible); - if (er != TState::ParsedOK) { - ythrow yexception() << "Unable to parse input string"; - } - - const TString& path = uri.PrintS(TField::FlagPath).substr(1); // start from 1 to remove first '/' - if (path.length() < 1) { - ythrow yexception() << "Invalid path length"; - } - + TUri uri; + TState::EParsed er = uri.Parse(string, TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible); + if (er != TState::ParsedOK) { + ythrow yexception() << "Unable to parse input string"; + } + + const TString& path = uri.PrintS(TField::FlagPath).substr(1); // start from 1 to remove first '/' + if (path.length() < 1) { + ythrow yexception() << "Invalid path length"; + } + int kind; - if (!TryFromString(path, kind)) { - ythrow yexception() << "Unable to cast \"kind\" field: " << path; - } - + if (!TryFromString(path, kind)) { + ythrow yexception() << "Unable to cast \"kind\" field: " << path; + } + if (!EKind_IsValid(kind)) { ythrow yexception() << "Invalid operation kind: " << kind; } SetKind(static_cast<Ydb::TOperationId::EKind>(kind)); - - const TString& query = uri.PrintS(TField::FlagQuery); - - if (query) { - TCgiParameters params(query.substr(1)); // start from 1 to remove first '?' - for (auto it : params) { - auto data = AddData(); - data->SetKey(it.first); - data->SetValue(it.second); - Index_[it.first].push_back(&data->GetValue()); - } - } -} - -const TVector<const TString*>& TOperationId::GetValue(const TString &key) const { - auto it = Index_.find(key); - if (it != Index_.end()) { - return it->second; - } - ythrow yexception() << "Unable to find key: " << key; -} - + + const TString& query = uri.PrintS(TField::FlagQuery); + + if (query) { + TCgiParameters params(query.substr(1)); // start from 1 to remove first '?' + for (auto it : params) { + auto data = AddData(); + data->SetKey(it.first); + data->SetValue(it.second); + Index_[it.first].push_back(&data->GetValue()); + } + } +} + +const TVector<const TString*>& TOperationId::GetValue(const TString &key) const { + auto it = Index_.find(key); + if (it != Index_.end()) { + return it->second; + } + ythrow yexception() << "Unable to find key: " << key; +} + TString TOperationId::GetSubKind() const { auto it = Index_.find("kind"); if (it == Index_.end()) { @@ -150,11 +150,11 @@ TString TOperationId::GetSubKind() const { } void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value) { - auto data = proto.AddData(); - data->SetKey(key); - data->SetValue(value); -} - + auto data = proto.AddData(); + data->SetKey(key); + data->SetValue(value); +} + Ydb::TOperationId::EKind ParseKind(const TStringBuf value) { if (value.StartsWith("export")) { return Ydb::TOperationId::EXPORT; @@ -164,12 +164,12 @@ Ydb::TOperationId::EKind ParseKind(const TStringBuf value) { return Ydb::TOperationId::IMPORT; } - if (value.StartsWith("buildindex")) { - return Ydb::TOperationId::BUILD_INDEX; - } - + if (value.StartsWith("buildindex")) { + return Ydb::TOperationId::BUILD_INDEX; + } + return Ydb::TOperationId::UNUSED; } } // namespace NOperationId -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/public/lib/operation_id/operation_id.h b/ydb/public/lib/operation_id/operation_id.h index 3604659984..23f91140d1 100644 --- a/ydb/public/lib/operation_id/operation_id.h +++ b/ydb/public/lib/operation_id/operation_id.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/operation_id/protos/operation_id.pb.h> - -#include <util/generic/hash.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NKikimr { + +#include <util/generic/hash.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NKikimr { namespace NOperationId { - + class TOperationId : public Ydb::TOperationId { -public: +public: TOperationId(); explicit TOperationId(const TString& string, bool allowEmpty = false); - const TVector<const TString*>& GetValue(const TString& key) const; + const TVector<const TString*>& GetValue(const TString& key) const; TString GetSubKind() const; -private: - THashMap<TString, TVector<const TString*>> Index_; -}; - +private: + THashMap<TString, TVector<const TString*>> Index_; +}; + TString ProtoToString(const Ydb::TOperationId& proto); void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value); void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const char* value, size_t size); Ydb::TOperationId::EKind ParseKind(const TStringBuf value); - + } // namespace NOperationId -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/public/lib/operation_id/operation_id_ut.cpp b/ydb/public/lib/operation_id/operation_id_ut.cpp index 1f6f80b754..64fbedce4b 100644 --- a/ydb/public/lib/operation_id/operation_id_ut.cpp +++ b/ydb/public/lib/operation_id/operation_id_ut.cpp @@ -1,52 +1,52 @@ -#include "operation_id.h" - +#include "operation_id.h" + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { + +namespace NKikimr { namespace NOperationId { - + Y_UNIT_TEST_SUITE(OperationIdTest) { Y_UNIT_TEST(ConvertKindOnly) { Ydb::TOperationId proto; proto.SetKind(Ydb::TOperationId::OPERATION_DDL); - auto str = ProtoToString(proto); - UNIT_ASSERT_EQUAL(str, "ydb://operation/1"); - auto newProto = TOperationId(str); - UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); - UNIT_ASSERT_EQUAL(newProto.DataSize(), 0); - } - + auto str = ProtoToString(proto); + UNIT_ASSERT_EQUAL(str, "ydb://operation/1"); + auto newProto = TOperationId(str); + UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); + UNIT_ASSERT_EQUAL(newProto.DataSize(), 0); + } + Y_UNIT_TEST(ConvertKindAndValues) { Ydb::TOperationId proto; proto.SetKind(Ydb::TOperationId::OPERATION_DDL); - { - auto data = proto.AddData(); - data->SetKey("key1"); - data->SetValue("value1"); - } - { - auto data = proto.AddData(); - data->SetKey("txId"); - data->SetValue("42"); - } - auto str = ProtoToString(proto); - UNIT_ASSERT_EQUAL(str, "ydb://operation/1?key1=value1&txId=42"); - auto newProto = TOperationId(str); - UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); - UNIT_ASSERT_EQUAL(newProto.DataSize(), 2); - { - auto data = newProto.GetData(0); - UNIT_ASSERT_EQUAL(data.GetKey(), "key1"); - UNIT_ASSERT_EQUAL(data.GetValue(), "value1"); - } - { - auto data = newProto.GetData(1); - UNIT_ASSERT_EQUAL(data.GetKey(), "txId"); - UNIT_ASSERT_EQUAL(data.GetValue(), "42"); - } - } -} - + { + auto data = proto.AddData(); + data->SetKey("key1"); + data->SetValue("value1"); + } + { + auto data = proto.AddData(); + data->SetKey("txId"); + data->SetValue("42"); + } + auto str = ProtoToString(proto); + UNIT_ASSERT_EQUAL(str, "ydb://operation/1?key1=value1&txId=42"); + auto newProto = TOperationId(str); + UNIT_ASSERT_EQUAL(newProto.GetKind(), proto.GetKind()); + UNIT_ASSERT_EQUAL(newProto.DataSize(), 2); + { + auto data = newProto.GetData(0); + UNIT_ASSERT_EQUAL(data.GetKey(), "key1"); + UNIT_ASSERT_EQUAL(data.GetValue(), "value1"); + } + { + auto data = newProto.GetData(1); + UNIT_ASSERT_EQUAL(data.GetKey(), "txId"); + UNIT_ASSERT_EQUAL(data.GetValue(), "42"); + } + } +} + } // namespace NOperationId -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/public/lib/operation_id/protos/operation_id.proto b/ydb/public/lib/operation_id/protos/operation_id.proto index fa0a286e34..b6534e794b 100644 --- a/ydb/public/lib/operation_id/protos/operation_id.proto +++ b/ydb/public/lib/operation_id/protos/operation_id.proto @@ -1,25 +1,25 @@ -syntax = "proto3"; - +syntax = "proto3"; + package Ydb; - -message TOperationId { - enum EKind { - UNUSED = 0; - OPERATION_DDL = 1; - OPERATION_DML = 2; - SESSION_YQL = 3; - PREPARED_QUERY_ID = 4; + +message TOperationId { + enum EKind { + UNUSED = 0; + OPERATION_DDL = 1; + OPERATION_DML = 2; + SESSION_YQL = 3; + PREPARED_QUERY_ID = 4; CMS_REQUEST = 5; EXPORT = 6; - BUILD_INDEX = 7; + BUILD_INDEX = 7; IMPORT = 8; - } + } - message TData { - string Key = 1; - string Value = 2; - } + message TData { + string Key = 1; + string Value = 2; + } - EKind Kind = 1; - repeated TData Data = 3; -} + EKind Kind = 1; + repeated TData Data = 3; +} diff --git a/ydb/public/lib/value/value.cpp b/ydb/public/lib/value/value.cpp index 380bfd991c..0ed8b82689 100644 --- a/ydb/public/lib/value/value.cpp +++ b/ydb/public/lib/value/value.cpp @@ -207,20 +207,20 @@ TWriteValue& TWriteValue::Yson(const TString& value) { return *this; } -TWriteValue& TWriteValue::Json(const char* value) { - Type.SetKind(NKikimrMiniKQL::ETypeKind::Data); - Type.MutableData()->SetScheme(NScheme::NTypeIds::Json); - Value.SetText(value); - return *this; -} - -TWriteValue& TWriteValue::Json(const TString& value) { - Type.SetKind(NKikimrMiniKQL::ETypeKind::Data); - Type.MutableData()->SetScheme(NScheme::NTypeIds::Json); - Value.SetText(value); - return *this; -} - +TWriteValue& TWriteValue::Json(const char* value) { + Type.SetKind(NKikimrMiniKQL::ETypeKind::Data); + Type.MutableData()->SetScheme(NScheme::NTypeIds::Json); + Value.SetText(value); + return *this; +} + +TWriteValue& TWriteValue::Json(const TString& value) { + Type.SetKind(NKikimrMiniKQL::ETypeKind::Data); + Type.MutableData()->SetScheme(NScheme::NTypeIds::Json); + Value.SetText(value); + return *this; +} + TWriteValue& TWriteValue::Date(ui16 value) { Type.SetKind(NKikimrMiniKQL::ETypeKind::Data); Type.MutableData()->SetScheme(NScheme::NTypeIds::Date); @@ -396,7 +396,7 @@ TString TValue::GetDataText() const { case NScheme::NTypeIds::Float: return ToString(Value.GetFloat()); case NScheme::NTypeIds::Utf8: - case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::Json: return Value.GetText(); case NScheme::NTypeIds::String: case NScheme::NTypeIds::String4k: @@ -945,12 +945,12 @@ TValue::operator TString() const { || Type.GetData().GetScheme() == NScheme::NTypeIds::String4k || Type.GetData().GetScheme() == NScheme::NTypeIds::String2m || Type.GetData().GetScheme() == NScheme::NTypeIds::Yson - || Type.GetData().GetScheme() == NScheme::NTypeIds::Json + || Type.GetData().GetScheme() == NScheme::NTypeIds::Json ); Y_ASSERT(Value.HasText() || Value.HasBytes()); - return (Type.GetData().GetScheme() == NScheme::NTypeIds::Utf8 - || Type.GetData().GetScheme() == NScheme::NTypeIds::Json) - ? Value.GetText() : Value.GetBytes(); + return (Type.GetData().GetScheme() == NScheme::NTypeIds::Utf8 + || Type.GetData().GetScheme() == NScheme::NTypeIds::Json) + ? Value.GetText() : Value.GetBytes(); } diff --git a/ydb/public/lib/ya.make b/ydb/public/lib/ya.make index adca5c0b07..a4f529b44b 100644 --- a/ydb/public/lib/ya.make +++ b/ydb/public/lib/ya.make @@ -4,7 +4,7 @@ RECURSE( base deprecated experimental - idx_test + idx_test json_value jwt operation_id diff --git a/ydb/public/lib/ydb_cli/commands/ydb_common.h b/ydb/public/lib/ydb_cli/commands/ydb_common.h index 65f85152ca..7561c6476e 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_common.h @@ -38,8 +38,8 @@ inline void ThrowOnError(NYdb::TStatus status) { } inline void ThrowOnError(const NYdb::TOperation& operation) { - if (!operation.Ready()) - return; + if (!operation.Ready()) + return; ThrowOnError(operation.Status()); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp index 12dfadc63d..45b694cf94 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp @@ -98,10 +98,10 @@ int TCommandGetOperation::Run(TConfig& config) { } else { throw TMissUseException() << "Invalid operation ID (unexpected sub-kind of operation)"; } - case Ydb::TOperationId::BUILD_INDEX: + case Ydb::TOperationId::BUILD_INDEX: return GetOperation<NTable::TBuildIndexOperation>(client, OperationId, OutputFormat); default: - throw TMissUseException() << "Invalid operation ID (unexpected kind of operation)"; + throw TMissUseException() << "Invalid operation ID (unexpected kind of operation)"; } return EXIT_SUCCESS; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index bd1869aa62..a6f81fe432 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -344,29 +344,29 @@ namespace { return; } Cout << Endl << "Indexes: " << Endl; - for (const auto& index : indexes) { - Cout << index.GetIndexName() << " [" << index.GetIndexType() << "] Index columns: ("; - const auto& columns = index.GetIndexColumns(); + for (const auto& index : indexes) { + Cout << index.GetIndexName() << " [" << index.GetIndexType() << "] Index columns: ("; + const auto& columns = index.GetIndexColumns(); for (auto colIt = columns.begin(); colIt != columns.end();) { Cout << (*colIt); if (++colIt != columns.end()) { Cout << ","; } } - - const auto& cover = index.GetDataColumns(); - if (!cover) { - Cout << ")" << Endl; - } else { - Cout << ") Cover columns: ("; - for (auto colIt = cover.begin(); colIt != cover.end();) { - Cout << (*colIt); - if (++colIt != cover.end()) { - Cout << ","; - } - } - Cout << ")" << Endl; - } + + const auto& cover = index.GetDataColumns(); + if (!cover) { + Cout << ")" << Endl; + } else { + Cout << ") Cover columns: ("; + for (auto colIt = cover.begin(); colIt != cover.end();) { + Cout << (*colIt); + if (++colIt != cover.end()) { + Cout << ","; + } + } + Cout << ")" << Endl; + } } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index 9d4932c05b..4a25c372ce 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -27,10 +27,10 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); - config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); + config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); AddParametersOption(config); - + AddInputFormats(config, { EOutputFormat::JsonUnicode, EOutputFormat::JsonBase64 diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 3a3818d590..2f94b2fc03 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -37,13 +37,13 @@ TCommandQuery::TCommandQuery() AddCommand(std::make_unique<TCommandExplain>()); } -TCommandIndex::TCommandIndex() - : TClientCommandTree("index", {}, "Index operations") -{ +TCommandIndex::TCommandIndex() + : TClientCommandTree("index", {}, "Index operations") +{ AddCommand(std::make_unique<TCommandIndexAdd>()); AddCommand(std::make_unique<TCommandIndexDrop>()); -} - +} + TCommandAttribute::TCommandAttribute() : TClientCommandTree("attribute", {"attr"}, "Attribute operations") { @@ -58,13 +58,13 @@ TCommandTtl::TCommandTtl() AddCommand(std::make_unique<TCommandTtlDrop>()); } -TCommandIndexAdd::TCommandIndexAdd() - : TClientCommandTree("add", {}, "Add index in to the specified table") -{ +TCommandIndexAdd::TCommandIndexAdd() + : TClientCommandTree("add", {}, "Add index in to the specified table") +{ AddCommand(std::make_unique<TCommandIndexAddGlobalSync>()); AddCommand(std::make_unique<TCommandIndexAddGlobalAsync>()); -} - +} + TTableCommand::TTableCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) : TYdbOperationCommand(name, aliases, description) {} @@ -845,47 +845,47 @@ TCommandIndexAddGlobal::TCommandIndexAddGlobal( const TString& description) : TYdbCommand(name, aliases, description) , IndexType(type) -{} - -void TCommandIndexAddGlobal::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("index-name", "Name of index to add.") - .RequiredArgument("NAME").StoreResult(&IndexName); - config.Opts->AddLongOption("columns", "Ordered comma separated list of columns to build index for") - .RequiredArgument("CSV").StoreResult(&Columns); - config.Opts->AddLongOption("cover", "Ordered comma separated list of cover columns. (Data for those columns will be duplicated to index)") - .RequiredArgument("CSV").StoreResult(&DataColumns); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); +{} + +void TCommandIndexAddGlobal::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("index-name", "Name of index to add.") + .RequiredArgument("NAME").StoreResult(&IndexName); + config.Opts->AddLongOption("columns", "Ordered comma separated list of columns to build index for") + .RequiredArgument("CSV").StoreResult(&Columns); + config.Opts->AddLongOption("cover", "Ordered comma separated list of cover columns. (Data for those columns will be duplicated to index)") + .RequiredArgument("CSV").StoreResult(&DataColumns); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); } - -void TCommandIndexAddGlobal::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParseFormats(); - ParsePath(config, 0); + +void TCommandIndexAddGlobal::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParseFormats(); + ParsePath(config, 0); +} + +int TCommandIndexAddGlobal::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + auto columns = StringSplitter(Columns).Split(',').ToList<TString>(); + TVector<TString> dataColumns; + if (DataColumns) { + dataColumns = StringSplitter(DataColumns).Split(',').ToList<TString>(); + } + + auto settings = NTable::TAlterTableSettings() + .AppendAddIndexes({NTable::TIndexDescription(IndexName, IndexType, columns, dataColumns)}); + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto opResult = session.GetSession().AlterTableLong(Path, settings).GetValueSync(); + ThrowOnError(opResult); + PrintOperation(opResult, OutputFormat); + + return EXIT_SUCCESS; } -int TCommandIndexAddGlobal::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - auto columns = StringSplitter(Columns).Split(',').ToList<TString>(); - TVector<TString> dataColumns; - if (DataColumns) { - dataColumns = StringSplitter(DataColumns).Split(',').ToList<TString>(); - } - - auto settings = NTable::TAlterTableSettings() - .AppendAddIndexes({NTable::TIndexDescription(IndexName, IndexType, columns, dataColumns)}); - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto opResult = session.GetSession().AlterTableLong(Path, settings).GetValueSync(); - ThrowOnError(opResult); - PrintOperation(opResult, OutputFormat); - - return EXIT_SUCCESS; -} - TCommandIndexAddGlobalSync::TCommandIndexAddGlobalSync() : TCommandIndexAddGlobal(NTable::EIndexType::GlobalSync, "global-sync", {"global"}, "Add global sync index. The command returns operation") {} @@ -894,38 +894,38 @@ TCommandIndexAddGlobalAsync::TCommandIndexAddGlobalAsync() : TCommandIndexAddGlobal(NTable::EIndexType::GlobalAsync, "global-async", {}, "Add global async index. The command returns operation") {} -TCommandIndexDrop::TCommandIndexDrop() - : TYdbCommand("drop", {}, "Drop index from the specified table") -{} - -void TCommandIndexDrop::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("index-name", "Name of index to drop.") - .RequiredArgument("NAME").StoreResult(&IndexName); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); -} - -void TCommandIndexDrop::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandIndexDrop::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - auto settings = NTable::TAlterTableSettings() - .AppendDropIndexes({IndexName}); - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); - - return EXIT_SUCCESS; -} - +TCommandIndexDrop::TCommandIndexDrop() + : TYdbCommand("drop", {}, "Drop index from the specified table") +{} + +void TCommandIndexDrop::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("index-name", "Name of index to drop.") + .RequiredArgument("NAME").StoreResult(&IndexName); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); +} + +void TCommandIndexDrop::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandIndexDrop::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + auto settings = NTable::TAlterTableSettings() + .AppendDropIndexes({IndexName}); + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); + ThrowOnError(result); + + return EXIT_SUCCESS; +} + TCommandAttributeAdd::TCommandAttributeAdd() : TYdbCommand("add", {}, "Add attributes to the specified table") {} @@ -940,13 +940,13 @@ void TCommandAttributeAdd::Config(TConfig& config) { config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<table path>", "Path to a table"); -} +} void TCommandAttributeAdd::Parse(TConfig& config) { TClientCommand::Parse(config); ParsePath(config, 0); -} - +} + int TCommandAttributeAdd::Run(TConfig& config) { NTable::TTableClient client(CreateDriver(config)); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index d4cb4d087e..5cd2514aa0 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -23,11 +23,11 @@ public: TCommandQuery(); }; -class TCommandIndex : public TClientCommandTree { -public: - TCommandIndex(); -}; - +class TCommandIndex : public TClientCommandTree { +public: + TCommandIndex(); +}; + class TCommandAttribute : public TClientCommandTree { public: TCommandAttribute(); @@ -151,30 +151,30 @@ private: TString To; }; -class TCommandIndexAdd : public TClientCommandTree { -public: - TCommandIndexAdd(); -}; - -class TCommandIndexAddGlobal : public TYdbCommand, public TCommandWithPath, public TCommandWithFormat { -public: +class TCommandIndexAdd : public TClientCommandTree { +public: + TCommandIndexAdd(); +}; + +class TCommandIndexAddGlobal : public TYdbCommand, public TCommandWithPath, public TCommandWithFormat { +public: TCommandIndexAddGlobal( NTable::EIndexType type, const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString() ); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; protected: - TString IndexName; - TString Columns; - TString DataColumns; + TString IndexName; + TString Columns; + TString DataColumns; private: const NTable::EIndexType IndexType; -}; - +}; + class TCommandIndexAddGlobalSync : public TCommandIndexAddGlobal { public: TCommandIndexAddGlobalSync(); @@ -185,16 +185,16 @@ public: TCommandIndexAddGlobalAsync(); }; -class TCommandIndexDrop : public TYdbCommand, public TCommandWithPath { -public: - TCommandIndexDrop(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -private: - TString IndexName; -}; - +class TCommandIndexDrop : public TYdbCommand, public TCommandWithPath { +public: + TCommandIndexDrop(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +private: + TString IndexName; +}; + class TCommandAttributeAdd : public TYdbCommand, public TCommandWithPath { public: TCommandAttributeAdd(); diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 65b360a31c..7184ab41d1 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -34,29 +34,29 @@ namespace { }; } -void TCommandWithResponseHeaders::PrintResponseHeader(const TStatus& status) { - if (!ShowHeaders) - return; - - PrintResponseHeaderPretty(status); -} - -void TCommandWithResponseHeaders::PrintResponseHeaderPretty(const TStatus& status) { - const auto columnNames = TVector<TString>{"meta key", "meta value"}; - TPrettyTable table(columnNames); - - const auto headers = status.GetResponseMetadata(); - for (const auto& h : headers) { - auto& row = table.AddRow(); - row.Column(0, h.first); - row.Column(1, h.second); - } - - Cout << table; -} - -const TString TCommandWithResponseHeaders::ResponseHeadersHelp = "Show response metadata for ydb call"; - +void TCommandWithResponseHeaders::PrintResponseHeader(const TStatus& status) { + if (!ShowHeaders) + return; + + PrintResponseHeaderPretty(status); +} + +void TCommandWithResponseHeaders::PrintResponseHeaderPretty(const TStatus& status) { + const auto columnNames = TVector<TString>{"meta key", "meta value"}; + TPrettyTable table(columnNames); + + const auto headers = status.GetResponseMetadata(); + for (const auto& h : headers) { + auto& row = table.AddRow(); + row.Column(0, h.first); + row.Column(1, h.second); + } + + Cout << table; +} + +const TString TCommandWithResponseHeaders::ResponseHeadersHelp = "Show response metadata for ydb call"; + // Deprecated void TCommandWithFormat::AddJsonOption(TClientCommand::TConfig& config, const TString& description) { config.Opts->AddLongOption("json", description).NoArgument() diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index 3c3b0a9573..2d8d6a2e38 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -10,15 +10,15 @@ namespace NYdb { namespace NConsoleClient { -class TCommandWithResponseHeaders { -protected: - void PrintResponseHeader(const NYdb::TStatus& status); - void PrintResponseHeaderPretty(const NYdb::TStatus& status); - - bool ShowHeaders = false; - static const TString ResponseHeadersHelp; -}; - +class TCommandWithResponseHeaders { +protected: + void PrintResponseHeader(const NYdb::TStatus& status); + void PrintResponseHeaderPretty(const NYdb::TStatus& status); + + bool ShowHeaders = false; + static const TString ResponseHeadersHelp; +}; + class TCommandWithFormat { protected: void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); diff --git a/ydb/public/lib/ydb_cli/common/print_operation.cpp b/ydb/public/lib/ydb_cli/common/print_operation.cpp index afabcb0334..28a84375dc 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.cpp +++ b/ydb/public/lib/ydb_cli/common/print_operation.cpp @@ -14,32 +14,32 @@ namespace { using namespace NKikimr::NOperationId; - /// Common - TPrettyTable MakeTable(const TOperation&) { - return TPrettyTable({"id", "ready", "status"}); - } - - void PrettyPrint(const TOperation& operation, TPrettyTable& table) { - const auto& status = operation.Status(); - - auto& row = table.AddRow(); - row - .Column(0, ProtoToString(operation.Id())) - .Column(1, operation.Ready() ? "true" : "false") - .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())); - - TStringBuilder freeText; - - if (!status.GetIssues().Empty()) { - freeText << "Issues: " << Endl; - for (const auto& issue : status.GetIssues()) { - freeText << " - " << issue << Endl; - } - } - - row.FreeText(freeText); - } - + /// Common + TPrettyTable MakeTable(const TOperation&) { + return TPrettyTable({"id", "ready", "status"}); + } + + void PrettyPrint(const TOperation& operation, TPrettyTable& table) { + const auto& status = operation.Status(); + + auto& row = table.AddRow(); + row + .Column(0, ProtoToString(operation.Id())) + .Column(1, operation.Ready() ? "true" : "false") + .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())); + + TStringBuilder freeText; + + if (!status.GetIssues().Empty()) { + freeText << "Issues: " << Endl; + for (const auto& issue : status.GetIssues()) { + freeText << " - " << issue << Endl; + } + } + + row.FreeText(freeText); + } + template <typename EProgress, typename TMetadata> TString PrintProgress(const TMetadata& metadata) { TStringBuilder result; @@ -187,36 +187,36 @@ namespace { } /// Index build - TPrettyTable MakeTable(const NYdb::NTable::TBuildIndexOperation&) { - return TPrettyTable({"id", "ready", "status", "state", "progress", "table", "index"}); - } - - void PrettyPrint(const NYdb::NTable::TBuildIndexOperation& operation, TPrettyTable& table) { - const auto& status = operation.Status(); - const auto& metadata = operation.Metadata(); - - auto& row = table.AddRow(); - row - .Column(0, ProtoToString(operation.Id())) - .Column(1, operation.Ready() ? "true" : "false") - .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())) - .Column(3, metadata.State) - .Column(4, FloatToString(metadata.Progress, PREC_POINT_DIGITS, 2) + "%") - .Column(5, metadata.Path) - .Column(6, metadata.Desctiption ? metadata.Desctiption->GetIndexName() : ""); - - TStringBuilder freeText; - - if (!status.GetIssues().Empty()) { - freeText << "Issues: " << Endl; - for (const auto& issue : status.GetIssues()) { - freeText << " - " << issue << Endl; - } - } - - row.FreeText(freeText); - } - + TPrettyTable MakeTable(const NYdb::NTable::TBuildIndexOperation&) { + return TPrettyTable({"id", "ready", "status", "state", "progress", "table", "index"}); + } + + void PrettyPrint(const NYdb::NTable::TBuildIndexOperation& operation, TPrettyTable& table) { + const auto& status = operation.Status(); + const auto& metadata = operation.Metadata(); + + auto& row = table.AddRow(); + row + .Column(0, ProtoToString(operation.Id())) + .Column(1, operation.Ready() ? "true" : "false") + .Column(2, status.GetStatus() == NYdb::EStatus::STATUS_UNDEFINED ? "" : ToString(status.GetStatus())) + .Column(3, metadata.State) + .Column(4, FloatToString(metadata.Progress, PREC_POINT_DIGITS, 2) + "%") + .Column(5, metadata.Path) + .Column(6, metadata.Desctiption ? metadata.Desctiption->GetIndexName() : ""); + + TStringBuilder freeText; + + if (!status.GetIssues().Empty()) { + freeText << "Issues: " << Endl; + for (const auto& issue : status.GetIssues()) { + freeText << " - " << issue << Endl; + } + } + + row.FreeText(freeText); + } + // Common template <typename T> void PrintOperationImpl(const T& operation, EOutputFormat format) { @@ -273,11 +273,11 @@ namespace { } -/// Common -void PrintOperation(const TOperation& operation, EOutputFormat format) { - PrintOperationImpl(operation, format); -} - +/// Common +void PrintOperation(const TOperation& operation, EOutputFormat format) { + PrintOperationImpl(operation, format); +} + /// YT void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format) { PrintOperationImpl(operation, format); @@ -307,13 +307,13 @@ void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS } /// Index build -void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format) { - PrintOperationImpl(operation, format); +void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format) { + PrintOperationImpl(operation, format); +} + +void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format) { + PrintOperationsListImpl(operations, format); +} + } - -void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format) { - PrintOperationsListImpl(operations, format); } - -} -} diff --git a/ydb/public/lib/ydb_cli/common/print_operation.h b/ydb/public/lib/ydb_cli/common/print_operation.h index 5a039f2592..95e359ce03 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.h +++ b/ydb/public/lib/ydb_cli/common/print_operation.h @@ -9,9 +9,9 @@ namespace NYdb { namespace NConsoleClient { -/// Common -void PrintOperation(const TOperation& operation, EOutputFormat format); - +/// Common +void PrintOperation(const TOperation& operation, EOutputFormat format); + /// YT void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format); void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToYtResponse>& operations, EOutputFormat format); @@ -24,9 +24,9 @@ void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToS3R void PrintOperation(const NImport::TImportFromS3Response& operation, EOutputFormat format); void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS3Response>& operations, EOutputFormat format); -/// Index build -void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format); -void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format); - +/// Index build +void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format); +void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format); + } } diff --git a/ydb/public/lib/ydb_cli/dump/util/util.cpp b/ydb/public/lib/ydb_cli/dump/util/util.cpp index 3592d3c329..37c8d4b646 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.cpp +++ b/ydb/public/lib/ydb_cli/dump/util/util.cpp @@ -125,11 +125,11 @@ TStatus RemoveDirectoryRecursive( break; } default: - return TStatus(EStatus::BAD_REQUEST, {}); + return TStatus(EStatus::BAD_REQUEST, {}); } } - return TStatus(EStatus::SUCCESS, {}); + return TStatus(EStatus::SUCCESS, {}); } } // NDump diff --git a/ydb/public/lib/ydb_cli/dump/util/util.h b/ydb/public/lib/ydb_cli/dump/util/util.h index b06ff18bee..d5e8fc3b97 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.h +++ b/ydb/public/lib/ydb_cli/dump/util/util.h @@ -15,7 +15,7 @@ inline TResult Result(EStatus code = EStatus::SUCCESS, const TString& error = {} if (error) { issues.AddIssue(NYql::TIssue(error)); } - return TResult(TStatus(code, std::move(issues))); + return TResult(TStatus(code, std::move(issues))); } template <typename TResult> diff --git a/ydb/public/lib/yson_value/ydb_yson_value.cpp b/ydb/public/lib/yson_value/ydb_yson_value.cpp index 5e97a6df1a..c855acf0a7 100644 --- a/ydb/public/lib/yson_value/ydb_yson_value.cpp +++ b/ydb/public/lib/yson_value/ydb_yson_value.cpp @@ -83,7 +83,7 @@ static void PrimitiveValueToYson(EPrimitiveType type, TValueParser& parser, NYso writer.OnStringScalar(parser.GetDyNumber()); break; default: - ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); + ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); } } @@ -91,7 +91,7 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr { switch (parser.GetKind()) { case TTypeParser::ETypeKind::Primitive: - PrimitiveValueToYson(parser.GetPrimitiveType(), parser, writer); + PrimitiveValueToYson(parser.GetPrimitiveType(), parser, writer); break; case TTypeParser::ETypeKind::Decimal: @@ -105,7 +105,7 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr } else { writer.OnBeginList(); writer.OnListItem(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); writer.OnEndList(); } parser.CloseOptional(); @@ -117,7 +117,7 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr while (parser.TryNextListItem()) { writer.OnListItem(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); } writer.OnEndList(); @@ -130,7 +130,7 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr while (parser.TryNextMember()) { writer.OnListItem(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); } writer.OnEndList(); @@ -143,7 +143,7 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr while (parser.TryNextElement()) { writer.OnListItem(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); } writer.OnEndList(); @@ -159,11 +159,11 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr writer.OnListItem(); parser.DictKey(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); writer.OnListItem(); parser.DictPayload(); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); writer.OnEndList(); } @@ -180,14 +180,14 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr break; default: - ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << parser.GetKind()); + ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << parser.GetKind()); } } void FormatValueYson(const TValue& value, NYson::TYsonWriter& writer) { TValueParser parser(value); - FormatValueYsonInternal(parser, writer); + FormatValueYsonInternal(parser, writer); } TString FormatValueYson(const TValue& value, NYson::EYsonFormat ysonFormat) @@ -195,7 +195,7 @@ TString FormatValueYson(const TValue& value, NYson::EYsonFormat ysonFormat) TStringStream out; NYson::TYsonWriter writer(&out, ysonFormat, ::NYson::EYsonType::Node, true); - FormatValueYson(value, writer); + FormatValueYson(value, writer); return out.Str(); } @@ -212,7 +212,7 @@ void FormatResultSetYson(const TResultSet& result, NYson::TYsonWriter& writer) writer.OnBeginList(); for (ui32 i = 0; i < columns.size(); ++i) { writer.OnListItem(); - FormatValueYsonInternal(parser.ColumnParser(i), writer); + FormatValueYsonInternal(parser.ColumnParser(i), writer); } writer.OnEndList(); } @@ -225,7 +225,7 @@ TString FormatResultSetYson(const TResultSet& result, NYson::EYsonFormat ysonFor TStringStream out; NYson::TYsonWriter writer(&out, ysonFormat, ::NYson::EYsonType::Node, true); - FormatResultSetYson(result, writer); + FormatResultSetYson(result, writer); return out.Str(); } diff --git a/ydb/public/sdk/cpp/client/draft/ya.make b/ydb/public/sdk/cpp/client/draft/ya.make index 0f9c2aa40b..a5f5fbf597 100644 --- a/ydb/public/sdk/cpp/client/draft/ya.make +++ b/ydb/public/sdk/cpp/client/draft/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - ydb_scripting.cpp +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + ydb_scripting.cpp ydb_long_tx.cpp -) - -PEERDIR( +) + +PEERDIR( ydb/public/api/grpc/draft ydb/public/sdk/cpp/client/ydb_table ydb/public/sdk/cpp/client/ydb_types/operation ydb/public/sdk/cpp/client/ydb_value -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp index 71adc678fd..38b8c8123d 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp @@ -1,5 +1,5 @@ -#include "ydb_scripting.h" - +#include "ydb_scripting.h" + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h> @@ -8,12 +8,12 @@ #include <ydb/public/api/grpc/ydb_scripting_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_scripting.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - -namespace NYdb { -namespace NScripting { - -using namespace NThreading; - + +namespace NYdb { +namespace NScripting { + +using namespace NThreading; + TExecuteYqlResult::TExecuteYqlResult(TStatus&& status, TVector<TResultSet>&& resultSets, const TMaybe<NTable::TQueryStats>& queryStats) : TStatus(std::move(status)) @@ -50,7 +50,7 @@ public: using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; using TGRpcStatus = NGrpc::TGrpcStatus; - TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) + TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) : StreamProcessor_(streamProcessor) , Finished_(false) , Endpoint_(endpoint) @@ -70,13 +70,13 @@ public: auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { if (!grpcStatus.Ok()) { self->Finished_ = true; - promise.SetValue({ TStatus(TPlainStatus(grpcStatus, self->Endpoint_)) }); + promise.SetValue({ TStatus(TPlainStatus(grpcStatus, self->Endpoint_)) }); } else { NYql::TIssues issues; NYql::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); TPlainStatus plainStatus{ clientStatus, std::move(issues), self->Endpoint_, {} }; - TStatus status{ std::move(plainStatus) }; + TStatus status{ std::move(plainStatus) }; TMaybe<NTable::TQueryStats> queryStats; if (self->Response_.result().has_query_stats()) { @@ -110,8 +110,8 @@ private: TYqlResultPartIterator::TYqlResultPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status) - : TStatus(std::move(status)) + TPlainStatus&& status) + : TStatus(std::move(status)) , ReaderImpl_(impl) {} @@ -142,59 +142,59 @@ const TString& TExplainYqlResult::GetPlan() const { //////////////////////////////////////////////////////////////////////////////// -class TScriptingClient::TImpl : public TClientImplCommon<TScriptingClient::TImpl> { -public: +class TScriptingClient::TImpl : public TClientImplCommon<TScriptingClient::TImpl> { +public: using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr; - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} - - template<typename TParamsType> + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) {} + + template<typename TParamsType> TAsyncExecuteYqlResult ExecuteYqlScript(const TString& script, TParamsType params, const TExecuteYqlRequestSettings& settings) { - auto request = MakeOperationRequest<Ydb::Scripting::ExecuteYqlRequest>(settings); - request.set_script(script); - SetParams(params, &request); + auto request = MakeOperationRequest<Ydb::Scripting::ExecuteYqlRequest>(settings); + request.set_script(script); + SetParams(params, &request); request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); - + auto promise = NewPromise<TExecuteYqlResult>(); - - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { - TVector<TResultSet> res; + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + TVector<TResultSet> res; TMaybe<NTable::TQueryStats> queryStats; - if (any) { - Ydb::Scripting::ExecuteYqlResult result; - any->UnpackTo(&result); - - for (size_t i = 0; i < result.result_setsSize(); i++) { - res.push_back(TResultSet(*result.mutable_result_sets(i))); - } + if (any) { + Ydb::Scripting::ExecuteYqlResult result; + any->UnpackTo(&result); + + for (size_t i = 0; i < result.result_setsSize(); i++) { + res.push_back(TResultSet(*result.mutable_result_sets(i))); + } if (result.has_query_stats()) { queryStats = NTable::TQueryStats(result.query_stats()); } - } - - TExecuteYqlResult executeResult(TStatus(std::move(status)), std::move(res), + } + + TExecuteYqlResult executeResult(TStatus(std::move(status)), std::move(res), queryStats); promise.SetValue(std::move(executeResult)); - }; - + }; + Connections_->RunDeferred<Ydb::Scripting::V1::ScriptingService, Ydb::Scripting::ExecuteYqlRequest, Ydb::Scripting::ExecuteYqlResponse>( std::move(request), extractor, &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExecuteYql, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); - - return promise.GetFuture(); - } - + + return promise.GetFuture(); + } + template<typename TParamsType> TFuture<std::pair<TPlainStatus, TYqlScriptProcessorPtr>> StreamExecuteYqlScriptInternal(const TString& script, TParamsType params, const TExecuteYqlRequestSettings& settings) @@ -229,16 +229,16 @@ public: { auto promise = NewPromise<TYqlResultPartIterator>(); - auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, + auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, TScriptingClient::TImpl::TYqlScriptProcessorPtr>> future) mutable { Y_ASSERT(future.HasValue()); auto pair = future.ExtractValue(); promise.SetValue(TYqlResultPartIterator( pair.second - ? std::make_shared<TYqlResultPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) + ? std::make_shared<TYqlResultPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) : nullptr, - std::move(pair.first))); + std::move(pair.first))); }; StreamExecuteYqlScriptInternal(query, params, settings).Subscribe(iteratorCallback); @@ -266,7 +266,7 @@ public: auto promise = NewPromise<TExplainYqlResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TString plan; ::google::protobuf::Map<TString, Ydb::Type> types; @@ -278,7 +278,7 @@ public: types = result.parameters_types(); } - TExplainYqlResult explainResult(TStatus(std::move(status)), + TExplainYqlResult explainResult(TStatus(std::move(status)), std::move(types), std::move(plan)); promise.SetValue(std::move(explainResult)); }; @@ -296,27 +296,27 @@ public: return promise.GetFuture(); } -private: +private: template<typename TRequest> static void SetParams(::google::protobuf::Map<TString, Ydb::TypedValue>* params, TRequest* request) { - if (params) { - request->mutable_parameters()->swap(*params); - } - } - + if (params) { + request->mutable_parameters()->swap(*params); + } + } + template<typename TRequest> static void SetParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& params, TRequest* request) { - *request->mutable_parameters() = params; - } - -}; - -TScriptingClient::TScriptingClient(const TDriver& driver, const TCommonClientSettings &settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) -{} - + *request->mutable_parameters() = params; + } + +}; + +TScriptingClient::TScriptingClient(const TDriver& driver, const TCommonClientSettings &settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{} + TParamsBuilder TScriptingClient::GetParamsBuilder() { - return TParamsBuilder(); + return TParamsBuilder(); } TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, NYdb::TParams&& params, @@ -324,31 +324,31 @@ TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, { auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); return Impl_->ExecuteYqlScript(query, paramsPtr, settings); -} - +} + TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, const NYdb::TParams& params, const TExecuteYqlRequestSettings &settings) { - if (params.Empty()) { + if (params.Empty()) { return Impl_->ExecuteYqlScript( - query, - nullptr, - settings); - } else { - using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; + query, + nullptr, + settings); + } else { + using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; return Impl_->ExecuteYqlScript<TProtoParamsType&>( - query, + query, params.GetProtoMap(), - settings); - } -} - + settings); + } +} + TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &script, const TExecuteYqlRequestSettings &settings) { return Impl_->ExecuteYqlScript(script, nullptr, settings); -} - +} + TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, const TExecuteYqlRequestSettings& settings) { @@ -379,5 +379,5 @@ TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script, return Impl_->ExplainYqlScript(script, settings); } -} // namespace NScheme -} // namespace NYdb +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h index 8807d590d1..e99f05daa2 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/api/protos/ydb_value.pb.h> - -namespace NYdb { -namespace NScripting { - + +namespace NYdb { +namespace NScripting { + class TExecuteYqlResult : public TStatus { public: TExecuteYqlResult(TStatus&& status, TVector<TResultSet>&& resultSets, const TMaybe<NTable::TQueryStats>& queryStats); - + const TVector<TResultSet>& GetResultSets() const; TResultSet GetResultSet(size_t resultIndex) const; @@ -77,7 +77,7 @@ public: private: TYqlResultPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status + TPlainStatus&& status ); std::shared_ptr<TReaderImpl> ReaderImpl_; }; @@ -98,12 +98,12 @@ using TAsyncExecuteYqlResult = NThreading::TFuture<TExecuteYqlResult>; using TAsyncYqlResultPartIterator = NThreading::TFuture<TYqlResultPartIterator>; using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>; -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + struct TExecuteYqlRequestSettings : public TOperationRequestSettings<TExecuteYqlRequestSettings> { FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); }; - + enum class ExplainYqlRequestMode { // Parse = 1, Validate = 2, @@ -116,24 +116,24 @@ struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYql //////////////////////////////////////////////////////////////////////////////// -class TScriptingClient { - class TImpl; - -public: - TScriptingClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - +class TScriptingClient { + class TImpl; + +public: + TScriptingClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + //! Returns new params builder TParamsBuilder GetParamsBuilder(); TAsyncExecuteYqlResult ExecuteYqlScript(const TString& script, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - + TAsyncExecuteYqlResult ExecuteYqlScript(const TString& script, const TParams& params, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - + TAsyncExecuteYqlResult ExecuteYqlScript(const TString& script, TParams&& params, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); @@ -146,9 +146,9 @@ public: TAsyncExplainYqlResult ExplainYqlScript(const TString& script, const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings()); -private: - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NScripting -} // namespace NYdb +private: + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NScripting +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp index 67fc942d04..1d67d6e580 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp @@ -1,67 +1,67 @@ - + #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_scripting_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> - -#include <grpcpp/server.h> -#include <grpcpp/server_builder.h> -#include <grpcpp/server_context.h> - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> - -using namespace NYdb; -using namespace NYdb::NScripting; - -namespace { - -template<class TService> -std::unique_ptr<grpc::Server> StartGrpcServer(const TString& address, TService& service) { - grpc::ServerBuilder builder; - builder.AddListeningPort(address, grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - return builder.BuildAndStart(); -} - -class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service -{ -public: - grpc::Status ExecuteYql( - grpc::ServerContext* context, - const Ydb::Scripting::ExecuteYqlRequest* request, - Ydb::Scripting::ExecuteYqlResponse* response) override { - context->AddInitialMetadata("key", "value"); - Y_UNUSED(request); - - // Just to make sdk core happy - auto* op = response->mutable_operation(); - op->set_ready(true); - op->set_status(Ydb::StatusIds::SUCCESS); - op->mutable_result(); - - return grpc::Status::OK; - } -}; - -} - -Y_UNIT_TEST_SUITE(ResponseHeaders) { - Y_UNIT_TEST(PassHeader) { - TMockSlyDbProxy slyDbProxy; - - TString addr = "localhost:2135"; - - auto server = StartGrpcServer(addr, slyDbProxy); - - auto config = TDriverConfig() - .SetEndpoint(addr); - TDriver driver(config); - TScriptingClient client(driver); - - auto result = client.ExecuteYqlScript("SMTH").GetValueSync(); - auto metadata = result.GetResponseMetadata(); - - UNIT_ASSERT(metadata.find("key") != metadata.end()); - UNIT_ASSERT_VALUES_EQUAL(metadata.find("key")->second, "value"); - } -} + +#include <grpcpp/server.h> +#include <grpcpp/server_builder.h> +#include <grpcpp/server_context.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + +using namespace NYdb; +using namespace NYdb::NScripting; + +namespace { + +template<class TService> +std::unique_ptr<grpc::Server> StartGrpcServer(const TString& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(address, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + return builder.BuildAndStart(); +} + +class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service +{ +public: + grpc::Status ExecuteYql( + grpc::ServerContext* context, + const Ydb::Scripting::ExecuteYqlRequest* request, + Ydb::Scripting::ExecuteYqlResponse* response) override { + context->AddInitialMetadata("key", "value"); + Y_UNUSED(request); + + // Just to make sdk core happy + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result(); + + return grpc::Status::OK; + } +}; + +} + +Y_UNIT_TEST_SUITE(ResponseHeaders) { + Y_UNIT_TEST(PassHeader) { + TMockSlyDbProxy slyDbProxy; + + TString addr = "localhost:2135"; + + auto server = StartGrpcServer(addr, slyDbProxy); + + auto config = TDriverConfig() + .SetEndpoint(addr); + TDriver driver(config); + TScriptingClient client(driver); + + auto result = client.ExecuteYqlScript("SMTH").GetValueSync(); + auto metadata = result.GetResponseMetadata(); + + UNIT_ASSERT(metadata.find("key") != metadata.end()); + UNIT_ASSERT_VALUES_EQUAL(metadata.find("key")->second, "value"); + } +} diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.cpp b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.cpp index db197fcf40..e158873ab6 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.cpp +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.cpp @@ -1 +1 @@ -#include "discovery_mutator.h" +#include "discovery_mutator.h" diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h index 758d7ffe56..08c54f6878 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> - -namespace NDiscoveryMutator { - -class TDiscoveryMutator: public NYdb::IExtension { -public: - using IApi = NYdb::IDiscoveryMutatorApi; - - using TCb = NYdb::IDiscoveryMutatorApi::TMutatorCb; - - class TParams { - friend class TDiscoveryMutator; - public: - TParams(TCb mutator) - : Mutator_(std::move(mutator)) - { } - - private: - TCb Mutator_; - }; - - TDiscoveryMutator(TParams params, IApi* api) { - api->SetMutatorCb(std::move(params.Mutator_)); - } -}; - -}; // namespace NDiscoveryModifie + +namespace NDiscoveryMutator { + +class TDiscoveryMutator: public NYdb::IExtension { +public: + using IApi = NYdb::IDiscoveryMutatorApi; + + using TCb = NYdb::IDiscoveryMutatorApi::TMutatorCb; + + class TParams { + friend class TDiscoveryMutator; + public: + TParams(TCb mutator) + : Mutator_(std::move(mutator)) + { } + + private: + TCb Mutator_; + }; + + TDiscoveryMutator(TParams params, IApi* api) { + api->SetMutatorCb(std::move(params.Mutator_)); + } +}; + +}; // namespace NDiscoveryModifie diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp index 39bbb26c5a..7a5fa43cc3 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp @@ -5,33 +5,33 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -using namespace NYdb; -using namespace NDiscoveryMutator; - -Y_UNIT_TEST_SUITE(DiscoveryMutator) { - Y_UNIT_TEST(Simple) { - - std::unordered_set<TString> dbs; - - auto mutator = [&dbs](Ydb::Discovery::ListEndpointsResult* proto, TStatus status, const TString& database) { - UNIT_ASSERT_VALUES_EQUAL("localhost:100", status.GetEndpoint()); - dbs.insert(database); - Y_UNUSED(proto); - return status; - }; - auto driver = TDriver( - TDriverConfig() - .SetDatabase("db1") - .SetEndpoint("localhost:100")); - - driver.AddExtension<TDiscoveryMutator>(TDiscoveryMutator::TParams(std::move(mutator))); - - auto clientSettings = NTable::TClientSettings(); - clientSettings.Database("db2"); - - // By default this is sync operation - auto client = NTable::TTableClient(driver, clientSettings); - UNIT_ASSERT(dbs.find("db2") != dbs.end()); - } -} + +using namespace NYdb; +using namespace NDiscoveryMutator; + +Y_UNIT_TEST_SUITE(DiscoveryMutator) { + Y_UNIT_TEST(Simple) { + + std::unordered_set<TString> dbs; + + auto mutator = [&dbs](Ydb::Discovery::ListEndpointsResult* proto, TStatus status, const TString& database) { + UNIT_ASSERT_VALUES_EQUAL("localhost:100", status.GetEndpoint()); + dbs.insert(database); + Y_UNUSED(proto); + return status; + }; + auto driver = TDriver( + TDriverConfig() + .SetDatabase("db1") + .SetEndpoint("localhost:100")); + + driver.AddExtension<TDiscoveryMutator>(TDiscoveryMutator::TParams(std::move(mutator))); + + auto clientSettings = NTable::TClientSettings(); + clientSettings.Database("db2"); + + // By default this is sync operation + auto client = NTable::TTableClient(driver, clientSettings); + UNIT_ASSERT(dbs.find("db2") != dbs.end()); + } +} diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make index 3efe91bd29..7d4997e94c 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - discovery_mutator.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + discovery_mutator.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/ydb_extension -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md index 0e3aed4ddd..869678d8dd 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md @@ -10,11 +10,11 @@ You can plug in YDB C++ SDK extension to monitor how your application interacts - Query size, latency, etc. You can get these metrics via http server provided by YDB C++ SDK or implement your own MetricRegistry if it more convenient. - + ## Setting up Solomon Monitoring > This is Yandex specific section for setting up internal monitoring called Solomon - + ### Setup Solomon Environment TSolomonStatPullExtension class allows you to quickly setup you application monitoring. You need to prepare a Solomon project that can accept your metrics beforehand. [Create project, cluster and service, and connect them.](https://wiki.yandex-team.ru/solomon/howtostart/). @@ -30,7 +30,7 @@ Fill in the following params: After creating NYdb::TDriver you need to add Solomon Monitoring extension. If you set up incorrect hostname or port **TSystemError** exception will be thrown. > **Important**: you must plug in monitoring before driver creation. - + ```cl #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h> @@ -60,26 +60,26 @@ After creating NYdb::TDriver you need to add Solomon Monitoring extension. If yo ## Setup Monitoring of Your Choice Implementing NMonitoring::IMetricRegistry provides more flexibility. You can deliver application metrics to Prometheus or any other system of your choice, just register your specific NMonitoring::IMetricRegistry implementation via AddMetricRegistry function. - + > **Important**: you must plug in monitoring before driver creation. - + Select a method which is right for you: -```cl +```cl #include <ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h> - -... - + +... + void AddMetricRegistry(NYdb::TDriver& driver, NMonitoring::IMetricRegistry* ptr); void AddMetricRegistry(NYdb::TDriver& driver, std::shared_ptr<NMonitoring::IMetricRegistry> ptr); void AddMetricRegistry(NYdb::TDriver& driver, TAtomicSharedPtr<NMonitoring::IMetricRegistry> ptr); -``` - +``` + If you provide a raw pointer, it's your responsibility to delete the registry. You must shutdown the SDK driver before destroying the registry. - + ## Metrics Description - + The most valuable metrics are the following: - + - Grpc/InFlightByYdbHost - queries in flight for a selected YDB hostname - Request/Latency - Query execution latency histogram from the client side, ms (without RetryOperation) - Request/ParamsSize - Query params size histogram in bytes @@ -89,4 +89,4 @@ The most valuable metrics are the following: - Sessions/InPool - Number of sessions in a pool - Sessions/SessionsLimitExceeded - Rate of events for exceeding the limit on the number of sessions on the client side - SessionsByYdbHost - Number of sessions per YDB hosts - + diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp index 32e78e78f5..ec7b0f0db6 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.cpp @@ -9,16 +9,16 @@ TSolomonStatPullExtension::TParams::TParams(const TString& host , const TString& cluster , const TVector<std::pair<TString, TString>>& labels) : Host_(host), Port_(port), Labels_() -{ - Labels_.Add("project", project); - Labels_.Add("service", service); - Labels_.Add("cluster", cluster); - for (const auto& label: labels) { - Labels_.Add(label.first, label.second); +{ + Labels_.Add("project", project); + Labels_.Add("service", service); + Labels_.Add("cluster", cluster); + for (const auto& label: labels) { + Labels_.Add(label.first, label.second); } -} +} -NMonitoring::TLabels TSolomonStatPullExtension::TParams::GetLabels() const { +NMonitoring::TLabels TSolomonStatPullExtension::TParams::GetLabels() const { return Labels_; } @@ -35,7 +35,7 @@ void TSolomonStatPullExtension::TSolomonStatPage::Output(NMonitoring::IMonHttpRe TSolomonStatPullExtension::TSolomonStatPullExtension(const TSolomonStatPullExtension::TParams& params, IApi* api) : MetricRegistry_(new NMonitoring::TMetricRegistry(params.GetLabels())) - , MonService_(params.Port_, params.Host_, 0), Page_( new TSolomonStatPage("stats", "Statistics", api) ) { + , MonService_(params.Port_, params.Host_, 0), Page_( new TSolomonStatPage("stats", "Statistics", api) ) { api->SetMetricRegistry(MetricRegistry_.get()); MonService_.Register(Page_); MonService_.StartOrThrow(); diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h index b3da1b9b29..8d410c2fa2 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h @@ -28,7 +28,7 @@ public: , const TString& cluster , const TVector<std::pair<TString, TString>>& labels = {}); - NMonitoring::TLabels GetLabels() const; + NMonitoring::TLabels GetLabels() const; private: const TString Host_; diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.cpp b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.cpp index 4f085f845d..bd6e24ee07 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.cpp +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.cpp @@ -1 +1 @@ -#include "pull_connector.h" +#include "pull_connector.h" diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h index eb35d7018b..24f5f30069 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h @@ -1,73 +1,73 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> - + #include <library/cpp/monlib/metrics/metric_registry.h> - -#include <util/generic/ptr.h> - -namespace NSolomonStatExtension { - + +#include <util/generic/ptr.h> + +namespace NSolomonStatExtension { + template <typename TMetricRegistryPtr> class TMetricRegistryConnector: public NYdb::IExtension { static NMonitoring::IMetricRegistry* ToRawPtr(NMonitoring::IMetricRegistry* p) { - return p; - } - + return p; + } + static NMonitoring::IMetricRegistry* ToRawPtr(std::shared_ptr<NMonitoring::IMetricRegistry> p) { - return p.get(); - } - + return p.get(); + } + static NMonitoring::IMetricRegistry* ToRawPtr(TAtomicSharedPtr<NMonitoring::IMetricRegistry> p) { - return p.Get(); - } - -public: - using IApi = NYdb::NSdkStats::IStatApi; - - class TParams : public TNonCopyable { + return p.Get(); + } + +public: + using IApi = NYdb::NSdkStats::IStatApi; + + class TParams : public TNonCopyable { friend class TMetricRegistryConnector; - - public: + + public: TParams(TMetricRegistryPtr registry) - : Registry(registry) - {} - - private: + : Registry(registry) + {} + + private: TMetricRegistryPtr Registry; - }; - + }; + TMetricRegistryConnector(const TParams& params, IApi* api) : MetricRegistry_(params.Registry) - { + { api->SetMetricRegistry(ToRawPtr(MetricRegistry_)); - } -private: + } +private: TMetricRegistryPtr MetricRegistry_; -}; - +}; + inline void AddMetricRegistry(NYdb::TDriver& driver, NMonitoring::IMetricRegistry* ptr) { - if (!ptr) - return; + if (!ptr) + return; using TConnector = TMetricRegistryConnector<NMonitoring::IMetricRegistry*>; - - driver.AddExtension<TConnector>(TConnector::TParams(ptr)); -} - + + driver.AddExtension<TConnector>(TConnector::TParams(ptr)); +} + inline void AddMetricRegistry(NYdb::TDriver& driver, std::shared_ptr<NMonitoring::IMetricRegistry> ptr) { - if (!ptr) - return; + if (!ptr) + return; using TConnector = TMetricRegistryConnector<std::shared_ptr<NMonitoring::IMetricRegistry>>; - - driver.AddExtension<TConnector>(TConnector::TParams(ptr)); -} - + + driver.AddExtension<TConnector>(TConnector::TParams(ptr)); +} + inline void AddMetricRegistry(NYdb::TDriver& driver, TAtomicSharedPtr<NMonitoring::IMetricRegistry> ptr) { - if (!ptr) - return; + if (!ptr) + return; using TConnector = TMetricRegistryConnector<TAtomicSharedPtr<NMonitoring::IMetricRegistry>>; - - driver.AddExtension<TConnector>(TConnector::TParams(ptr)); -} - -} // namespace NSolomonStatExtension + + driver.AddExtension<TConnector>(TConnector::TParams(ptr)); +} + +} // namespace NSolomonStatExtension diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/ya.make b/ydb/public/sdk/cpp/client/extensions/solomon_stats/ya.make index 5586a4c015..18fa0b584f 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/ya.make +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/ya.make @@ -7,7 +7,7 @@ OWNER( SRCS( pull_client.cpp - pull_connector.cpp + pull_connector.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp index 35ca4a901d..b6d03cc50a 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp @@ -1,127 +1,127 @@ #include "endpoints.h" - + #include <library/cpp/monlib/metrics/metric_registry.h> #include <library/cpp/string_utils/quote/quote.h> #include <util/random/random.h> - + #include <set> #include <unordered_set> -namespace NYdb { - -class TEndpointElectorSafe::TObjRegistry : public IObjRegistryHandle { -public: +namespace NYdb { + +class TEndpointElectorSafe::TObjRegistry : public IObjRegistryHandle { +public: TObjRegistry(const TStringType& endpoint) - : Endpoint_(endpoint) - {} - - bool Add(TEndpointObj* obj) { + : Endpoint_(endpoint) + {} + + bool Add(TEndpointObj* obj) { std::unique_lock lock(Mutex_); - return Objs_.insert(obj).second; - } - - void Remove(TEndpointObj* obj) { + return Objs_.insert(obj).second; + } + + void Remove(TEndpointObj* obj) { std::unique_lock lock(Mutex_); - Y_VERIFY(Objs_.find(obj) != Objs_.end()); - Objs_.erase(obj); - } - - void NotifyEndpointRemoved() { + Y_VERIFY(Objs_.find(obj) != Objs_.end()); + Objs_.erase(obj); + } + + void NotifyEndpointRemoved() { std::shared_lock lock(Mutex_); - for (auto obj : Objs_) { - obj->OnEndpointRemoved(); - } - } - - size_t Size() const override { + for (auto obj : Objs_) { + obj->OnEndpointRemoved(); + } + } + + size_t Size() const override { std::shared_lock lock(Mutex_); - return Objs_.size(); - } - + return Objs_.size(); + } + const TStringType& GetEndpointName() const { - return Endpoint_; - } - -private: - std::set<TEndpointObj*> Objs_; + return Endpoint_; + } + +private: + std::set<TEndpointObj*> Objs_; const TStringType Endpoint_; - + mutable std::shared_mutex Mutex_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -// Returns index of last resord with same priority or -1 in case of empty input -static i32 GetBestK(const std::vector<TEndpointRecord>& records) { - if (records.empty()) { - return -1; - } - - const i32 bestPriority = records[0].Priority; - - size_t pos = 1; - while (pos < records.size()) { - if (records[pos].Priority != bestPriority) { - break; - } - ++pos; - } - return pos - 1; -} - +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Returns index of last resord with same priority or -1 in case of empty input +static i32 GetBestK(const std::vector<TEndpointRecord>& records) { + if (records.empty()) { + return -1; + } + + const i32 bestPriority = records[0].Priority; + + size_t pos = 1; + while (pos < records.size()) { + if (records[pos].Priority != bestPriority) { + break; + } + ++pos; + } + return pos - 1; +} + std::vector<TStringType> TEndpointElectorSafe::SetNewState(std::vector<TEndpointRecord>&& records) { std::unordered_set<TStringType> index; - std::vector<TEndpointRecord> uniqRec; - - for (auto&& record : records) { - if (index.insert(record.Endpoint).second) { - uniqRec.emplace_back(std::move(record)); - } - } - - Sort(uniqRec.begin(), uniqRec.end()); - - auto bestK = GetBestK(uniqRec); - + std::vector<TEndpointRecord> uniqRec; + + for (auto&& record : records) { + if (index.insert(record.Endpoint).second) { + uniqRec.emplace_back(std::move(record)); + } + } + + Sort(uniqRec.begin(), uniqRec.end()); + + auto bestK = GetBestK(uniqRec); + std::vector<TStringType> removed; - std::vector<std::shared_ptr<TObjRegistry>> notifyRemoved; - - { + std::vector<std::shared_ptr<TObjRegistry>> notifyRemoved; + + { std::unique_lock guard(Mutex_); // Find endpoins which were removed - for (const auto& record : Records_) { - if (index.find(record.Endpoint) == index.end()) { - removed.emplace_back(record.Endpoint); - - auto it = KnownEndpoints_.find(record.Endpoint); + for (const auto& record : Records_) { + if (index.find(record.Endpoint) == index.end()) { + removed.emplace_back(record.Endpoint); + + auto it = KnownEndpoints_.find(record.Endpoint); Y_VERIFY(it != KnownEndpoints_.end()); for (const auto& registry : it->second.TaggedObjs) { - notifyRemoved.emplace_back(registry.second); - } - KnownEndpoints_.erase(it); - } - } + notifyRemoved.emplace_back(registry.second); + } + KnownEndpoints_.erase(it); + } + } // Find endpoints which were added Records_ = std::move(uniqRec); for (const auto& record : Records_) { KnownEndpoints_[record.Endpoint].Record = record; - } - Y_VERIFY(Records_.size() == KnownEndpoints_.size()); - EndpointCountGauge_.SetValue(Records_.size()); - EndpointActiveGauge_.SetValue(Records_.size()); - BestK_ = bestK; + } + Y_VERIFY(Records_.size() == KnownEndpoints_.size()); + EndpointCountGauge_.SetValue(Records_.size()); + EndpointActiveGauge_.SetValue(Records_.size()); + BestK_ = bestK; PessimizationRatio_.store(0); - PessimizationRatioGauge_.SetValue(0); - } - - for (auto& obj : notifyRemoved) { - obj->NotifyEndpointRemoved(); - } - - return removed; -} - + PessimizationRatioGauge_.SetValue(0); + } + + for (auto& obj : notifyRemoved) { + obj->NotifyEndpointRemoved(); + } + + return removed; +} + TEndpointRecord TEndpointElectorSafe::GetEndpoint(const TStringType& preferredEndpoint) const { std::shared_lock guard(Mutex_); if (!preferredEndpoint.empty()) { @@ -129,125 +129,125 @@ TEndpointRecord TEndpointElectorSafe::GetEndpoint(const TStringType& preferredEn if (it != KnownEndpoints_.end()) { return it->second.Record; } - } - if (BestK_ == -1) { - Y_ASSERT(Records_.empty()); + } + if (BestK_ == -1) { + Y_ASSERT(Records_.empty()); return TEndpointRecord(); - } else { - // returns value in range [0, n) - auto idx = RandomNumber<size_t>(BestK_ + 1); + } else { + // returns value in range [0, n) + auto idx = RandomNumber<size_t>(BestK_ + 1); return Records_[idx]; - } -} - -// TODO: Suboptimal, but should not be used often + } +} + +// TODO: Suboptimal, but should not be used often void TEndpointElectorSafe::PessimizeEndpoint(const TStringType& endpoint) { std::unique_lock guard(Mutex_); - for (auto& r : Records_) { - if (r.Endpoint == endpoint && r.Priority != Max<i32>()) { + for (auto& r : Records_) { + if (r.Endpoint == endpoint && r.Priority != Max<i32>()) { int pessimizationRatio = PessimizationRatio_.load(); - auto newRatio = (pessimizationRatio * Records_.size() + 100) / Records_.size(); + auto newRatio = (pessimizationRatio * Records_.size() + 100) / Records_.size(); PessimizationRatio_.store(newRatio); - PessimizationRatioGauge_.SetValue(newRatio); - EndpointActiveGauge_.Dec(); - r.Priority = Max<i32>(); + PessimizationRatioGauge_.SetValue(newRatio); + EndpointActiveGauge_.Dec(); + r.Priority = Max<i32>(); auto it = KnownEndpoints_.find(endpoint); if (it != KnownEndpoints_.end()) { it->second.Record.Priority = Max<i32>(); } - } - } - Sort(Records_.begin(), Records_.end()); - BestK_ = GetBestK(Records_); -} - -// % of endpoints which was pessimized -int TEndpointElectorSafe::GetPessimizationRatio() const { + } + } + Sort(Records_.begin(), Records_.end()); + BestK_ = GetBestK(Records_); +} + +// % of endpoints which was pessimized +int TEndpointElectorSafe::GetPessimizationRatio() const { return PessimizationRatio_.load(); -} - -void TEndpointElectorSafe::SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector) { - EndpointCountGauge_.Set(endpointStatCollector.EndpointCount); - PessimizationRatioGauge_.Set(endpointStatCollector.PessimizationRatio); - EndpointActiveGauge_.Set(endpointStatCollector.EndpointActive); -} - +} + +void TEndpointElectorSafe::SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector) { + EndpointCountGauge_.Set(endpointStatCollector.EndpointCount); + PessimizationRatioGauge_.Set(endpointStatCollector.PessimizationRatio); + EndpointActiveGauge_.Set(endpointStatCollector.EndpointActive); +} + bool TEndpointElectorSafe::LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag) { - { + { std::unique_lock guard(Mutex_); - // Find obj registry for given endpoint - // No endpoint - no registry, return false - auto objIt = KnownEndpoints_.find(endpoint); - if (objIt == KnownEndpoints_.end()) { - return false; - } - + // Find obj registry for given endpoint + // No endpoint - no registry, return false + auto objIt = KnownEndpoints_.find(endpoint); + if (objIt == KnownEndpoints_.end()) { + return false; + } + TTaggedObjRegistry& taggedObjs = objIt->second.TaggedObjs; TTaggedObjRegistry::iterator registryIt = taggedObjs.find(tag); - + if (registryIt == taggedObjs.end()) { registryIt = taggedObjs.emplace(tag, new TObjRegistry(endpoint)).first; - } - - // Call Link under endpoint elector mutex. - // Probably a bit more optimal way is: - // - get TObjRegistry (as shared ptr) and release this mutex - // - call obj->Link whithout mutex - // - check KnownEndpoints_ stil has same TObjRegistry for given endpoint - // - in case of false send notification - return obj->Link(registryIt->second); - } -} - -void TEndpointElectorSafe::ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const { + } + + // Call Link under endpoint elector mutex. + // Probably a bit more optimal way is: + // - get TObjRegistry (as shared ptr) and release this mutex + // - call obj->Link whithout mutex + // - check KnownEndpoints_ stil has same TObjRegistry for given endpoint + // - in case of false send notification + return obj->Link(registryIt->second); + } +} + +void TEndpointElectorSafe::ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const { std::shared_lock guard(Mutex_); - - auto it = std::lower_bound(Records_.begin(), Records_.end(), minPriority, [](const TEndpointRecord& l, i32 r) { - return l.Priority < r; - }); - - while (it != Records_.end()) { - if (it->Priority > maxPriority) - break; - + + auto it = std::lower_bound(Records_.begin(), Records_.end(), minPriority, [](const TEndpointRecord& l, i32 r) { + return l.Priority < r; + }); + + while (it != Records_.end()) { + if (it->Priority > maxPriority) + break; + const TTaggedObjRegistry& taggedObjs = KnownEndpoints_.at(it->Endpoint).TaggedObjs; - - auto registry = taggedObjs.find(tag); - if (registry != taggedObjs.end()) { - cb(it->Endpoint, *registry->second); - } else { - cb(it->Endpoint, TObjRegistry(it->Endpoint)); - } - - it++; - } -} - -void TEndpointObj::Unlink() { - if (ObjRegistry_) { - ObjRegistry_->Remove(this); - } - ObjRegistry_ = nullptr; -} - -bool TEndpointObj::Link(std::shared_ptr<TEndpointElectorSafe::TObjRegistry> registry) { - if (registry->Add(this)) { - if (ObjRegistry_) { - ObjRegistry_->Remove(this); - } - ObjRegistry_ = registry; - return true; - } - return false; -} - -size_t TEndpointObj::ObjectCount() const { - return ObjRegistry_->Size(); -} - -bool TEndpointObj::ObjectRegistred() const { - return bool(ObjRegistry_); -} - -} + + auto registry = taggedObjs.find(tag); + if (registry != taggedObjs.end()) { + cb(it->Endpoint, *registry->second); + } else { + cb(it->Endpoint, TObjRegistry(it->Endpoint)); + } + + it++; + } +} + +void TEndpointObj::Unlink() { + if (ObjRegistry_) { + ObjRegistry_->Remove(this); + } + ObjRegistry_ = nullptr; +} + +bool TEndpointObj::Link(std::shared_ptr<TEndpointElectorSafe::TObjRegistry> registry) { + if (registry->Add(this)) { + if (ObjRegistry_) { + ObjRegistry_->Remove(this); + } + ObjRegistry_ = registry; + return true; + } + return false; +} + +size_t TEndpointObj::ObjectCount() const { + return ObjRegistry_->Size(); +} + +bool TEndpointObj::ObjectRegistred() const { + return bool(ObjRegistry_); +} + +} diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h index 969ead973a..a69c37901c 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include <atomic> #include <shared_mutex> -#include <unordered_map> -#include <vector> - +#include <unordered_map> +#include <vector> + #include <ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> - -namespace NYdb { - -struct TEndpointRecord { + +namespace NYdb { + +struct TEndpointRecord { TStringType Endpoint; - i32 Priority; + i32 Priority; TStringType SslTargetNameOverride; TEndpointRecord() @@ -33,47 +33,47 @@ struct TEndpointRecord { return !Endpoint.empty(); } - bool operator<(const TEndpointRecord& rhs) const { - return Priority < rhs.Priority; - } -}; - -class IObjRegistryHandle { -public: - virtual ~IObjRegistryHandle() = default; - virtual size_t Size() const = 0; -}; - -class TEndpointObj; -class TEndpointElectorSafe { -public: - TEndpointElectorSafe() = default; - - // Sets new endpoints, returns removed + bool operator<(const TEndpointRecord& rhs) const { + return Priority < rhs.Priority; + } +}; + +class IObjRegistryHandle { +public: + virtual ~IObjRegistryHandle() = default; + virtual size_t Size() const = 0; +}; + +class TEndpointObj; +class TEndpointElectorSafe { +public: + TEndpointElectorSafe() = default; + + // Sets new endpoints, returns removed std::vector<TStringType> SetNewState(std::vector<TEndpointRecord>&& records); - - // Allows to get stats - void SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector); - - // Returns prefered (if presents) or best endpoint + + // Allows to get stats + void SetStatCollector(const NSdkStats::TStatCollector::TEndpointElectorStatCollector& endpointStatCollector); + + // Returns prefered (if presents) or best endpoint TEndpointRecord GetEndpoint(const TStringType& preferredEndpoint) const; - - // Move endpoint to the end + + // Move endpoint to the end void PessimizeEndpoint(const TStringType& endpoint); - - // Returns % of pessimized endpoints - int GetPessimizationRatio() const; - - // Associate object with the endpoint - // Returns false if no required endpoint, or object already registered + + // Returns % of pessimized endpoints + int GetPessimizationRatio() const; + + // Associate object with the endpoint + // Returns false if no required endpoint, or object already registered bool LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag); - - // Perform some action for each object group associated with endpoint + + // Perform some action for each object group associated with endpoint using THandleCb = std::function<void(const TStringType& host, const IObjRegistryHandle& handle)>; - void ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const; - - class TObjRegistry; -private: + void ForEachEndpoint(const THandleCb& cb, i32 minPriority, i32 maxPriority, const void* tag) const; + + class TObjRegistry; +private: using TTaggedObjRegistry = std::unordered_map<const void*, std::shared_ptr<TObjRegistry>>; struct TKnownEndpoint { @@ -83,31 +83,31 @@ private: private: mutable std::shared_mutex Mutex_; - std::vector<TEndpointRecord> Records_; + std::vector<TEndpointRecord> Records_; std::unordered_map<TStringType, TKnownEndpoint> KnownEndpoints_; - i32 BestK_ = -1; + i32 BestK_ = -1; std::atomic_int PessimizationRatio_ = 0; - NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> EndpointCountGauge_; - NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> PessimizationRatioGauge_; - NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> EndpointActiveGauge_; -}; - -// Used to track object -// The derived class must call Unlink() before destroying -class TEndpointObj { - friend class TEndpointElectorSafe; -public: - virtual ~TEndpointObj() = default; - - virtual void OnEndpointRemoved() {} - - size_t ObjectCount() const; - bool ObjectRegistred() const; - void Unlink(); - -private: - bool Link(std::shared_ptr<TEndpointElectorSafe::TObjRegistry> registry); - std::shared_ptr<TEndpointElectorSafe::TObjRegistry> ObjRegistry_; -}; - -} // namespace NYdb + NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> EndpointCountGauge_; + NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> PessimizationRatioGauge_; + NSdkStats::TAtomicCounter<NMonitoring::TIntGauge> EndpointActiveGauge_; +}; + +// Used to track object +// The derived class must call Unlink() before destroying +class TEndpointObj { + friend class TEndpointElectorSafe; +public: + virtual ~TEndpointObj() = default; + + virtual void OnEndpointRemoved() {} + + size_t ObjectCount() const; + bool ObjectRegistred() const; + void Unlink(); + +private: + bool Link(std::shared_ptr<TEndpointElectorSafe::TObjRegistry> registry); + std::shared_ptr<TEndpointElectorSafe::TObjRegistry> ObjRegistry_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp index 3f75fcc1d9..7823fb6460 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp @@ -3,301 +3,301 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/threading/future/core/future.h> - -#include <util/system/thread.h> -#include <util/random/random.h> - + +#include <util/system/thread.h> +#include <util/random/random.h> + #include <unordered_set> - -using namespace NYdb; - -class TTestObj : public NYdb::TEndpointObj { -public: - TTestObj(int& counter) - : Counter_(&counter) - { } - - TTestObj() - : Counter_(nullptr) - { } - - ~TTestObj() { - if (ObjectRegistred() && Counter_) { - (*Counter_)++; - } - Unlink(); - } - - virtual void OnEndpointRemoved() { - HostRemoved_ = true; - } - - bool HostRemoved() const { - return HostRemoved_; - } - - bool HostRemoved_ = false; - - int* Counter_; -}; - -class TDiscoveryEmulator : public TThread { -public: - TDiscoveryEmulator(TEndpointElectorSafe& elector, ui64 maxEvents) - : TThread(&ThreadProc, this) - , Elector_(elector) - , MaxEvents_(maxEvents) - { + +using namespace NYdb; + +class TTestObj : public NYdb::TEndpointObj { +public: + TTestObj(int& counter) + : Counter_(&counter) + { } + + TTestObj() + : Counter_(nullptr) + { } + + ~TTestObj() { + if (ObjectRegistred() && Counter_) { + (*Counter_)++; + } + Unlink(); + } + + virtual void OnEndpointRemoved() { + HostRemoved_ = true; + } + + bool HostRemoved() const { + return HostRemoved_; + } + + bool HostRemoved_ = false; + + int* Counter_; +}; + +class TDiscoveryEmulator : public TThread { +public: + TDiscoveryEmulator(TEndpointElectorSafe& elector, ui64 maxEvents) + : TThread(&ThreadProc, this) + , Elector_(elector) + , MaxEvents_(maxEvents) + { Finished_.store(false); - } - - static void* ThreadProc(void* _this) { - SetCurrentThreadName("TDiscoveryEmulator"); - static_cast<TDiscoveryEmulator*>(_this)->Exec(); - return nullptr; - } - - void Exec() { - for (ui64 i = 0; i < MaxEvents_; i++) { - ui8 mask = RandomNumber<ui8>(16); - TVector<TEndpointRecord> endpoints; - - for (size_t i = 0; i < Pool_.size(); i++) { - if (mask & (1 << i)) - endpoints.emplace_back(Pool_[i]); - } - - Elector_.SetNewState(std::move(endpoints)); - - if (i % 256 == 0) { - Sleep(TDuration::MilliSeconds(10)); - } - } + } + + static void* ThreadProc(void* _this) { + SetCurrentThreadName("TDiscoveryEmulator"); + static_cast<TDiscoveryEmulator*>(_this)->Exec(); + return nullptr; + } + + void Exec() { + for (ui64 i = 0; i < MaxEvents_; i++) { + ui8 mask = RandomNumber<ui8>(16); + TVector<TEndpointRecord> endpoints; + + for (size_t i = 0; i < Pool_.size(); i++) { + if (mask & (1 << i)) + endpoints.emplace_back(Pool_[i]); + } + + Elector_.SetNewState(std::move(endpoints)); + + if (i % 256 == 0) { + Sleep(TDuration::MilliSeconds(10)); + } + } Finished_.store(true); - } - - const TVector<TEndpointRecord>& GetPool() const { - return Pool_; - } - - bool Finished() const { + } + + const TVector<TEndpointRecord>& GetPool() const { + return Pool_; + } + + bool Finished() const { return Finished_.load(); - } - -private: - TEndpointElectorSafe& Elector_; - ui64 MaxEvents_; + } + +private: + TEndpointElectorSafe& Elector_; + ui64 MaxEvents_; std::atomic_bool Finished_; - - static const TVector<TEndpointRecord> Pool_; -}; - -const TVector<TEndpointRecord> TDiscoveryEmulator::Pool_ = TVector<TEndpointRecord>{{"One", 1}, {"Two", 2}, {"Three", 3}, {"Four", 4}}; - -Y_UNIT_TEST_SUITE(CheckUtils) { - - Y_UNIT_TEST(NewPromiseInitialized) { - NThreading::TPromise<void> promise = NThreading::NewPromise<void>(); - UNIT_ASSERT(promise.Initialized()); - } - - Y_UNIT_TEST(PromiseDefaultCtorNotInitialized) { - NThreading::TPromise<void> promise; - UNIT_ASSERT(!promise.Initialized()); - } -} - -Y_UNIT_TEST_SUITE(EndpointElector) { - - Y_UNIT_TEST(Empty) { - TEndpointElectorSafe elector; + + static const TVector<TEndpointRecord> Pool_; +}; + +const TVector<TEndpointRecord> TDiscoveryEmulator::Pool_ = TVector<TEndpointRecord>{{"One", 1}, {"Two", 2}, {"Three", 3}, {"Four", 4}}; + +Y_UNIT_TEST_SUITE(CheckUtils) { + + Y_UNIT_TEST(NewPromiseInitialized) { + NThreading::TPromise<void> promise = NThreading::NewPromise<void>(); + UNIT_ASSERT(promise.Initialized()); + } + + Y_UNIT_TEST(PromiseDefaultCtorNotInitialized) { + NThreading::TPromise<void> promise; + UNIT_ASSERT(!promise.Initialized()); + } +} + +Y_UNIT_TEST_SUITE(EndpointElector) { + + Y_UNIT_TEST(Empty) { + TEndpointElectorSafe elector; UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint("").Endpoint, ""); - } - - Y_UNIT_TEST(DiffOnRemove) { - TEndpointElectorSafe elector; - auto removed = elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One", 1}}); - UNIT_ASSERT_VALUES_EQUAL(removed.size(), 0); - removed = elector.SetNewState(TVector<TEndpointRecord>{{"One", 1}}); - UNIT_ASSERT_VALUES_EQUAL(removed.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(removed[0], TString("Two")); - } - - Y_UNIT_TEST(Pessimization) { - TEndpointElectorSafe elector; - elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One", 1}}); - UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); - elector.PessimizeEndpoint("One"); - UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 50); - elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}}); - UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); - } - - Y_UNIT_TEST(Election) { - TEndpointElectorSafe elector; - elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"Three", 3}, {"One_B", 1}}); - std::unordered_set<TString> endpoints; - // Just to make sure no possible to get more than expected - size_t extra_attempts = 1000; - while (endpoints.size() != 2 || --extra_attempts) { + } + + Y_UNIT_TEST(DiffOnRemove) { + TEndpointElectorSafe elector; + auto removed = elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(removed.size(), 0); + removed = elector.SetNewState(TVector<TEndpointRecord>{{"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(removed.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(removed[0], TString("Two")); + } + + Y_UNIT_TEST(Pessimization) { + TEndpointElectorSafe elector; + elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One", 1}}); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); + elector.PessimizeEndpoint("One"); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 50); + elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}}); + UNIT_ASSERT_VALUES_EQUAL(elector.GetPessimizationRatio(), 0); + } + + Y_UNIT_TEST(Election) { + TEndpointElectorSafe elector; + elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"Three", 3}, {"One_B", 1}}); + std::unordered_set<TString> endpoints; + // Just to make sure no possible to get more than expected + size_t extra_attempts = 1000; + while (endpoints.size() != 2 || --extra_attempts) { endpoints.insert(elector.GetEndpoint("").Endpoint); UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint("Three").Endpoint, "Three"); - } - UNIT_ASSERT_VALUES_EQUAL(endpoints.size(), 2); - UNIT_ASSERT(endpoints.find("One_A") != endpoints.end()); - UNIT_ASSERT(endpoints.find("One_B") != endpoints.end()); - - elector.SetNewState(TVector<TEndpointRecord>{{"One", 1}}); - // no prefered endpoint, expect avaliable + } + UNIT_ASSERT_VALUES_EQUAL(endpoints.size(), 2); + UNIT_ASSERT(endpoints.find("One_A") != endpoints.end()); + UNIT_ASSERT(endpoints.find("One_B") != endpoints.end()); + + elector.SetNewState(TVector<TEndpointRecord>{{"One", 1}}); + // no prefered endpoint, expect avaliable UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint("Three").Endpoint, "One"); UNIT_ASSERT_VALUES_EQUAL(elector.GetEndpoint("").Endpoint, "One"); - } - - Y_UNIT_TEST(EndpointAssociationTwoThreadsNoRace) { - TEndpointElectorSafe elector; - - TDiscoveryEmulator emulator(elector, 10000); - - emulator.Start(); - - int counter1 = 0; - int counter2 = 0; - - TVector<std::unique_ptr<TTestObj>> storage; - while (!emulator.Finished()) { - auto obj = std::make_unique<TTestObj>(counter2); - if (elector.LinkObjToEndpoint("Two", obj.get(), nullptr)) { - counter1++; - } - storage.emplace_back(std::move(obj)); - // collect some objects - if (counter1 % 10 == 0) - storage.clear(); - } - - emulator.Join(); - - storage.clear(); - UNIT_ASSERT_VALUES_EQUAL(counter1, counter2); - } - - Y_UNIT_TEST(EndpointAssiciationSingleThread) { - TEndpointElectorSafe elector; - elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"Three", 3}, {"One_B", 1}}); - - auto obj1 = std::make_unique<TTestObj>(); - auto obj2 = std::make_unique<TTestObj>(); - auto obj3 = std::make_unique<TTestObj>(); - auto obj4 = std::make_unique<TTestObj>(); - auto obj5 = std::make_unique<TTestObj>(); - - UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), false); - - UNIT_ASSERT(elector.LinkObjToEndpoint("Two", obj1.get(), nullptr)); - // Registred of same object twice is not allowed - UNIT_ASSERT_VALUES_EQUAL(elector.LinkObjToEndpoint("Two", obj1.get(), nullptr), false); - - UNIT_ASSERT(elector.LinkObjToEndpoint("Three", obj2.get(), nullptr)); - UNIT_ASSERT(elector.LinkObjToEndpoint("Three", obj3.get(), nullptr)); - UNIT_ASSERT(elector.LinkObjToEndpoint("One_B", obj4.get(), nullptr)); - UNIT_ASSERT(elector.LinkObjToEndpoint("One_B", obj5.get(), (void*)1)); - - UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), true); - - UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); - - { - TMap<TString, size_t> sizes; - size_t i = 0; - elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { - sizes[endpoint] = handle.Size(); - i++; - }, 0, Max<i32>(), nullptr); - UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); - UNIT_ASSERT_VALUES_EQUAL(i, 4); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 1); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 2); - } - - { - TMap<TString, size_t> sizes; - size_t i = 0; - elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { - sizes[endpoint] = handle.Size(); - i++; - }, 0, Max<i32>(), (void*)1); - UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); - UNIT_ASSERT_VALUES_EQUAL(i, 4); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 0); - } - - UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), false); - - obj1.reset(); - - UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); - - { - TMap<TString, size_t> sizes; - size_t i = 0; - elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { - sizes[endpoint] = handle.Size(); - i++; - }, 0, Max<i32>(), nullptr); - UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); - UNIT_ASSERT_VALUES_EQUAL(i, 4); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 2); - } - - obj1.reset(new TTestObj()); - elector.LinkObjToEndpoint("Two", obj1.get(), nullptr); - - elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"One_C", 1}}); - - UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); - UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), true); - UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), true); - UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), true); - UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), true); - - { - TMap<TString, size_t> sizes; - size_t i = 0; - elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { - sizes[endpoint] = handle.Size(); - i++; - }, 0, Max<i32>(), nullptr); - UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 3); - UNIT_ASSERT_VALUES_EQUAL(i, 3); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 1); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_C"), 0); - UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); - } - - UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); - - obj2.reset(); - - UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 1); - } -} + } + + Y_UNIT_TEST(EndpointAssociationTwoThreadsNoRace) { + TEndpointElectorSafe elector; + + TDiscoveryEmulator emulator(elector, 10000); + + emulator.Start(); + + int counter1 = 0; + int counter2 = 0; + + TVector<std::unique_ptr<TTestObj>> storage; + while (!emulator.Finished()) { + auto obj = std::make_unique<TTestObj>(counter2); + if (elector.LinkObjToEndpoint("Two", obj.get(), nullptr)) { + counter1++; + } + storage.emplace_back(std::move(obj)); + // collect some objects + if (counter1 % 10 == 0) + storage.clear(); + } + + emulator.Join(); + + storage.clear(); + UNIT_ASSERT_VALUES_EQUAL(counter1, counter2); + } + + Y_UNIT_TEST(EndpointAssiciationSingleThread) { + TEndpointElectorSafe elector; + elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"Three", 3}, {"One_B", 1}}); + + auto obj1 = std::make_unique<TTestObj>(); + auto obj2 = std::make_unique<TTestObj>(); + auto obj3 = std::make_unique<TTestObj>(); + auto obj4 = std::make_unique<TTestObj>(); + auto obj5 = std::make_unique<TTestObj>(); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), false); + + UNIT_ASSERT(elector.LinkObjToEndpoint("Two", obj1.get(), nullptr)); + // Registred of same object twice is not allowed + UNIT_ASSERT_VALUES_EQUAL(elector.LinkObjToEndpoint("Two", obj1.get(), nullptr), false); + + UNIT_ASSERT(elector.LinkObjToEndpoint("Three", obj2.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint("Three", obj3.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint("One_B", obj4.get(), nullptr)); + UNIT_ASSERT(elector.LinkObjToEndpoint("One_B", obj5.get(), (void*)1)); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectRegistred(), true); + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); + + { + TMap<TString, size_t> sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { + sizes[endpoint] = handle.Size(); + i++; + }, 0, Max<i32>(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 2); + } + + { + TMap<TString, size_t> sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { + sizes[endpoint] = handle.Size(); + i++; + }, 0, Max<i32>(), (void*)1); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 0); + } + + UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), false); + + obj1.reset(); + + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), false); + + { + TMap<TString, size_t> sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { + sizes[endpoint] = handle.Size(); + i++; + }, 0, Max<i32>(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(i, 4); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_B"), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Three"), 2); + } + + obj1.reset(new TTestObj()); + elector.LinkObjToEndpoint("Two", obj1.get(), nullptr); + + elector.SetNewState(TVector<TEndpointRecord>{{"Two", 2}, {"One_A", 1}, {"One_C", 1}}); + + UNIT_ASSERT_VALUES_EQUAL(obj1->HostRemoved(), false); + UNIT_ASSERT_VALUES_EQUAL(obj2->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj3->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj4->HostRemoved(), true); + UNIT_ASSERT_VALUES_EQUAL(obj5->HostRemoved(), true); + + { + TMap<TString, size_t> sizes; + size_t i = 0; + elector.ForEachEndpoint([&sizes, &i](const TString& endpoint, const NYdb::IObjRegistryHandle& handle) { + sizes[endpoint] = handle.Size(); + i++; + }, 0, Max<i32>(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(sizes.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(i, 3); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("Two"), 1); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_C"), 0); + UNIT_ASSERT_VALUES_EQUAL(sizes.at("One_A"), 0); + } + + UNIT_ASSERT_VALUES_EQUAL(obj1->ObjectCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(obj2->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(obj4->ObjectCount(), 1); + + obj2.reset(); + + UNIT_ASSERT_VALUES_EQUAL(obj3->ObjectCount(), 1); + } +} diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make index 7aafdf4f90..da6a27f707 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( endpoints.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/monlib/metrics ydb/public/api/grpc -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp index 3d845bbfe9..7b7947578d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp @@ -9,19 +9,19 @@ TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInt {} TEndpointPool::~TEndpointPool() { - try { - NThreading::TFuture<TEndpointUpdateResult> future; - { - std::lock_guard guard(Mutex_); - if (DiscoveryPromise_.Initialized()) { - future = DiscoveryPromise_.GetFuture(); - } + try { + NThreading::TFuture<TEndpointUpdateResult> future; + { + std::lock_guard guard(Mutex_); + if (DiscoveryPromise_.Initialized()) { + future = DiscoveryPromise_.GetFuture(); + } + } + if (future.Initialized()) { + future.Wait(); } - if (future.Initialized()) { - future.Wait(); - } - } catch (...) { - Y_FAIL("Unexpected exception from endpoint pool dtor"); + } catch (...) { + Y_FAIL("Unexpected exception from endpoint pool dtor"); } } @@ -40,7 +40,7 @@ std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::Updat TListEndpointsResult result = future.GetValue(); std::vector<TStringType> removed; if (result.DiscoveryStatus.Status == EStatus::SUCCESS) { - std::vector<TEndpointRecord> records; + std::vector<TEndpointRecord> records; // Is used to convert float to integer load factor // same integer values will be selected randomly. const float multiplicator = 10.0; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp index 7a87d0a8fa..7364c34524 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp @@ -6,9 +6,9 @@ #include <library/cpp/string_utils/quote/quote.h> -#include <thread> -#include <unordered_map> - +#include <thread> +#include <unordered_map> + namespace NYdb { constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck @@ -36,12 +36,12 @@ TDbDriverState::TDbDriverState( , StatCollector(database, client->GetMetricRegistry()) , Log(Client->GetLog()) { - EndpointPool.SetStatCollector(StatCollector); - Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database))); -} - -void TDbDriverState::SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider) { - CredentialsProvider = std::move(credentialsProvider); + EndpointPool.SetStatCollector(StatCollector); + Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database))); +} + +void TDbDriverState::SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider) { + CredentialsProvider = std::move(credentialsProvider); #ifndef YDB_GRPC_UNSECURE_AUTH CallCredentials = grpc::MetadataCredentialsFromPlugin( std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(CredentialsProvider))); @@ -133,12 +133,12 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( clientIdentity = credentialsProviderFactory->GetClientIdentity(); } Quote(database); - const TStateKey key{database, discoveryEndpoint, clientIdentity, discoveryMode, enableSsl}; + const TStateKey key{database, discoveryEndpoint, clientIdentity, discoveryMode, enableSsl}; { std::shared_lock lock(Lock_); - auto state = States_.find(key); - if (state != States_.end()) { - auto strong = state->second.lock(); + auto state = States_.find(key); + if (state != States_.end()) { + auto strong = state->second.lock(); if (strong) { return strong; } @@ -149,9 +149,9 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( for (;;) { std::unique_lock lock(Lock_); { - auto state = States_.find(key); - if (state != States_.end()) { - auto strong = state->second.lock(); + auto state = States_.find(key); + if (state != States_.end()) { + auto strong = state->second.lock(); if (strong) { return strong; } else { @@ -161,7 +161,7 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( // remove expired record from hashmap. So give him chance // to do it after that we will be able to create new state lock.unlock(); - std::this_thread::yield(); + std::this_thread::yield(); continue; } } @@ -182,41 +182,41 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( enableSsl, DiscoveryClient_), deleter); - - strongState->SetCredentialsProvider( - credentialsProviderFactory - ? credentialsProviderFactory->CreateProvider(strongState) - : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); - + + strongState->SetCredentialsProvider( + credentialsProviderFactory + ? credentialsProviderFactory->CreateProvider(strongState) + : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); + DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); Y_VERIFY(States_.emplace(key, strongState).second); break; } } auto updateResult = strongState->EndpointPool.UpdateAsync(); - + if (strongState->DiscoveryMode == EDiscoveryMode::Sync) { - const auto& discoveryStatus = updateResult.first.GetValueSync().DiscoveryStatus; - // Almost always true, except the situation when the current thread was - // preempted just before UpdateAsync call and other one get - // state from cache and call UpdateAsync before us. - if (Y_LIKELY(updateResult.second)) { + const auto& discoveryStatus = updateResult.first.GetValueSync().DiscoveryStatus; + // Almost always true, except the situation when the current thread was + // preempted just before UpdateAsync call and other one get + // state from cache and call UpdateAsync before us. + if (Y_LIKELY(updateResult.second)) { std::unique_lock guard(strongState->LastDiscoveryStatusRWLock); - strongState->LastDiscoveryStatus = discoveryStatus; - } + strongState->LastDiscoveryStatus = discoveryStatus; + } } - + return strongState; } -void TDbDriverState::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { - Client->AddPeriodicTask(std::move(cb), period); -} - +void TDbDriverState::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { + Client->AddPeriodicTask(std::move(cb), period); +} + NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( - TDbDriverState::ENotifyType type + TDbDriverState::ENotifyType type ) { - std::vector<std::weak_ptr<TDbDriverState>> states; + std::vector<std::weak_ptr<TDbDriverState>> states; { std::shared_lock lock(Lock_); states.reserve(States_.size()); @@ -224,7 +224,7 @@ NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( states.push_back(weak.second); } } - std::vector<NThreading::TFuture<void>> results; + std::vector<NThreading::TFuture<void>> results; for (auto& state : states) { auto strong = state.lock(); if (strong) { @@ -244,7 +244,7 @@ NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( } void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) { - std::vector<std::weak_ptr<TDbDriverState>> states; + std::vector<std::weak_ptr<TDbDriverState>> states; { std::shared_lock lock(Lock_); states.reserve(States_.size()); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h index a671488240..e0ce63e4b8 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h @@ -8,22 +8,22 @@ #include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h> namespace NYdb { - + class ICredentialsProvider; class ICredentialsProviderFactory; // Represents state of driver for one particular database -class TDbDriverState - : public std::enable_shared_from_this<TDbDriverState> - , public ICoreFacility -{ -public: +class TDbDriverState + : public std::enable_shared_from_this<TDbDriverState> + , public ICoreFacility +{ +public: enum class ENotifyType : size_t { STOP = 0, COUNT = 1 // types count }; - using TCb = std::function<NThreading::TFuture<void>()>; + using TCb = std::function<NThreading::TFuture<void>()>; using TPtr = std::shared_ptr<TDbDriverState>; TDbDriverState( @@ -34,15 +34,15 @@ public: IInternalClient* client ); - void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; - + void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; + void AddCb(TCb&& cb, ENotifyType type); void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; EBalancingPolicy GetBalancingPolicy() const; TStringType GetEndpoint() const; - void SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider); + void SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider); const TStringType Database; const TStringType DiscoveryEndpoint; @@ -53,7 +53,7 @@ public: TEndpointPool EndpointPool; // StopCb allow client to subscribe for notifications from lower layer std::mutex NotifyCbsLock; - std::array<std::vector<TCb>, static_cast<size_t>(ENotifyType::COUNT)> NotifyCbs; + std::array<std::vector<TCb>, static_cast<size_t>(ENotifyType::COUNT)> NotifyCbs; #ifndef YDB_GRPC_UNSECURE_AUTH std::shared_ptr<grpc::CallCredentials> CallCredentials; #endif @@ -66,17 +66,17 @@ public: // Tracker allows to get driver state by database and credentials class TDbDriverStateTracker { - using TStateKey = std::tuple<TStringType, TStringType, TStringType, EDiscoveryMode, bool>; - struct TStateKeyHash { - size_t operator()(const TStateKey& k) const noexcept { + using TStateKey = std::tuple<TStringType, TStringType, TStringType, EDiscoveryMode, bool>; + struct TStateKeyHash { + size_t operator()(const TStateKey& k) const noexcept { THash<TStringType> strHash; - const size_t h0 = strHash(std::get<0>(k)); - const size_t h1 = strHash(std::get<1>(k)); - const size_t h2 = strHash(std::get<2>(k)); - const size_t h3 = ((size_t)std::get<3>(k) << 1) + (size_t)std::get<4>(k); - return (h0 ^ h1 ^ h2 ^ h3); - } - }; + const size_t h0 = strHash(std::get<0>(k)); + const size_t h1 = strHash(std::get<1>(k)); + const size_t h2 = strHash(std::get<2>(k)); + const size_t h3 = ((size_t)std::get<3>(k) << 1) + (size_t)std::get<4>(k); + return (h0 ^ h1 ^ h2 ^ h3); + } + }; public: TDbDriverStateTracker(IInternalClient* client); TDbDriverState::TPtr GetDriverState( @@ -87,11 +87,11 @@ public: std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory ); NThreading::TFuture<void> SendNotification( - TDbDriverState::ENotifyType type); + TDbDriverState::ENotifyType type); void SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry); private: IInternalClient* DiscoveryClient_; - std::unordered_map<TStateKey, std::weak_ptr<TDbDriverState>, TStateKeyHash> States_; + std::unordered_map<TStateKey, std::weak_ptr<TDbDriverState>, TStateKeyHash> States_; std::shared_mutex Lock_; }; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index 8870dd5ae2..8f2420043e 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -149,7 +149,7 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> p , BalancingSettings_(params->GetBalancingSettings()) , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout()) , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls()) - , MemoryQuota_(params->GetMemoryQuota()) + , MemoryQuota_(params->GetMemoryQuota()) , QueuedRequests_(0) #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL , ChannelPool_(params->GetTcpKeepAliveSettings(), params->GetSocketIdleTimeout()) @@ -297,7 +297,7 @@ void TGRpcConnectionsImpl::WaitIdle() { } void TGRpcConnectionsImpl::Stop(bool wait) { - StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait(); + StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait(); GRpcClientLow_.Stop(wait); } @@ -352,7 +352,7 @@ TListEndpointsResult TGRpcConnectionsImpl::MutateDiscovery(TListEndpointsResult return result; auto endpoint = result.DiscoveryStatus.Endpoint; - auto ydbStatus = NYdb::TStatus(std::move(result.DiscoveryStatus)); + auto ydbStatus = NYdb::TStatus(std::move(result.DiscoveryStatus)); ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), database); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h index 790da29736..bfc1b7ec38 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -75,20 +75,20 @@ public: template<typename TService> std::pair<std::unique_ptr<TServiceConnection<TService>>, TStringType> GetServiceConnection( - TDbDriverStatePtr dbState, const TStringType& preferredEndpoint, - TRpcRequestSettings::TEndpointPolicy endpointPolicy) + TDbDriverStatePtr dbState, const TStringType& preferredEndpoint, + TRpcRequestSettings::TEndpointPolicy endpointPolicy) { auto clientConfig = NGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint); clientConfig.EnableSsl = dbState->EnableSsl; clientConfig.SslCaCert = CaCert_; - clientConfig.MemQuota = MemoryQuota_; - - if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>() - || dbState->Database.empty() - || endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint) - { - SetGrpcKeepAlive(clientConfig, GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY, GRpcKeepAlivePermitWithoutCalls_); - } else { + clientConfig.MemQuota = MemoryQuota_; + + if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>() + || dbState->Database.empty() + || endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint) + { + SetGrpcKeepAlive(clientConfig, GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY, GRpcKeepAlivePermitWithoutCalls_); + } else { auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint); if (!endpoint) { return {nullptr, ""}; @@ -154,8 +154,8 @@ public: }); } - auto endpointPolicy = requestSettings.EndpointPolicy; - + auto endpointPolicy = requestSettings.EndpointPolicy; + WithServiceConnection<TService>( [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), clientTimeout, dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable -> void { @@ -171,7 +171,7 @@ public: #ifndef YDB_GRPC_UNSECURE_AUTH meta.CallCredentials = dbState->CallCredentials; #else - if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { try { meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); } catch (const yexception& e) { @@ -256,7 +256,7 @@ public: serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta, context.get()); - }, dbState, preferredEndpoint, endpointPolicy); + }, dbState, preferredEndpoint, endpointPolicy); } template<typename TService, typename TRequest, typename TResponse> @@ -316,36 +316,36 @@ public: std::move(context)); } - // Run request using discovery endpoint. - // Mostly usefull to make calls from credential provider + // Run request using discovery endpoint. + // Mostly usefull to make calls from credential provider + template<typename TService, typename TRequest, typename TResponse> + static void RunOnDiscoveryEndpoint( + std::shared_ptr<ICoreFacility> facility, + TRequest&& request, + TResponseCb<TResponse>&& responseCb, + TSimpleRpc<TService, TRequest, TResponse> rpc, + TRpcRequestSettings requestSettings, + TDuration clientTimeout) + { + requestSettings.EndpointPolicy = TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint; + requestSettings.UseAuth = false; + // TODO: Change implementation of Run, to use ICoreFacility and remove this cast + auto dbState = std::dynamic_pointer_cast<TDbDriverState>(facility); + Y_VERIFY(dbState); + auto self = dynamic_cast<TGRpcConnectionsImpl*>(dbState->Client); + Y_VERIFY(self); + self->Run<TService, TRequest, TResponse>( + std::move(request), + std::move(responseCb), + rpc, + dbState, + requestSettings, + clientTimeout, + TString(), + nullptr); + } + template<typename TService, typename TRequest, typename TResponse> - static void RunOnDiscoveryEndpoint( - std::shared_ptr<ICoreFacility> facility, - TRequest&& request, - TResponseCb<TResponse>&& responseCb, - TSimpleRpc<TService, TRequest, TResponse> rpc, - TRpcRequestSettings requestSettings, - TDuration clientTimeout) - { - requestSettings.EndpointPolicy = TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint; - requestSettings.UseAuth = false; - // TODO: Change implementation of Run, to use ICoreFacility and remove this cast - auto dbState = std::dynamic_pointer_cast<TDbDriverState>(facility); - Y_VERIFY(dbState); - auto self = dynamic_cast<TGRpcConnectionsImpl*>(dbState->Client); - Y_VERIFY(self); - self->Run<TService, TRequest, TResponse>( - std::move(request), - std::move(responseCb), - rpc, - dbState, - requestSettings, - clientTimeout, - TString(), - nullptr); - } - - template<typename TService, typename TRequest, typename TResponse> void RunDeferred( TRequest&& request, TDeferredResultCb&& userResponseCb, @@ -358,12 +358,12 @@ public: std::shared_ptr<IQueueClientContext> context = nullptr) { auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable { - if (operation) { - status.SetCostInfo(std::move(*operation->mutable_cost_info())); - userResponseCb(operation->mutable_result(), std::move(status)); - } else { - userResponseCb(nullptr, std::move(status)); - } + if (operation) { + status.SetCostInfo(std::move(*operation->mutable_cost_info())); + userResponseCb(operation->mutable_result(), std::move(status)); + } else { + userResponseCb(nullptr, std::move(status)); + } }; RunDeferred<TService, TRequest, TResponse>( @@ -404,8 +404,8 @@ public: return; } - auto endpointPolicy = requestSettings.EndpointPolicy; - + auto endpointPolicy = requestSettings.EndpointPolicy; + WithServiceConnection<TService>( [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable { @@ -418,7 +418,7 @@ public: #ifndef YDB_GRPC_UNSECURE_AUTH meta.CallCredentials = dbState->CallCredentials; #else - if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { try { meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); } catch (const yexception& e) { @@ -482,7 +482,7 @@ public: std::move(rpc), std::move(meta), context.get()); - }, dbState, TStringType(), endpointPolicy); + }, dbState, TStringType(), endpointPolicy); } template<class TService, class TRequest, class TResponse, class TCallback> @@ -502,8 +502,8 @@ public: return; } - auto endpointPolicy = requestSettings.EndpointPolicy; - + auto endpointPolicy = requestSettings.EndpointPolicy; + WithServiceConnection<TService>( [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable { @@ -516,7 +516,7 @@ public: #ifndef YDB_GRPC_UNSECURE_AUTH meta.CallCredentials = dbState->CallCredentials; #else - if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { try { meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); } catch (const yexception& e) { @@ -579,7 +579,7 @@ public: std::move(rpc), std::move(meta), context.get()); - }, dbState, TStringType(), endpointPolicy); + }, dbState, TStringType(), endpointPolicy); } TAsyncListEndpointsResult GetEndpoints(TDbDriverStatePtr dbState) override; @@ -604,32 +604,32 @@ public: private: template <typename TService, typename TCallback> - void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState, - const TStringType& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) - { + void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState, + const TStringType& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) + { using TConnection = std::unique_ptr<TServiceConnection<TService>>; TConnection serviceConnection; TStringType endpoint; - std::tie(serviceConnection, endpoint) = GetServiceConnection<TService>(dbState, preferredEndpoint, endpointPolicy); + std::tie(serviceConnection, endpoint) = GetServiceConnection<TService>(dbState, preferredEndpoint, endpointPolicy); if (!serviceConnection) { if (dbState->DiscoveryMode == EDiscoveryMode::Sync) { TStringStream errString; errString << "Endpoint list is empty for database " << dbState->Database; errString << ", cluster endpoint " << dbState->DiscoveryEndpoint; - TPlainStatus discoveryStatus; + TPlainStatus discoveryStatus; { std::shared_lock guard(dbState->LastDiscoveryStatusRWLock); - discoveryStatus = dbState->LastDiscoveryStatus; + discoveryStatus = dbState->LastDiscoveryStatus; + } + + // Discovery returns success but no serviceConnection - in this case we unable to continue processing + if (discoveryStatus.Ok()) { + errString << " while last discovery returned success status. Unable to continue processing."; + discoveryStatus = TPlainStatus(EStatus::UNAVAILABLE, errString.Str()); + } else { + errString <<"."; + discoveryStatus.Issues.AddIssues({NYql::TIssue(errString.Str())}); } - - // Discovery returns success but no serviceConnection - in this case we unable to continue processing - if (discoveryStatus.Ok()) { - errString << " while last discovery returned success status. Unable to continue processing."; - discoveryStatus = TPlainStatus(EStatus::UNAVAILABLE, errString.Str()); - } else { - errString <<"."; - discoveryStatus.Issues.AddIssues({NYql::TIssue(errString.Str())}); - } dbState->StatCollector.IncReqFailNoEndpoint(); callback( discoveryStatus, @@ -654,7 +654,7 @@ private: // UpdateAsync guarantee one update in progress for state auto asyncResult = dbState->EndpointPool.UpdateAsync(); const bool needUpdateChannels = asyncResult.second; - asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy] + asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy] (const NThreading::TFuture<TEndpointUpdateResult>& future) mutable { --QueuedRequests_; const auto& updateResult = future.GetValue(); @@ -665,7 +665,7 @@ private: } auto discoveryStatus = updateResult.DiscoveryStatus; if (discoveryStatus.Status == EStatus::SUCCESS) { - WithServiceConnection<TService>(std::move(callback), dbState, preferredEndpoint, endpointPolicy); + WithServiceConnection<TService>(std::move(callback), dbState, preferredEndpoint, endpointPolicy); } else { callback( TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), @@ -703,7 +703,7 @@ private: const TBalancingSettings BalancingSettings_; const TDuration GRpcKeepAliveTimeout_; const bool GRpcKeepAlivePermitWithoutCalls_; - const ui64 MemoryQuota_; + const ui64 MemoryQuota_; std::atomic_int64_t QueuedRequests_; #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL @@ -712,8 +712,8 @@ private: // State for default database, token pair TDbDriverStatePtr DefaultState_; - std::vector<std::unique_ptr<IExtension>> Extensions_; - std::vector<std::unique_ptr<IExtensionApi>> ExtensionApis_; + std::vector<std::unique_ptr<IExtension>> Extensions_; + std::vector<std::unique_ptr<IExtensionApi>> ExtensionApis_; IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h index f2dcbfccc6..7833c60947 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h @@ -24,7 +24,7 @@ public: virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0; virtual TDuration GetSocketIdleTimeout() const = 0; virtual const TLog& GetLog() const = 0; - virtual ui64 GetMemoryQuota() const = 0; + virtual ui64 GetMemoryQuota() const = 0; }; } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h index e8e58633cb..07fbadf88b 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h @@ -18,7 +18,7 @@ namespace NMonitoring { namespace NYdb { -class TDbDriverState; +class TDbDriverState; struct TListEndpointsResult; class IInternalClient { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h index d2042e0d3c..c055ee9609 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h @@ -24,10 +24,10 @@ void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) { SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout()); } - - if (settings.ReportCostInfo_) { - operationParams.set_report_cost_info(Ydb::FeatureFlag::ENABLED); - } + + if (settings.ReportCostInfo_) { + operationParams.set_report_cost_info(Ydb::FeatureFlag::ENABLED); + } } template <typename TProtoRequest> diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h index a4a15bb536..084f667892 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h @@ -5,7 +5,7 @@ #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/api/protos/ydb_operation.pb.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> #include <library/cpp/grpc/client/grpc_client_low.h> @@ -17,7 +17,7 @@ struct TPlainStatus { NYql::TIssues Issues; TStringType Endpoint; std::multimap<TStringType, TStringType> Metadata; - Ydb::CostInfo ConstInfo; + Ydb::CostInfo ConstInfo; TPlainStatus() : Status(EStatus::SUCCESS) @@ -48,11 +48,11 @@ struct TPlainStatus { const NGrpc::TGrpcStatus& grpcStatus, const TStringType& endpoint = TStringType(), std::multimap<TStringType, TStringType>&& metadata = {}); - template<class T> - void SetCostInfo(T&& costInfo) { - ConstInfo = std::forward<T>(costInfo); - } - + template<class T> + void SetCostInfo(T&& costInfo) { + ConstInfo = std::forward<T>(costInfo); + } + bool Ok() const { return Status == EStatus::SUCCESS; } diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h index e325649754..2a2611794d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h @@ -9,11 +9,11 @@ struct TRpcRequestSettings { TStringType TraceId; TStringType RequestType; std::vector<std::pair<TStringType, TStringType>> Header; - enum class TEndpointPolicy { - UsePreferedEndpoint, - UseDiscoveryEndpoint // Use single discovery endpoint for request - } EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; - bool UseAuth = true; + enum class TEndpointPolicy { + UsePreferedEndpoint, + UseDiscoveryEndpoint // Use single discovery endpoint for request + } EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; + bool UseAuth = true; template<typename TRequestSettings> static TRpcRequestSettings Make(const TRequestSettings& settings) { @@ -21,8 +21,8 @@ struct TRpcRequestSettings { rpcSettings.TraceId = settings.TraceId_; rpcSettings.RequestType = settings.RequestType_; rpcSettings.Header = settings.Header_; - rpcSettings.EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; - rpcSettings.UseAuth = true; + rpcSettings.EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; + rpcSettings.UseAuth = true; return rpcSettings; } }; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp index 71f95a57d1..21a58b33cb 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp @@ -1,56 +1,56 @@ #include "stats.h" - -namespace NYdb { -namespace NSdkStats { - -const NMonitoring::TLabel SESSIONS_ON_KQP_HOST_LABEL = NMonitoring::TLabel {"sensor", "SessionsByYdbHost"}; -const NMonitoring::TLabel TRANSPORT_ERRORS_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "TransportErrorsByYdbHost"}; -const NMonitoring::TLabel GRPC_INFLIGHT_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "Grpc/InFlightByYdbHost"}; - + +namespace NYdb { +namespace NSdkStats { + +const NMonitoring::TLabel SESSIONS_ON_KQP_HOST_LABEL = NMonitoring::TLabel {"sensor", "SessionsByYdbHost"}; +const NMonitoring::TLabel TRANSPORT_ERRORS_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "TransportErrorsByYdbHost"}; +const NMonitoring::TLabel GRPC_INFLIGHT_BY_HOST_LABEL = NMonitoring::TLabel {"sensor", "Grpc/InFlightByYdbHost"}; + void TStatCollector::IncSessionsOnHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Inc(); - } -} - + ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + void TStatCollector::DecSessionsOnHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Dec(); - } -} - + ptr->IntGauge({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, {"YdbHost", host} })->Dec(); + } +} + void TStatCollector::IncTransportErrorsByHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - ptr->Rate({ DatabaseLabel_, TRANSPORT_ERRORS_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); - } -} - + ptr->Rate({ DatabaseLabel_, TRANSPORT_ERRORS_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + void TStatCollector::IncGRpcInFlightByHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); - } -} - + ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Inc(); + } +} + void TStatCollector::DecGRpcInFlightByHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Dec(); - } -} - + ptr->IntGauge({ DatabaseLabel_, GRPC_INFLIGHT_BY_HOST_LABEL, {"YdbHost", host} })->Dec(); + } +} + void TStatCollector::DeleteHost(const TStringType& host) { if (TMetricRegistry* ptr = MetricRegistryPtr_.Get()) { - const NMonitoring::TLabel hostLabel = NMonitoring::TLabel {"YdbHost", host}; - - { - NMonitoring::TLabels label({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, hostLabel }); + const NMonitoring::TLabel hostLabel = NMonitoring::TLabel {"YdbHost", host}; + + { + NMonitoring::TLabels label({ DatabaseLabel_, SESSIONS_ON_KQP_HOST_LABEL, hostLabel }); ptr->RemoveMetric(label); - } - { - NMonitoring::TLabels label({ DatabaseLabel_, TRANSPORT_ERRORS_BY_HOST_LABEL, hostLabel }); + } + { + NMonitoring::TLabels label({ DatabaseLabel_, TRANSPORT_ERRORS_BY_HOST_LABEL, hostLabel }); ptr->RemoveMetric(label); - } - } -} - -} // namespace NSdkStats -} // namespace NYdb + } + } +} + +} // namespace NSdkStats +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h index 3dec8bfc1b..cae4099682 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h @@ -16,33 +16,33 @@ namespace NYdb { namespace NSdkStats { -// works only for case normal (foo_bar) underscore - +// works only for case normal (foo_bar) underscore + inline TStringType UnderscoreToUpperCamel(const TStringType& in) { TStringType result; - result.reserve(in.size()); - - if (in.empty()) - return {}; - - result.push_back(toupper(in[0])); - - size_t i = 1; - - while (i < in.size()) { - if (in[i] == '_') { - if (++i < in.size()) { - result.push_back(toupper(in[i++])); - } else { - break; - } - } else { - result.push_back(tolower(in[i++])); - } - } - return result; -} - + result.reserve(in.size()); + + if (in.empty()) + return {}; + + result.push_back(toupper(in[0])); + + size_t i = 1; + + while (i < in.size()) { + if (in[i] == '_') { + if (++i < in.size()) { + result.push_back(toupper(in[i++])); + } else { + break; + } + } else { + result.push_back(tolower(in[i++])); + } + } + return result; +} + template<typename TPointer> class TAtomicPointer { public: @@ -189,13 +189,13 @@ public: TSessionPoolStatCollector(NMonitoring::TIntGauge* activeSessions = nullptr , NMonitoring::TIntGauge* inPoolSessions = nullptr - , NMonitoring::TRate* fakeSessions = nullptr) - : ActiveSessions(activeSessions), InPoolSessions(inPoolSessions), FakeSessions(fakeSessions) + , NMonitoring::TRate* fakeSessions = nullptr) + : ActiveSessions(activeSessions), InPoolSessions(inPoolSessions), FakeSessions(fakeSessions) { } NMonitoring::TIntGauge* ActiveSessions; NMonitoring::TIntGauge* InPoolSessions; - NMonitoring::TRate* FakeSessions; + NMonitoring::TRate* FakeSessions; }; struct TClientRetryOperationStatCollector { @@ -208,17 +208,17 @@ public: void IncSyncRetryOperation(const EStatus& status) { if (auto registry = MetricRegistry_.Get()) { - TString statusName = TStringBuilder() << status; - TString sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); - registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); + TString statusName = TStringBuilder() << status; + TString sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); + registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); } } void IncAsyncRetryOperation(const EStatus& status) { if (auto registry = MetricRegistry_.Get()) { - TString statusName = TStringBuilder() << status; - TString sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); - registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); + TString statusName = TStringBuilder() << status; + TString sensor = TStringBuilder() << "RetryOperation/" << UnderscoreToUpperCamel(statusName); + registry->Rate({ {"database", Database_}, {"sensor", sensor} })->Inc(); } } @@ -229,31 +229,31 @@ public: struct TClientStatCollector { - TClientStatCollector(NMonitoring::TRate* cacheMiss = nullptr + TClientStatCollector(NMonitoring::TRate* cacheMiss = nullptr , NMonitoring::THistogram* querySize = nullptr , NMonitoring::THistogram* paramsSize = nullptr - , NMonitoring::TRate* sessionRemoved = nullptr - , NMonitoring::TRate* requestMigrated = nullptr + , NMonitoring::TRate* sessionRemoved = nullptr + , NMonitoring::TRate* requestMigrated = nullptr , TClientRetryOperationStatCollector retryOperationStatCollector = TClientRetryOperationStatCollector()) - : CacheMiss(cacheMiss) - , QuerySize(querySize) - , ParamsSize(paramsSize) - , SessionRemovedDueBalancing(sessionRemoved) - , RequestMigrated(requestMigrated) - , RetryOperationStatCollector(retryOperationStatCollector) + : CacheMiss(cacheMiss) + , QuerySize(querySize) + , ParamsSize(paramsSize) + , SessionRemovedDueBalancing(sessionRemoved) + , RequestMigrated(requestMigrated) + , RetryOperationStatCollector(retryOperationStatCollector) { } - NMonitoring::TRate* CacheMiss; + NMonitoring::TRate* CacheMiss; NMonitoring::THistogram* QuerySize; NMonitoring::THistogram* ParamsSize; - NMonitoring::TRate* SessionRemovedDueBalancing; - NMonitoring::TRate* RequestMigrated; + NMonitoring::TRate* SessionRemovedDueBalancing; + NMonitoring::TRate* RequestMigrated; TClientRetryOperationStatCollector RetryOperationStatCollector; }; TStatCollector(const TStringType& database, TMetricRegistry* sensorsRegistry) - : Database_(database) - , DatabaseLabel_({"database", database}) + : Database_(database) + , DatabaseLabel_({"database", database}) { if (sensorsRegistry) { SetMetricRegistry(sensorsRegistry); @@ -263,29 +263,29 @@ public: void SetMetricRegistry(TMetricRegistry* sensorsRegistry) { Y_VERIFY(sensorsRegistry, "TMetricRegistry is null in stats collector."); MetricRegistryPtr_.Set(sensorsRegistry); - DiscoveryDuePessimization_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/TooManyBadEndpoints"} })); - DiscoveryDueExpiration_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/Regular"} })); - DiscoveryFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/FailedTransportError"} })); - RequestFailDueQueueOverflow_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedDiscoveryQueueOverflow"} })); - RequestFailDueNoEndpoint_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedNoEndpoint"} })); - RequestFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedTransportError"} })); - CacheMiss_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/ClientQueryCacheMiss"} })); - ActiveSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InUse"} })); - InPoolSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InPool"} })); - SettlerSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InSettler"} })); - SessionCV_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "SessionBalancer/Variation"} })); - SessionRemovedDueBalancing_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/SessionsRemoved"} })); - RequestMigrated_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/RequestsMigrated"} })); - FakeSessions_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Sessions/SessionsLimitExceeded"} })); - GRpcInFlight_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Grpc/InFlight"} })); - - RequestLatency_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/Latency"} }, + DiscoveryDuePessimization_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/TooManyBadEndpoints"} })); + DiscoveryDueExpiration_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/Regular"} })); + DiscoveryFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Discovery/FailedTransportError"} })); + RequestFailDueQueueOverflow_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedDiscoveryQueueOverflow"} })); + RequestFailDueNoEndpoint_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedNoEndpoint"} })); + RequestFailDueTransportError_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/FailedTransportError"} })); + CacheMiss_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Request/ClientQueryCacheMiss"} })); + ActiveSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InUse"} })); + InPoolSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InPool"} })); + SettlerSessions_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Sessions/InSettler"} })); + SessionCV_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "SessionBalancer/Variation"} })); + SessionRemovedDueBalancing_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/SessionsRemoved"} })); + RequestMigrated_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "SessionBalancer/RequestsMigrated"} })); + FakeSessions_.Set(sensorsRegistry->Rate({ DatabaseLabel_, {"sensor", "Sessions/SessionsLimitExceeded"} })); + GRpcInFlight_.Set(sensorsRegistry->IntGauge({ DatabaseLabel_, {"sensor", "Grpc/InFlight"} })); + + RequestLatency_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/Latency"} }, NMonitoring::ExponentialHistogram(20, 2, 1))); - QuerySize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/QuerySize"} }, + QuerySize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/QuerySize"} }, NMonitoring::ExponentialHistogram(20, 2, 32))); - ParamsSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ParamsSize"} }, + ParamsSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ParamsSize"} }, NMonitoring::ExponentialHistogram(10, 2, 32))); - ResultSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ResultSize"} }, + ResultSize_.Set(sensorsRegistry->HistogramRate({ DatabaseLabel_, {"sensor", "Request/ResultSize"} }, NMonitoring::ExponentialHistogram(20, 2, 32))); } @@ -327,23 +327,23 @@ public: } } - void SetSessionCV(ui32 cv) { - SessionCV_.SetValue(cv); - } - - void IncGRpcInFlight () { - GRpcInFlight_.Inc(); - } - - void DecGRpcInFlight () { - GRpcInFlight_.Dec(); - } - + void SetSessionCV(ui32 cv) { + SessionCV_.SetValue(cv); + } + + void IncGRpcInFlight () { + GRpcInFlight_.Inc(); + } + + void DecGRpcInFlight () { + GRpcInFlight_.Dec(); + } + TEndpointElectorStatCollector GetEndpointElectorStatCollector() { if (auto registry = MetricRegistryPtr_.Get()) { - auto endpointCoint = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Total"} }); - auto pessimizationRatio = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/BadRatio"} }); - auto activeEndpoints = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Good"} }); + auto endpointCoint = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Total"} }); + auto pessimizationRatio = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/BadRatio"} }); + auto activeEndpoints = registry->IntGauge({ {"database", Database_}, {"sensor", "Endpoints/Good"} }); return TEndpointElectorStatCollector(endpointCoint, pessimizationRatio, activeEndpoints); } else { return TEndpointElectorStatCollector(); @@ -356,11 +356,11 @@ public: } switch (type) { - case TSessionPoolStatCollector::EStatCollectorType::SESSIONPOOL: - return TSessionPoolStatCollector(ActiveSessions_.Get(), InPoolSessions_.Get(), FakeSessions_.Get()); - + case TSessionPoolStatCollector::EStatCollectorType::SESSIONPOOL: + return TSessionPoolStatCollector(ActiveSessions_.Get(), InPoolSessions_.Get(), FakeSessions_.Get()); + case TSessionPoolStatCollector::EStatCollectorType::SETTLERPOOL: - return TSessionPoolStatCollector(nullptr, SettlerSessions_.Get(), nullptr); + return TSessionPoolStatCollector(nullptr, SettlerSessions_.Get(), nullptr); } return TSessionPoolStatCollector(); @@ -368,8 +368,8 @@ public: TClientStatCollector GetClientStatCollector() { if (IsCollecting()) { - return TClientStatCollector(CacheMiss_.Get(), QuerySize_.Get(), ParamsSize_.Get(), - SessionRemovedDueBalancing_.Get(), RequestMigrated_.Get(), + return TClientStatCollector(CacheMiss_.Get(), QuerySize_.Get(), ParamsSize_.Get(), + SessionRemovedDueBalancing_.Get(), RequestMigrated_.Get(), TClientRetryOperationStatCollector(MetricRegistryPtr_.Get(), Database_)); } else { return TClientStatCollector(); @@ -382,32 +382,32 @@ public: void IncSessionsOnHost(const TStringType& host); void DecSessionsOnHost(const TStringType& host); - + void IncTransportErrorsByHost(const TStringType& host); - + void IncGRpcInFlightByHost(const TStringType& host); void DecGRpcInFlightByHost(const TStringType& host); - + void DeleteHost(const TStringType& host); private: const TStringType Database_; - const NMonitoring::TLabel DatabaseLabel_; + const NMonitoring::TLabel DatabaseLabel_; TAtomicPointer<TMetricRegistry> MetricRegistryPtr_; - TAtomicCounter<NMonitoring::TRate> DiscoveryDuePessimization_; - TAtomicCounter<NMonitoring::TRate> DiscoveryDueExpiration_; - TAtomicCounter<NMonitoring::TRate> RequestFailDueQueueOverflow_; - TAtomicCounter<NMonitoring::TRate> RequestFailDueNoEndpoint_; - TAtomicCounter<NMonitoring::TRate> RequestFailDueTransportError_; - TAtomicCounter<NMonitoring::TRate> DiscoveryFailDueTransportError_; + TAtomicCounter<NMonitoring::TRate> DiscoveryDuePessimization_; + TAtomicCounter<NMonitoring::TRate> DiscoveryDueExpiration_; + TAtomicCounter<NMonitoring::TRate> RequestFailDueQueueOverflow_; + TAtomicCounter<NMonitoring::TRate> RequestFailDueNoEndpoint_; + TAtomicCounter<NMonitoring::TRate> RequestFailDueTransportError_; + TAtomicCounter<NMonitoring::TRate> DiscoveryFailDueTransportError_; TAtomicPointer<NMonitoring::TIntGauge> ActiveSessions_; TAtomicPointer<NMonitoring::TIntGauge> InPoolSessions_; TAtomicPointer<NMonitoring::TIntGauge> SettlerSessions_; - TAtomicCounter<NMonitoring::TIntGauge> SessionCV_; - TAtomicCounter<NMonitoring::TRate> SessionRemovedDueBalancing_; - TAtomicCounter<NMonitoring::TRate> RequestMigrated_; - TAtomicCounter<NMonitoring::TRate> FakeSessions_; - TAtomicCounter<NMonitoring::TRate> CacheMiss_; - TAtomicCounter<NMonitoring::TIntGauge> GRpcInFlight_; + TAtomicCounter<NMonitoring::TIntGauge> SessionCV_; + TAtomicCounter<NMonitoring::TRate> SessionRemovedDueBalancing_; + TAtomicCounter<NMonitoring::TRate> RequestMigrated_; + TAtomicCounter<NMonitoring::TRate> FakeSessions_; + TAtomicCounter<NMonitoring::TRate> CacheMiss_; + TAtomicCounter<NMonitoring::TIntGauge> GRpcInFlight_; TAtomicHistogram<NMonitoring::THistogram> RequestLatency_; TAtomicHistogram<NMonitoring::THistogram> QuerySize_; TAtomicHistogram<NMonitoring::THistogram> ParamsSize_; diff --git a/ydb/public/sdk/cpp/client/resources/ya.make b/ydb/public/sdk/cpp/client/resources/ya.make index fc3119fbfd..a9cdec932b 100644 --- a/ydb/public/sdk/cpp/client/resources/ya.make +++ b/ydb/public/sdk/cpp/client/resources/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - ydb_resources.cpp +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + ydb_resources.cpp ydb_ca.cpp -) - -RESOURCE( +) + +RESOURCE( ydb/public/sdk/cpp/client/resources/ydb_sdk_version.txt ydb_sdk_version.txt ydb/public/sdk/cpp/client/resources/ydb_root_ca.pem ydb_root_ca.pem -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp b/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp index f33b0f4138..079e8a59a2 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp +++ b/ydb/public/sdk/cpp/client/resources/ydb_resources.cpp @@ -1,16 +1,16 @@ -#include <library/cpp/resource/resource.h> - -#include "ydb_resources.h" - -namespace NYdb { - +#include <library/cpp/resource/resource.h> + +#include "ydb_resources.h" + +namespace NYdb { + const char* YDB_AUTH_TICKET_HEADER = "x-ydb-auth-ticket"; const char* YDB_DATABASE_HEADER = "x-ydb-database"; const char* YDB_TRACE_ID_HEADER = "x-ydb-trace-id"; const char* YDB_SDK_BUILD_INFO_HEADER = "x-ydb-sdk-build-info"; const char* YDB_REQUEST_TYPE_HEADER = "x-ydb-request-type"; const char* YDB_CONSUMED_UNITS_HEADER = "x-ydb-consumed-units"; - + // The x-ydb-server-hints header. // This header can be sent in the trailing metadata with response. // The only possible value is "session-close". In this case client should gracefully close the session, where the request was executed. @@ -23,8 +23,8 @@ const char* YDB_SESSION_CLOSE = "session-close"; // Send ("x-ydb-client-capabilities", "session-balancer") pair in metadata with a СreateSession request to enable server side session balancing feature. const char* YDB_CLIENT_CAPABILITIES = "x-ydb-client-capabilities"; -TString GetSdkSemver() { - return NResource::Find("ydb_sdk_version.txt"); -} - -} // namespace NYdb +TString GetSdkSemver() { + return NResource::Find("ydb_sdk_version.txt"); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/resources/ydb_resources.h b/ydb/public/sdk/cpp/client/resources/ydb_resources.h index cd046ba53a..61dee0d2f4 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_resources.h +++ b/ydb/public/sdk/cpp/client/resources/ydb_resources.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/string.h> - -namespace NYdb { - +#pragma once + +#include <util/generic/string.h> + +namespace NYdb { + extern const char* YDB_AUTH_TICKET_HEADER; extern const char* YDB_DATABASE_HEADER; extern const char* YDB_TRACE_ID_HEADER; @@ -13,7 +13,7 @@ extern const char* YDB_CONSUMED_UNITS_HEADER; extern const char* YDB_SERVER_HINTS; extern const char* YDB_CLIENT_CAPABILITIES; extern const char* YDB_SESSION_CLOSE; - -TString GetSdkSemver(); - -} // namespace NYdb + +TString GetSdkSemver(); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/resources/ydb_sdk_version.txt b/ydb/public/sdk/cpp/client/resources/ydb_sdk_version.txt index 32bc969786..308b6faa75 100644 --- a/ydb/public/sdk/cpp/client/resources/ydb_sdk_version.txt +++ b/ydb/public/sdk/cpp/client/resources/ydb_sdk_version.txt @@ -1 +1 @@ -1.6.2
\ No newline at end of file +1.6.2
\ No newline at end of file diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 57256db9fc..c43e9aab69 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -1,48 +1,48 @@ -#pragma once - -#define INCLUDE_YDB_INTERNAL_H +#pragma once + +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> #include <memory> namespace NYdb { - -template<typename T> -class TClientImplCommon : public std::enable_shared_from_this<T> { + +template<typename T> +class TClientImplCommon : public std::enable_shared_from_this<T> { public: - TClientImplCommon( - std::shared_ptr<TGRpcConnectionsImpl>&& connections, - const TMaybe<TString>& database, - const TMaybe<TString>& discoveryEndpoint, - const TMaybe<EDiscoveryMode>& discoveryMode, + TClientImplCommon( + std::shared_ptr<TGRpcConnectionsImpl>&& connections, + const TMaybe<TString>& database, + const TMaybe<TString>& discoveryEndpoint, + const TMaybe<EDiscoveryMode>& discoveryMode, const TMaybe<bool>& enableSsl, const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory) - : Connections_(std::move(connections)) + : Connections_(std::move(connections)) , DbDriverState_(Connections_->GetDriverState(database, discoveryEndpoint, discoveryMode, enableSsl, credentialsProviderFactory)) - { - Y_VERIFY(DbDriverState_); - } - - TClientImplCommon( - std::shared_ptr<TGRpcConnectionsImpl>&& connections, - const TCommonClientSettings& settings) - : Connections_(std::move(connections)) - , DbDriverState_( - Connections_->GetDriverState( - settings.Database_, - settings.DiscoveryEndpoint_, - settings.DiscoveryMode_, + { + Y_VERIFY(DbDriverState_); + } + + TClientImplCommon( + std::shared_ptr<TGRpcConnectionsImpl>&& connections, + const TCommonClientSettings& settings) + : Connections_(std::move(connections)) + , DbDriverState_( + Connections_->GetDriverState( + settings.Database_, + settings.DiscoveryEndpoint_, + settings.DiscoveryMode_, settings.EnableSsl_, - settings.CredentialsProviderFactory_ - ) - ) - { - Y_VERIFY(DbDriverState_); - } - + settings.CredentialsProviderFactory_ + ) + ) + { + Y_VERIFY(DbDriverState_); + } + protected: template<typename TService, typename TRequest, typename TResponse> using TAsyncRequest = typename NGrpc::TSimpleRequestProcessor< @@ -55,14 +55,14 @@ protected: TRequest&& request, TAsyncRequest<TService, TRequest, TResponse> rpc, const TRpcRequestSettings& requestSettings = {}, - TDuration timeout = TDuration::Zero(), - const TString& preferredEndpoint = TString()) + TDuration timeout = TDuration::Zero(), + const TString& preferredEndpoint = TString()) { auto promise = NThreading::NewPromise<TStatus>(); - auto extractor = [promise] - (google::protobuf::Any*, TPlainStatus status) mutable { - TStatus st(std::move(status)); + auto extractor = [promise] + (google::protobuf::Any*, TPlainStatus status) mutable { + TStatus st(std::move(status)); promise.SetValue(std::move(st)); }; @@ -70,11 +70,11 @@ protected: std::move(request), extractor, rpc, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, requestSettings, - timeout, - preferredEndpoint); + timeout, + preferredEndpoint); return promise.GetFuture(); } @@ -88,9 +88,9 @@ protected: { auto promise = NThreading::NewPromise<TOp>(); - auto extractor = [promise] - (Ydb::Operations::Operation* operation, TPlainStatus status) mutable { - TStatus st(std::move(status)); + auto extractor = [promise] + (Ydb::Operations::Operation* operation, TPlainStatus status) mutable { + TStatus st(std::move(status)); if (!operation) { promise.SetValue(TOp(std::move(st))); } else { @@ -103,7 +103,7 @@ protected: extractor, rpc, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, requestSettings, timeout); @@ -112,7 +112,7 @@ protected: protected: std::shared_ptr<TGRpcConnectionsImpl> Connections_; - TDbDriverStatePtr DbDriverState_; + TDbDriverStatePtr DbDriverState_; }; } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp index fab7db7483..7a99c95214 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp @@ -8,7 +8,7 @@ #include <ydb/public/api/grpc/ydb_coordination_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> -#include <util/generic/deque.h> +#include <util/generic/deque.h> #include <util/random/entropy.h> namespace NYdb { @@ -185,11 +185,11 @@ class TSessionContext : public TThrRefBase { public: TSessionContext( TGRpcConnectionsImpl* connections, - TDbDriverStatePtr dbState, + TDbDriverStatePtr dbState, const TString& path, const TSessionSettings& settings) - : Connections_(connections) - , DbDriverState_(dbState) + : Connections_(connections) + , DbDriverState_(dbState) , Path_(path) , Settings_(settings) , ProtectionKey_(GenerateProtectionKey(8)) @@ -634,12 +634,12 @@ private: prevConnectContext->Cancel(); } - Connections_->StartBidirectionalStream<TService, TRequest, TResponse>( + Connections_->StartBidirectionalStream<TService, TRequest, TResponse>( [self = TPtr(this)] (auto status, auto processor) { self->OnConnect(std::move(status), std::move(processor)); }, &TService::Stub::AsyncSession, - DbDriverState_, + DbDriverState_, TRpcRequestSettings::Make(Settings_), std::move(connectContext)); @@ -668,20 +668,20 @@ private: } TStatus MakeStatus() const { - return TStatus(TPlainStatus()); + return TStatus(TPlainStatus()); } template<class TSource> TStatus MakeStatus(TSource&& source) const { - return TStatus(std::forward<TSource>(source)); + return TStatus(std::forward<TSource>(source)); } TStatus MakeStatus(EStatus status, NYql::TIssues&& issues) const { - return TStatus(TPlainStatus(status, std::move(issues))); + return TStatus(TPlainStatus(status, std::move(issues))); } TStatus MakeStatus(EStatus status, const TString& message) const { - return TStatus(TPlainStatus(status, message)); + return TStatus(TPlainStatus(status, message)); } private: @@ -698,7 +698,7 @@ private: template<class TSource> void SetCurrentFailure(TSource&& source) { Y_VERIFY(!CurrentFailure); - CurrentFailure.Reset(new TStatus(std::forward<TSource>(source))); + CurrentFailure.Reset(new TStatus(std::forward<TSource>(source))); } template<class T> @@ -1725,7 +1725,7 @@ private: private: TGRpcConnectionsImpl* const Connections_; - TDbDriverStatePtr DbDriverState_; + TDbDriverStatePtr DbDriverState_; const TString Path_; const TSessionSettings Settings_; const TString ProtectionKey_; @@ -1786,10 +1786,10 @@ private: //////////////////////////////////////////////////////////////////////////////// -class TClient::TImpl : public TClientImplCommon<TClient::TImpl> { +class TClient::TImpl : public TClientImplCommon<TClient::TImpl> { public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) { } @@ -1799,7 +1799,7 @@ public: { auto session = MakeIntrusive<TSessionContext>( Connections_.get(), - DbDriverState_, + DbDriverState_, path, settings); @@ -1855,15 +1855,15 @@ public: { auto promise = NewPromise<TDescribeNodeResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::Coordination::DescribeNodeResult result; if (any) { any->UnpackTo(&result); } promise.SetValue( TDescribeNodeResult( - TStatus(std::move(status)), + TStatus(std::move(status)), TNodeDescription(std::move(result)))); }; @@ -1873,8 +1873,8 @@ public: std::move(request), std::move(extractor), &Ydb::Coordination::V1::CoordinationService::Stub::AsyncDescribeNode, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); @@ -1882,8 +1882,8 @@ public: } }; -TClient::TClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +TClient::TClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) { } TClient::~TClient() { diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h index d6405fef37..fa8ab572d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h @@ -285,7 +285,7 @@ struct TDescribeSemaphoreSettings { class TClient { public: - TClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + TClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); ~TClient(); diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp index eceffcfefa..f8297f4323 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp @@ -18,7 +18,7 @@ namespace NYdb::NDataStreams::V1 { template<class TProtoResult, class TResultWrapper> auto MakeResultExtractor(NThreading::TPromise <TResultWrapper> promise) { - return [promise = std::move(promise)] + return [promise = std::move(promise)] (google::protobuf::Any *any, TPlainStatus status) mutable { std::unique_ptr <TProtoResult> result; if (any) { @@ -28,7 +28,7 @@ namespace NYdb::NDataStreams::V1 { promise.SetValue( TResultWrapper( - TStatus(std::move(status)), + TStatus(std::move(status)), std::move(result))); }; } diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp index 7004357f78..6957a558ec 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp @@ -1,26 +1,26 @@ #include "discovery.h" - + #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> -namespace NYdb { -namespace NDiscovery { - -TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& proto) - : TStatus(std::move(status)) -{ - const auto& endpoints = proto.endpoints(); - Info_.reserve(proto.endpoints().size()); - for (const auto& endpointInfo : endpoints) { +namespace NYdb { +namespace NDiscovery { + +TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& proto) + : TStatus(std::move(status)) +{ + const auto& endpoints = proto.endpoints(); + Info_.reserve(proto.endpoints().size()); + for (const auto& endpointInfo : endpoints) { TEndpointInfo& info = Info_.emplace_back(); - info.Address = endpointInfo.address(); - info.Port = endpointInfo.port(); - info.Location = endpointInfo.location(); - info.LoadFactor = endpointInfo.load_factor(); - info.Ssl = endpointInfo.ssl(); - info.Services.reserve(endpointInfo.service().size()); - for (const auto& s : endpointInfo.service()) { - info.Services.emplace_back(s); - } + info.Address = endpointInfo.address(); + info.Port = endpointInfo.port(); + info.Location = endpointInfo.location(); + info.LoadFactor = endpointInfo.load_factor(); + info.Ssl = endpointInfo.ssl(); + info.Services.reserve(endpointInfo.service().size()); + for (const auto& s : endpointInfo.service()) { + info.Services.emplace_back(s); + } info.IPv4Addrs.reserve(endpointInfo.ip_v4().size()); for (const auto& addr : endpointInfo.ip_v4()) { info.IPv4Addrs.emplace_back(addr); @@ -30,13 +30,13 @@ TListEndpointsResult::TListEndpointsResult(TStatus&& status, const Ydb::Discover info.IPv6Addrs.emplace_back(addr); } info.SslTargetNameOverride = endpointInfo.ssl_target_name_override(); - } -} - -const TVector<TEndpointInfo>& TListEndpointsResult::GetEndpointsInfo() const { - return Info_; -} - + } +} + +const TVector<TEndpointInfo>& TListEndpointsResult::GetEndpointsInfo() const { + return Info_; +} + TWhoAmIResult::TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto) : TStatus(std::move(status)) { @@ -56,39 +56,39 @@ const TVector<TString>& TWhoAmIResult::GetGroups() const { return Groups_; } -class TDiscoveryClient::TImpl : public TClientImplCommon<TDiscoveryClient::TImpl> { -public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) - { } - - TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings) { - Ydb::Discovery::ListEndpointsRequest request; - request.set_database(DbDriverState_->Database); - - auto promise = NThreading::NewPromise<TListEndpointsResult>(); - - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Discovery::ListEndpointsResult result; - if (any) { - any->UnpackTo(&result); - } - TListEndpointsResult val{TStatus(std::move(status)), result}; - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>( - std::move(request), - extractor, - &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, +class TDiscoveryClient::TImpl : public TClientImplCommon<TDiscoveryClient::TImpl> { +public: + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { } + + TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings) { + Ydb::Discovery::ListEndpointsRequest request; + request.set_database(DbDriverState_->Database); + + auto promise = NThreading::NewPromise<TListEndpointsResult>(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::ListEndpointsResult result; + if (any) { + any->UnpackTo(&result); + } + TListEndpointsResult val{TStatus(std::move(status)), result}; + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return promise.GetFuture(); - } + settings.ClientTimeout_); + + return promise.GetFuture(); + } TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings) { Ydb::Discovery::WhoAmIRequest request; @@ -98,13 +98,13 @@ public: auto promise = NThreading::NewPromise<TWhoAmIResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::Discovery::WhoAmIResult result; if (any) { any->UnpackTo(&result); } - TWhoAmIResult val{ TStatus(std::move(status)), result }; + TWhoAmIResult val{ TStatus(std::move(status)), result }; promise.SetValue(std::move(val)); }; @@ -113,25 +113,25 @@ public: extractor, &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncWhoAmI, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); return promise.GetFuture(); } -}; - -TDiscoveryClient::TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) -{ } - -TAsyncListEndpointsResult TDiscoveryClient::ListEndpoints(const TListEndpointsSettings& settings) { - return Impl_->ListEndpoints(settings); -} - +}; + +TDiscoveryClient::TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ } + +TAsyncListEndpointsResult TDiscoveryClient::ListEndpoints(const TListEndpointsSettings& settings) { + return Impl_->ListEndpoints(settings); +} + TAsyncWhoAmIResult TDiscoveryClient::WhoAmI(const TWhoAmISettings& settings) { return Impl_->WhoAmI(settings); } -} // namespace NDiscovery -} // namespace NYdb +} // namespace NDiscovery +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h index e4d3533c21..d81275b7f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h @@ -1,47 +1,47 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -namespace Ydb { -namespace Discovery { - class ListEndpointsResult; + +namespace Ydb { +namespace Discovery { + class ListEndpointsResult; class WhoAmIResult; -} // namespace Discovery -} // namespace Ydb - -namespace NYdb { -namespace NDiscovery { - -//////////////////////////////////////////////////////////////////////////////// - -class TListEndpointsSettings : public TSimpleRequestSettings<TListEndpointsSettings> {}; - +} // namespace Discovery +} // namespace Ydb + +namespace NYdb { +namespace NDiscovery { + +//////////////////////////////////////////////////////////////////////////////// + +class TListEndpointsSettings : public TSimpleRequestSettings<TListEndpointsSettings> {}; + struct TWhoAmISettings : public TSimpleRequestSettings<TWhoAmISettings> { FLUENT_SETTING_DEFAULT(bool, WithGroups, false); }; -struct TEndpointInfo { - TString Address; - ui32 Port = 0; - float LoadFactor = 0.0; - bool Ssl = false; - TVector<TString> Services; - TString Location; +struct TEndpointInfo { + TString Address; + ui32 Port = 0; + float LoadFactor = 0.0; + bool Ssl = false; + TVector<TString> Services; + TString Location; TVector<TString> IPv4Addrs; TVector<TString> IPv6Addrs; TString SslTargetNameOverride; -}; - -class TListEndpointsResult : public TStatus { -public: - TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& endpoints); - const TVector<TEndpointInfo>& GetEndpointsInfo() const; -private: - TVector<TEndpointInfo> Info_; -}; - -using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>; - +}; + +class TListEndpointsResult : public TStatus { +public: + TListEndpointsResult(TStatus&& status, const Ydb::Discovery::ListEndpointsResult& endpoints); + const TVector<TEndpointInfo>& GetEndpointsInfo() const; +private: + TVector<TEndpointInfo> Info_; +}; + +using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>; + class TWhoAmIResult : public TStatus { public: TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto); @@ -54,19 +54,19 @@ private: using TAsyncWhoAmIResult = NThreading::TFuture<TWhoAmIResult>; -//////////////////////////////////////////////////////////////////////////////// - -class TDiscoveryClient { -public: - explicit TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - - TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings = TListEndpointsSettings()); +//////////////////////////////////////////////////////////////////////////////// + +class TDiscoveryClient { +public: + explicit TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings = TListEndpointsSettings()); TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings = TWhoAmISettings()); - -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NTable -} // namespace NDiscovery + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NTable +} // namespace NDiscovery diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp index 9e963ce649..de17c6e679 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp @@ -1,11 +1,11 @@ #include "driver.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <library/cpp/logger/log.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/getenv.h> @@ -13,7 +13,7 @@ #include <ydb/public/sdk/cpp/client/resources/ydb_ca.h> namespace NYdb { - + using NGrpc::TGRpcClientLow; using NGrpc::TServiceConnection; using NGrpc::TSimpleRequestProcessor; @@ -21,58 +21,58 @@ using NGrpc::TGRpcClientConfig; using NGrpc::TResponseCallback; using NGrpc::TGrpcStatus; using NGrpc::TTcpKeepAliveSettings; - + using Ydb::StatusIds; - -using namespace NThreading; - + +using namespace NThreading; + class TDriverConfig::TImpl : public IConnectionsParams { -public: +public: TStringType GetEndpoint() const override { return Endpoint; } - size_t GetNetworkThreadsNum() const override { return NetworkThreadsNum; } - size_t GetClientThreadsNum() const override { return ClientThreadsNum; } - size_t GetMaxQueuedResponses() const override { return MaxQueuedResponses; } + size_t GetNetworkThreadsNum() const override { return NetworkThreadsNum; } + size_t GetClientThreadsNum() const override { return ClientThreadsNum; } + size_t GetMaxQueuedResponses() const override { return MaxQueuedResponses; } bool IsSslEnabled() const override { return EnableSsl; } TStringType GetCaCert() const override { return CaCert; } TStringType GetDatabase() const override { return Database; } std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const override { return CredentialsProviderFactory; } - EDiscoveryMode GetDiscoveryMode() const override { return DiscoveryMode; } - size_t GetMaxQueuedRequests() const override { return MaxQueuedRequests; } - TTcpKeepAliveSettings GetTcpKeepAliveSettings() const override { return TcpKeepAliveSettings; } - bool GetDrinOnDtors() const override { return DrainOnDtors; } - TBalancingSettings GetBalancingSettings() const override { return BalancingSettings; } - TDuration GetGRpcKeepAliveTimeout() const override { return GRpcKeepAliveTimeout; } + EDiscoveryMode GetDiscoveryMode() const override { return DiscoveryMode; } + size_t GetMaxQueuedRequests() const override { return MaxQueuedRequests; } + TTcpKeepAliveSettings GetTcpKeepAliveSettings() const override { return TcpKeepAliveSettings; } + bool GetDrinOnDtors() const override { return DrainOnDtors; } + TBalancingSettings GetBalancingSettings() const override { return BalancingSettings; } + TDuration GetGRpcKeepAliveTimeout() const override { return GRpcKeepAliveTimeout; } bool GetGRpcKeepAlivePermitWithoutCalls() const override { return GRpcKeepAlivePermitWithoutCalls; } TDuration GetSocketIdleTimeout() const override { return SocketIdleTimeout; } - ui64 GetMemoryQuota() const override { return MemoryQuota; } + ui64 GetMemoryQuota() const override { return MemoryQuota; } const TLog& GetLog() const override { return Log; } - + TStringType Endpoint; - size_t NetworkThreadsNum = 2; - size_t ClientThreadsNum = 0; - size_t MaxQueuedResponses = 0; + size_t NetworkThreadsNum = 2; + size_t ClientThreadsNum = 0; + size_t MaxQueuedResponses = 0; bool EnableSsl = false; TStringType CaCert; TStringType Database; std::shared_ptr<ICredentialsProviderFactory> CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); - EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; - size_t MaxQueuedRequests = 100; + EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; + size_t MaxQueuedRequests = 100; NGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings = - { - true, - TCP_KEEPALIVE_IDLE, - TCP_KEEPALIVE_COUNT, - TCP_KEEPALIVE_INTERVAL - }; - bool DrainOnDtors = true; + { + true, + TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_COUNT, + TCP_KEEPALIVE_INTERVAL + }; + bool DrainOnDtors = true; TBalancingSettings BalancingSettings = TBalancingSettings{EBalancingPolicy::UsePreferableLocation, TStringType()}; - TDuration GRpcKeepAliveTimeout; + TDuration GRpcKeepAliveTimeout; bool GRpcKeepAlivePermitWithoutCalls = false; TDuration SocketIdleTimeout = TDuration::Minutes(6); - ui64 MemoryQuota = 0; + ui64 MemoryQuota = 0; TLog Log; // Null by default. -}; - +}; + TDriverConfig::TDriverConfig(const TStringType& connectionString) : Impl_(new TImpl) { if (connectionString != ""){ @@ -84,84 +84,84 @@ TDriverConfig::TDriverConfig(const TStringType& connectionString) } TDriverConfig& TDriverConfig::SetEndpoint(const TStringType& endpoint) { - Impl_->Endpoint = endpoint; - return *this; -} - + Impl_->Endpoint = endpoint; + return *this; +} + TDriverConfig& TDriverConfig::SetNetworkThreadsNum(size_t sz) { - Impl_->NetworkThreadsNum = sz; - return *this; -} - + Impl_->NetworkThreadsNum = sz; + return *this; +} + TDriverConfig& TDriverConfig::SetClientThreadsNum(size_t sz) { Impl_->ClientThreadsNum = sz; - return *this; -} - -TDriverConfig& TDriverConfig::SetMaxClientQueueSize(size_t sz) { - Impl_->MaxQueuedResponses = sz; - return *this; -} - + return *this; +} + +TDriverConfig& TDriverConfig::SetMaxClientQueueSize(size_t sz) { + Impl_->MaxQueuedResponses = sz; + return *this; +} + TDriverConfig& TDriverConfig::UseSecureConnection(const TStringType& cert) { Impl_->EnableSsl = true; - Impl_->CaCert = cert; - return *this; -} - + Impl_->CaCert = cert; + return *this; +} + TDriverConfig& TDriverConfig::SetAuthToken(const TStringType& token) { return SetCredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token)); -} - +} + TDriverConfig& TDriverConfig::SetDatabase(const TStringType& database) { - Impl_->Database = database; + Impl_->Database = database; Impl_->Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Impl_->Database))); - return *this; -} - + return *this; +} + TDriverConfig& TDriverConfig::SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory) { Impl_->CredentialsProviderFactory = credentialsProviderFactory; return *this; } TDriverConfig& TDriverConfig::SetDiscoveryMode(EDiscoveryMode discoveryMode) { - Impl_->DiscoveryMode = discoveryMode; - return *this; -} - + Impl_->DiscoveryMode = discoveryMode; + return *this; +} + TDriverConfig& TDriverConfig::SetMaxQueuedRequests(size_t sz) { - Impl_->MaxQueuedRequests = sz; - return *this; -} - -TDriverConfig& TDriverConfig::SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval) { - Impl_->TcpKeepAliveSettings.Enabled = enable; - Impl_->TcpKeepAliveSettings.Idle = idle; - Impl_->TcpKeepAliveSettings.Count = count; - Impl_->TcpKeepAliveSettings.Interval = interval; - return *this; -} - -TDriverConfig& TDriverConfig::SetGrpcMemoryQuota(ui64 bytes) { - Impl_->MemoryQuota = bytes; - return *this; -} - -TDriverConfig& TDriverConfig::SetDrainOnDtors(bool allowed) { - Impl_->DrainOnDtors = allowed; - return *this; -} - + Impl_->MaxQueuedRequests = sz; + return *this; +} + +TDriverConfig& TDriverConfig::SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval) { + Impl_->TcpKeepAliveSettings.Enabled = enable; + Impl_->TcpKeepAliveSettings.Idle = idle; + Impl_->TcpKeepAliveSettings.Count = count; + Impl_->TcpKeepAliveSettings.Interval = interval; + return *this; +} + +TDriverConfig& TDriverConfig::SetGrpcMemoryQuota(ui64 bytes) { + Impl_->MemoryQuota = bytes; + return *this; +} + +TDriverConfig& TDriverConfig::SetDrainOnDtors(bool allowed) { + Impl_->DrainOnDtors = allowed; + return *this; +} + TDriverConfig& TDriverConfig::SetBalancingPolicy(EBalancingPolicy policy, const TStringType& params) { - Impl_->BalancingSettings = TBalancingSettings{policy, params}; - return *this; -} - -TDriverConfig& TDriverConfig::SetGRpcKeepAliveTimeout(TDuration timeout) { - Impl_->GRpcKeepAliveTimeout = timeout; - return *this; -} - + Impl_->BalancingSettings = TBalancingSettings{policy, params}; + return *this; +} + +TDriverConfig& TDriverConfig::SetGRpcKeepAliveTimeout(TDuration timeout) { + Impl_->GRpcKeepAliveTimeout = timeout; + return *this; +} + TDriverConfig& TDriverConfig::SetGRpcKeepAlivePermitWithoutCalls(bool permitWithoutCalls) { Impl_->GRpcKeepAlivePermitWithoutCalls = permitWithoutCalls; return *this; @@ -177,22 +177,22 @@ TDriverConfig& TDriverConfig::SetLog(THolder<TLogBackend> log) { return *this; } -//////////////////////////////////////////////////////////////////////////////// - -std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection) { - return connection.Impl_; -} - -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection) { + return connection.Impl_; +} + +//////////////////////////////////////////////////////////////////////////////// + TDriver::TDriver(const TDriverConfig& config) { if (!config.Impl_) { ythrow yexception() << "Invalid config object"; - } + } Impl_.reset(new TGRpcConnectionsImpl(config.Impl_)); -} - +} + void TDriver::Stop(bool wait) { Impl_->Stop(wait); } diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.h b/ydb/public/sdk/cpp/client/ydb_driver/driver.h index 1a9176251e..39c91d8852 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.h +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.h @@ -1,51 +1,51 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_common_client/settings.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> #include <ydb/public/sdk/cpp/client/ydb_types/request_settings.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - + #include <library/cpp/logger/backend.h> //////////////////////////////////////////////////////////////////////////////// namespace NYdb { - -class TDriver; -class TGRpcConnectionsImpl; - -//////////////////////////////////////////////////////////////////////////////// - + +class TDriver; +class TGRpcConnectionsImpl; + +//////////////////////////////////////////////////////////////////////////////// + //! Represents configuration of YDB driver class TDriverConfig { - friend class TDriver; + friend class TDriver; -public: +public: //! Connection string format: "<protocol>://<hostname:port>/?database=<database-path>", //! where "<protocol>://" can be "grpc://" or "grpcs://" or be absent, "<hostname:port>" is endpoint, //! "/?database=<database-path>" is optional TDriverConfig(const TStringType& connectionString = ""); //! Endpoint to initiate connections with Ydb cluster, - //! client will connect to others nodes according to client loadbalancing + //! client will connect to others nodes according to client loadbalancing TDriverConfig& SetEndpoint(const TStringType& endpoint); - //! Set number of network threads, default: 2 + //! Set number of network threads, default: 2 TDriverConfig& SetNetworkThreadsNum(size_t sz); - //! Set number of client pool threads, if 0 adaptive thread pool will be used. - //! NOTE: in case of no zero value it is possible to get deadlock if all threads - //! of this pool is blocked somewhere in user code. + //! Set number of client pool threads, if 0 adaptive thread pool will be used. + //! NOTE: in case of no zero value it is possible to get deadlock if all threads + //! of this pool is blocked somewhere in user code. //! default: 0 TDriverConfig& SetClientThreadsNum(size_t sz); //! Warning: not recommended to change - //! Set max number of queued responses. 0 - no limit - //! There is a queue to perform async calls to user code, - //! if this queue is full, attempts to enqueue responses inside sdk will be blocked + //! Set max number of queued responses. 0 - no limit + //! There is a queue to perform async calls to user code, + //! if this queue is full, attempts to enqueue responses inside sdk will be blocked //! Size of this queue must be greater than max size of all requests inflight - //! Note: if this limit is reached network threads will be blocked. - //! Note: set of this limit can cause deadlock in some case of using async interface - //! This value doesn't make sense if SetClientThreadsNum is 0 - //! default: 0 + //! Note: if this limit is reached network threads will be blocked. + //! Note: set of this limit can cause deadlock in some case of using async interface + //! This value doesn't make sense if SetClientThreadsNum is 0 + //! default: 0 TDriverConfig& SetMaxClientQueueSize(size_t sz); //! Enable Ssl. //! caCerts - The buffer containing the PEM encoding of the server root certificates. @@ -57,45 +57,45 @@ public: TDriverConfig& SetDatabase(const TStringType& database); //! Set credentials data, this option can be overridden for client by ClientSettings TDriverConfig& SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory); - //! Set behaviour of discovery routine - //! See EDiscoveryMode enum comments - //! default: EDiscoveryMode::Sync + //! Set behaviour of discovery routine + //! See EDiscoveryMode enum comments + //! default: EDiscoveryMode::Sync TDriverConfig& SetDiscoveryMode(EDiscoveryMode discoveryMode); - //! Max number of requests in queue waiting for discovery if "Async" mode chosen - //! default: 100 + //! Max number of requests in queue waiting for discovery if "Async" mode chosen + //! default: 100 TDriverConfig& SetMaxQueuedRequests(size_t sz); - //! Limit using of memory for grpc buffer pool. 0 means disabled. + //! Limit using of memory for grpc buffer pool. 0 means disabled. //! If enabled the size must be greater than size of recieved message. - //! default: 0 - TDriverConfig& SetGrpcMemoryQuota(ui64 bytes); + //! default: 0 + TDriverConfig& SetGrpcMemoryQuota(ui64 bytes); //! Specify tcp keep alive settings - //! This option allows to adjust tcp keep alive settings, useful to work - //! with balancers or to detect unexpected connectivity problem. + //! This option allows to adjust tcp keep alive settings, useful to work + //! with balancers or to detect unexpected connectivity problem. //! enable - if true enable tcp keep alive and use following settings - //! - if false disable tcp keep alive - //! idle - (Linux only) the interval between the last data packet sent and the first keepalive probe, sec - //! if zero use OS default - //! count - (Linux only) the number of unacknowledged probes to send before considering the connection dead - //! if zero use OS default - //! interval - (Linux only) the interval between subsequential keepalive probes, sec - //! if zero use OS default - //! NOTE: Please read OS documentation and investigate your network topology before touching this option. - //! default: true, 30, 5, 10 for linux, and true and OS default for others POSIX - TDriverConfig& SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval); - //! Enable or disable drain of client logic (e.g. session pool drain) during dtor call - TDriverConfig& SetDrainOnDtors(bool allowed); - //! Set policy for balancing + //! - if false disable tcp keep alive + //! idle - (Linux only) the interval between the last data packet sent and the first keepalive probe, sec + //! if zero use OS default + //! count - (Linux only) the number of unacknowledged probes to send before considering the connection dead + //! if zero use OS default + //! interval - (Linux only) the interval between subsequential keepalive probes, sec + //! if zero use OS default + //! NOTE: Please read OS documentation and investigate your network topology before touching this option. + //! default: true, 30, 5, 10 for linux, and true and OS default for others POSIX + TDriverConfig& SetTcpKeepAliveSettings(bool enable, size_t idle, size_t count, size_t interval); + //! Enable or disable drain of client logic (e.g. session pool drain) during dtor call + TDriverConfig& SetDrainOnDtors(bool allowed); + //! Set policy for balancing //! Params is a optionally field to set policy settings - //! default: EBalancingPolicy::UsePreferableLocation + //! default: EBalancingPolicy::UsePreferableLocation TDriverConfig& SetBalancingPolicy(EBalancingPolicy policy, const TStringType& params = TStringType()); - //! !!! EXPERIMENTAL !!! + //! !!! EXPERIMENTAL !!! //! Set grpc level keep alive. If keepalive ping was delayed more than given timeout - //! internal grpc routine fails request with TRANSIENT_FAILURE or TRANSPORT_UNAVAILABLE error - //! Note: this timeout should not be too small to prevent fail due to - //! network buffers delay. I.e. values less than 5 seconds may cause request failure - //! even with fast network - //! default: disabled - TDriverConfig& SetGRpcKeepAliveTimeout(TDuration timeout); + //! internal grpc routine fails request with TRANSIENT_FAILURE or TRANSPORT_UNAVAILABLE error + //! Note: this timeout should not be too small to prevent fail due to + //! network buffers delay. I.e. values less than 5 seconds may cause request failure + //! even with fast network + //! default: disabled + TDriverConfig& SetGRpcKeepAliveTimeout(TDuration timeout); TDriverConfig& SetGRpcKeepAlivePermitWithoutCalls(bool permitWithoutCalls); //! Set inactive socket timeout. //! Used to close connections, that were inactive for given time. @@ -106,20 +106,20 @@ public: //! Log backend. TDriverConfig& SetLog(THolder<TLogBackend> log); -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -//! Represents connection pool to the database -class TDriver { - friend std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver); - -public: +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Represents connection pool to the database +class TDriver { + friend std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver); + +public: TDriver(const TDriverConfig& config); - + //! Cancel all currently running and future requests //! This method is useful to make sure there are no new asynchronous //! callbacks and it is safe to destroy the driver @@ -127,11 +127,11 @@ public: //! client thread pool is stopped completely void Stop(bool wait = false); - template<typename TExtension> - void AddExtension(typename TExtension::TParams params = typename TExtension::TParams()); - -private: - std::shared_ptr<TGRpcConnectionsImpl> Impl_; -}; - + template<typename TExtension> + void AddExtension(typename TExtension::TParams params = typename TExtension::TParams()); + +private: + std::shared_ptr<TGRpcConnectionsImpl> Impl_; +}; + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp index 692299da47..c391c033e1 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp +++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp @@ -109,7 +109,7 @@ const TExportToS3Response::TMetadata& TExportToS3Response::Metadata() const { class TExportClient::TImpl : public TClientImplCommon<TExportClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) + : TClientImplCommon(std::move(connections), settings) { } diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp index 835572276e..4cb966a5ce 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp +++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp @@ -17,28 +17,28 @@ void IExtensionApi::SelfRegister(TDriver driver) { namespace NSdkStats { -IStatApi* IStatApi::Create(TDriver driver) { - return new TStatsExtractor(CreateInternalInterface(driver)); +IStatApi* IStatApi::Create(TDriver driver) { + return new TStatsExtractor(CreateInternalInterface(driver)); } } // namespace YSdkStats -class TDiscoveryMutator : public IDiscoveryMutatorApi { -public: - TDiscoveryMutator(std::shared_ptr<TGRpcConnectionsImpl> driverImpl) - : DriverImpl(driverImpl.get()) - { } - - void SetMutatorCb(TMutatorCb&& cb) override { - DriverImpl->SetDiscoveryMutator(std::move(cb)); - } -private: - TGRpcConnectionsImpl* DriverImpl; -}; - -IDiscoveryMutatorApi* IDiscoveryMutatorApi::Create(TDriver driver) { - return new TDiscoveryMutator(CreateInternalInterface(driver)); -} - +class TDiscoveryMutator : public IDiscoveryMutatorApi { +public: + TDiscoveryMutator(std::shared_ptr<TGRpcConnectionsImpl> driverImpl) + : DriverImpl(driverImpl.get()) + { } + + void SetMutatorCb(TMutatorCb&& cb) override { + DriverImpl->SetDiscoveryMutator(std::move(cb)); + } +private: + TGRpcConnectionsImpl* DriverImpl; +}; + +IDiscoveryMutatorApi* IDiscoveryMutatorApi::Create(TDriver driver) { + return new TDiscoveryMutator(CreateInternalInterface(driver)); +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.h b/ydb/public/sdk/cpp/client/ydb_extension/extension.h index 54cb3c7b0b..3371bffddc 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/extension.h +++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.h @@ -4,14 +4,14 @@ #include <library/cpp/monlib/metrics/metric_registry.h> -namespace Ydb { -namespace Discovery { - -class ListEndpointsResult; - -} -} - +namespace Ydb { +namespace Discovery { + +class ListEndpointsResult; + +} +} + namespace NYdb { class IExtensionApi { @@ -35,7 +35,7 @@ namespace NSdkStats { class IStatApi: public IExtensionApi { public: - static IStatApi* Create(TDriver driver); + static IStatApi* Create(TDriver driver); public: virtual void SetMetricRegistry(NMonitoring::IMetricRegistry* sensorsRegistry) = 0; virtual void Accept(NMonitoring::IMetricConsumer* consumer) const = 0; @@ -45,24 +45,24 @@ class DestroyedClientException: public yexception {}; } // namespace NSdkStats - -class IDiscoveryMutatorApi: public IExtensionApi { -public: + +class IDiscoveryMutatorApi: public IExtensionApi { +public: using TMutatorCb = std::function<TStatus(Ydb::Discovery::ListEndpointsResult* proto, TStatus status, const TStringType& database)>; - static IDiscoveryMutatorApi* Create(TDriver driver); -public: - virtual void SetMutatorCb(TMutatorCb&& mutator) = 0; -}; - - + static IDiscoveryMutatorApi* Create(TDriver driver); +public: + virtual void SetMutatorCb(TMutatorCb&& mutator) = 0; +}; + + template<typename TExtension> -void TDriver::AddExtension(typename TExtension::TParams params) { - typename TExtension::IApi* api = TExtension::IApi::Create(*this); +void TDriver::AddExtension(typename TExtension::TParams params) { + typename TExtension::IApi* api = TExtension::IApi::Create(*this); auto extension = new TExtension(params, api); extension->SelfRegister(*this); - if (api) - api->SelfRegister(*this); + if (api) + api->SelfRegister(*this); } } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp index f4cf9c3175..f0a55678ef 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp +++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp @@ -97,9 +97,9 @@ public: TAsyncImportDataResult ImportData(ImportDataRequest&& request, const TSettings& settings) { auto promise = NThreading::NewPromise<TImportDataResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any*, TPlainStatus status) mutable { - TImportDataResult result(TStatus(std::move(status))); + TImportDataResult result(TStatus(std::move(status))); promise.SetValue(std::move(result)); }; diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp index aea913d419..fac441c505 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp @@ -16,8 +16,8 @@ namespace NYdb { namespace NOperation { -constexpr TDuration OPERATION_CLIENT_TIMEOUT = TDuration::Seconds(5); - +constexpr TDuration OPERATION_CLIENT_TIMEOUT = TDuration::Seconds(5); + using namespace NThreading; using namespace Ydb::Operation; using namespace Ydb::Operations; @@ -30,15 +30,15 @@ class TOperationClient::TImpl : public TClientImplCommon<TOperationClient::TImpl TAsyncStatus Run(TRequest&& request, TSimpleRpc<TRequest, TResponse> rpc) { auto promise = NewPromise<TStatus>(); - auto extractor = [promise] - (TResponse* response, TPlainStatus status) mutable { + auto extractor = [promise] + (TResponse* response, TPlainStatus status) mutable { if (response) { NYql::TIssues opIssues; NYql::IssuesFromMessage(response->issues(), opIssues); - TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); + TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); promise.SetValue(std::move(st)); } else { - TStatus st(std::move(status)); + TStatus st(std::move(status)); promise.SetValue(std::move(st)); } }; @@ -47,17 +47,17 @@ class TOperationClient::TImpl : public TClientImplCommon<TOperationClient::TImpl std::move(request), extractor, rpc, - DbDriverState_, + DbDriverState_, TRpcRequestSettings(), - OPERATION_CLIENT_TIMEOUT, - TString()); + OPERATION_CLIENT_TIMEOUT, + TString()); return promise.GetFuture(); } public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) + : TClientImplCommon(std::move(connections), settings) { } @@ -82,25 +82,25 @@ public: TFuture<TOperationsList<TOp>> List(ListOperationsRequest&& request) { auto promise = NewPromise<TOperationsList<TOp>>(); - auto extractor = [promise] - (ListOperationsResponse* response, TPlainStatus status) mutable { + auto extractor = [promise] + (ListOperationsResponse* response, TPlainStatus status) mutable { if (response) { NYql::TIssues opIssues; NYql::IssuesFromMessage(response->issues(), opIssues); - TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); + TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); TVector<TOp> operations(Reserve(response->operations_size())); for (auto& operation : *response->mutable_operations()) { NYql::TIssues opIssues; NYql::IssuesFromMessage(operation.issues(), opIssues); operations.emplace_back( - TStatus(static_cast<EStatus>(operation.status()), std::move(opIssues)), + TStatus(static_cast<EStatus>(operation.status()), std::move(opIssues)), std::move(operation)); } promise.SetValue(TOperationsList<TOp>(std::move(st), std::move(operations), response->next_page_token())); } else { - TStatus st(std::move(status)); + TStatus st(std::move(status)); promise.SetValue(TOperationsList<TOp>(std::move(st))); } }; @@ -109,10 +109,10 @@ public: std::move(request), extractor, &V1::OperationService::Stub::AsyncListOperations, - DbDriverState_, + DbDriverState_, TRpcRequestSettings(), - OPERATION_CLIENT_TIMEOUT, - TString()); + OPERATION_CLIENT_TIMEOUT, + TString()); return promise.GetFuture(); } @@ -183,11 +183,11 @@ TFuture<TOperationsList<NImport::TImportFromS3Response>> TOperationClient::List( return List<NImport::TImportFromS3Response>("import/s3", pageSize, pageToken); } -template TFuture<NTable::TBuildIndexOperation> TOperationClient::Get(const TOperation::TOperationId& id); -template <> -TFuture<TOperationsList<NTable::TBuildIndexOperation>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { - return List<NTable::TBuildIndexOperation>("buildindex", pageSize, pageToken); -} - +template TFuture<NTable::TBuildIndexOperation> TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture<TOperationsList<NTable::TBuildIndexOperation>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { + return List<NTable::TBuildIndexOperation>("buildindex", pageSize, pageToken); +} + } // namespace NOperation } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp index 5807f49036..9e8c5e1e7d 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp @@ -2,48 +2,48 @@ #include <ydb/public/api/protos/ydb_scheme.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> - -#include <util/generic/map.h> - - -namespace NYdb { - -TParams::TImpl::TImpl(::google::protobuf::Map<TString, Ydb::TypedValue>&& paramsMap) { - ParamsMap_.swap(paramsMap); -} - -bool TParams::TImpl::Empty() const { - return ParamsMap_.empty(); -} - -TMap<TString, TValue> TParams::TImpl::GetValues() const { - TMap<TString, TValue> valuesMap; - for (auto it = ParamsMap_.begin(); it != ParamsMap_.end(); ++it) { - auto paramType = TType(it->second.type()); - auto paramValue = TValue(paramType, it->second.value()); - - valuesMap.emplace(it->first, paramValue); - } - - return valuesMap; -} - -TMaybe<TValue> TParams::TImpl::GetValue(const TString& name) const { - auto it = ParamsMap_.find(name); - if (it != ParamsMap_.end()) { - auto paramType = TType(it->second.type()); - return TValue(paramType, it->second.value()); - } - - return TMaybe<TValue>(); -} - -::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::TImpl::GetProtoMapPtr() { - return &ParamsMap_; -} - + +#include <util/generic/map.h> + + +namespace NYdb { + +TParams::TImpl::TImpl(::google::protobuf::Map<TString, Ydb::TypedValue>&& paramsMap) { + ParamsMap_.swap(paramsMap); +} + +bool TParams::TImpl::Empty() const { + return ParamsMap_.empty(); +} + +TMap<TString, TValue> TParams::TImpl::GetValues() const { + TMap<TString, TValue> valuesMap; + for (auto it = ParamsMap_.begin(); it != ParamsMap_.end(); ++it) { + auto paramType = TType(it->second.type()); + auto paramValue = TValue(paramType, it->second.value()); + + valuesMap.emplace(it->first, paramValue); + } + + return valuesMap; +} + +TMaybe<TValue> TParams::TImpl::GetValue(const TString& name) const { + auto it = ParamsMap_.find(name); + if (it != ParamsMap_.end()) { + auto paramType = TType(it->second.type()); + return TValue(paramType, it->second.value()); + } + + return TMaybe<TValue>(); +} + +::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::TImpl::GetProtoMapPtr() { + return &ParamsMap_; +} + const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::TImpl::GetProtoMap() const { - return ParamsMap_; -} - -} // namespace NYdb + return ParamsMap_; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.h b/ydb/public/sdk/cpp/client/ydb_params/impl.h index 61eba25cd9..9e38d2463d 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.h +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include "params.h" - -namespace NYdb { - -class TParams::TImpl { -public: - TImpl(::google::protobuf::Map<TString, Ydb::TypedValue>&& paramsMap); - - bool Empty() const; - TMap<TString, TValue> GetValues() const; - TMaybe<TValue> GetValue(const TString& name) const; - ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); + +namespace NYdb { + +class TParams::TImpl { +public: + TImpl(::google::protobuf::Map<TString, Ydb::TypedValue>&& paramsMap); + + bool Empty() const; + TMap<TString, TValue> GetValues() const; + TMaybe<TValue> GetValue(const TString& name) const; + ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; - -private: - ::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_; -}; - -} // namespace NYdb + +private: + ::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.cpp b/ydb/public/sdk/cpp/client/ydb_params/params.cpp index 669b24da00..a69d29b2f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/params.cpp @@ -3,17 +3,17 @@ #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> - + #include <util/generic/map.h> #include <util/string/builder.h> - + namespace NYdb { - + //////////////////////////////////////////////////////////////////////////////// - -TParams::TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap) + +TParams::TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap) : Impl_(new TImpl(std::move(protoMap))) {} - + ::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::GetProtoMapPtr() { return Impl_->GetProtoMapPtr(); } @@ -25,55 +25,55 @@ const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::GetProtoMap() bool TParams::Empty() const { return Impl_->Empty(); } - + TMap<TString, TValue> TParams::GetValues() const { return Impl_->GetValues(); } - + TMaybe<TValue> TParams::GetValue(const TString& name) const { return Impl_->GetValue(name); } //////////////////////////////////////////////////////////////////////////////// - + class TParamsBuilder::TImpl { public: - TImpl() = default; - - TImpl(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo) - : HasTypeInfo_(true) + TImpl() = default; + + TImpl(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo) + : HasTypeInfo_(true) { for (const auto& pair : typeInfo) { ParamsMap_[pair.first].mutable_type()->CopyFrom(pair.second); } - } - - TImpl(const TMap<TString, TType>& typeInfo) - : HasTypeInfo_(true) + } + + TImpl(const TMap<TString, TType>& typeInfo) + : HasTypeInfo_(true) { for (const auto& pair : typeInfo) { ParamsMap_[pair.first].mutable_type()->CopyFrom(TProtoAccessor::GetProto(pair.second)); } - } - + } + bool HasTypeInfo() const { return HasTypeInfo_; - } - + } + TParamValueBuilder& AddParam(TParamsBuilder& owner, const TString& name) { auto param = GetParam(name); Y_VERIFY(param); - + auto result = ValueBuildersMap_.emplace(name, TParamValueBuilder(owner, *param->mutable_type(), - *param->mutable_value())); - + *param->mutable_value())); + return result.first->second; - } - + } + void AddParam(const TString& name, const TValue& value) { auto param = GetParam(name); Y_VERIFY(param); - + if (HasTypeInfo()) { if (!TypesEqual(param->type(), TProtoAccessor::GetProto(value.GetType()))) { FatalError(TStringBuilder() << "Type mismatch for parameter: " << name << ", expected: " @@ -82,10 +82,10 @@ public: } else { param->mutable_type()->CopyFrom(TProtoAccessor::GetProto(value.GetType())); } - + param->mutable_value()->CopyFrom(TProtoAccessor::GetProto(value)); - } - + } + TParams Build() { for (auto& pair : ValueBuildersMap_) { if (!pair.second.Finished()) { @@ -93,86 +93,86 @@ public: << ", call Build() on parameter value builder"); } } - + ValueBuildersMap_.clear(); - - ::google::protobuf::Map<TString, Ydb::TypedValue> paramsMap; + + ::google::protobuf::Map<TString, Ydb::TypedValue> paramsMap; paramsMap.swap(ParamsMap_); return TParams(std::move(paramsMap)); - } - + } + private: - Ydb::TypedValue* GetParam(const TString& name) { + Ydb::TypedValue* GetParam(const TString& name) { if (HasTypeInfo()) { auto it = ParamsMap_.find(name); if (it == ParamsMap_.end()) { FatalError(TStringBuilder() << "Parameter not found: " << name); return nullptr; } - + return &it->second; - } else { + } else { return &ParamsMap_[name]; - } - } - + } + } + void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TParamsBuilder: " << msg); + ThrowFatalError(TStringBuilder() << "TParamsBuilder: " << msg); } -private: +private: bool HasTypeInfo_ = false; - ::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_; + ::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_; TMap<TString, TParamValueBuilder> ValueBuildersMap_; -}; - +}; + //////////////////////////////////////////////////////////////////////////////// - -TParamValueBuilder::TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto) - : TValueBuilderBase(typeProto, valueProto) + +TParamValueBuilder::TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto) + : TValueBuilderBase(typeProto, valueProto) , Owner_(owner) , Finished_(false) {} - + bool TParamValueBuilder::Finished() { return Finished_; -} - +} + TParamsBuilder& TParamValueBuilder::Build() { CheckValue(); - + Finished_ = true; return Owner_; -} - +} + //////////////////////////////////////////////////////////////////////////////// - + TParamsBuilder::TParamsBuilder(TParamsBuilder&&) = default; TParamsBuilder::~TParamsBuilder() = default; - + TParamsBuilder::TParamsBuilder() - : Impl_(new TImpl()) {} - + : Impl_(new TImpl()) {} + TParamsBuilder::TParamsBuilder(const TMap<TString, TType>& typeInfo) - : Impl_(new TImpl(typeInfo)) {} - -TParamsBuilder::TParamsBuilder(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo) - : Impl_(new TImpl(typeInfo)) {} - + : Impl_(new TImpl(typeInfo)) {} + +TParamsBuilder::TParamsBuilder(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo) + : Impl_(new TImpl(typeInfo)) {} + bool TParamsBuilder::HasTypeInfo() const { return Impl_->HasTypeInfo(); -} - +} + TParamValueBuilder& TParamsBuilder::AddParam(const TString& name) { return Impl_->AddParam(*this, name); -} - +} + TParamsBuilder& TParamsBuilder::AddParam(const TString& name, const TValue& value) { Impl_->AddParam(name, value); - return *this; -} - -TParams TParamsBuilder::Build() { + return *this; +} + +TParams TParamsBuilder::Build() { return Impl_->Build(); -} - +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.h b/ydb/public/sdk/cpp/client/ydb_params/params.h index 34369eb72b..0f85f18329 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.h +++ b/ydb/public/sdk/cpp/client/ydb_params/params.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_value/value.h> - + #include <google/protobuf/map.h> namespace Ydb { - class TypedValue; -} - + class TypedValue; +} + namespace NYdb { - -namespace NScripting { - class TScriptingClient; -} - + +namespace NScripting { + class TScriptingClient; +} + namespace NTable { class TTableClient; class TSession; class TDataQuery; } - + namespace NExperimental { class TStreamQueryClient; } class TParamsBuilder; - -class TParams { + +class TParams { friend class TParamsBuilder; friend class NTable::TTableClient; friend class NTable::TSession; @@ -35,40 +35,40 @@ class TParams { friend class NExperimental::TStreamQueryClient; public: bool Empty() const; - + TMap<TString, TValue> GetValues() const; TMaybe<TValue> GetValue(const TString& name) const; private: - TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap); + TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap); ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; class TImpl; - std::shared_ptr<TImpl> Impl_; + std::shared_ptr<TImpl> Impl_; }; class TParamValueBuilder : public TValueBuilderBase<TParamValueBuilder> { - friend class TParamsBuilder; -public: + friend class TParamsBuilder; +public: TParamsBuilder& Build(); bool Finished(); -private: - TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto); +private: + TParamValueBuilder(TParamsBuilder& owner, Ydb::Type& typeProto, Ydb::Value& valueProto); TParamsBuilder& Owner_; bool Finished_; -}; - +}; + class TParamsBuilder : public TMoveOnly { friend class NTable::TDataQuery; -public: +public: TParamsBuilder(TParamsBuilder&&); - TParamsBuilder(); + TParamsBuilder(); TParamsBuilder(const TMap<TString, TType>& typeInfo); - + ~TParamsBuilder(); TParamValueBuilder& AddParam(const TString& name); @@ -76,13 +76,13 @@ public: bool HasTypeInfo() const; - TParams Build(); - -private: - TParamsBuilder(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo); + TParams Build(); - class TImpl; +private: + TParamsBuilder(const ::google::protobuf::Map<TString, Ydb::Type>& typeInfo); + + class TImpl; std::unique_ptr<TImpl> Impl_; -}; - +}; + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp index b8f6a95338..1b516de577 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp @@ -1,12 +1,12 @@ #include <ydb/public/lib/yson_value/ydb_yson_value.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - + using namespace NYdb; - -using TExpectedErrorException = yexception; + +using TExpectedErrorException = yexception; Y_UNIT_TEST_SUITE(ParamsBuilder) { Y_UNIT_TEST(Build) { @@ -35,8 +35,8 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { R"(String)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(*params.GetValue("$param2")), R"("test")"); - } - + } + Y_UNIT_TEST(BuildFromValue) { auto value2 = TValueBuilder() .BeginList() @@ -70,7 +70,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { R"(List<Tuple<Int32,String>?>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(*params.GetValue("$param2")), R"([[[-11;"test2"]];#])"); - } + } Y_UNIT_TEST(BuildWithTypeInfo) { auto param1Type = TTypeBuilder() @@ -114,7 +114,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { R"(Uint32?)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(*params.GetValue("$param2")), R"(#)"); - } + } Y_UNIT_TEST(MissingParam) { auto param1Type = TTypeBuilder() @@ -134,7 +134,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { paramsMap.emplace("$param2", param2Type); try { - auto params = TParamsBuilder(paramsMap) + auto params = TParamsBuilder(paramsMap) .AddParam("$param3") .EmptyOptional() .Build() @@ -144,10 +144,10 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { } UNIT_ASSERT(false); - } - + } + Y_UNIT_TEST(IncompleteParam) { - auto paramsBuilder = TParamsBuilder(); + auto paramsBuilder = TParamsBuilder(); auto& param1Builder = paramsBuilder.AddParam("$param1"); auto& param2Builder = paramsBuilder.AddParam("$param2"); @@ -172,7 +172,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { } UNIT_ASSERT(false); - } + } Y_UNIT_TEST(TypeMismatch) { auto param1Type = TTypeBuilder() @@ -192,7 +192,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { paramsMap.emplace("$param2", param2Type); try { - auto params = TParamsBuilder(paramsMap) + auto params = TParamsBuilder(paramsMap) .AddParam("$param1") .EmptyOptional() .Build() @@ -202,7 +202,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { } UNIT_ASSERT(false); - } + } Y_UNIT_TEST(TypeMismatchFromValue) { auto param1Type = TTypeBuilder() @@ -231,7 +231,7 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { .Build(); try { - auto params = TParamsBuilder(paramsMap) + auto params = TParamsBuilder(paramsMap) .AddParam("$param2", value1) .Build(); } catch (const TExpectedErrorException& e) { @@ -239,5 +239,5 @@ Y_UNIT_TEST_SUITE(ParamsBuilder) { } UNIT_ASSERT(false); - } -} + } +} diff --git a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make index e58119853b..cd5ee478c9 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_params) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/lib/yson_value -) - -SRCS( +) + +SRCS( params_ut.cpp -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp index 1a933139ce..bd466fd6f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp @@ -18,7 +18,7 @@ std::shared_ptr<IReadSession> TPersQueueClient::TImpl::CreateReadSession(const T } } auto session = std::make_shared<TReadSession>(maybeSettings.GetOrElse(settings), shared_from_this(), Connections_, DbDriverState_); - session->Start(); + session->Start(); return std::move(session); } @@ -41,7 +41,7 @@ std::shared_ptr<IWriteSession> TPersQueueClient::TImpl::CreateWriteSession( } } auto session = std::make_shared<TWriteSession>( - maybeSettings.GetOrElse(settings), shared_from_this(), Connections_, DbDriverState_ + maybeSettings.GetOrElse(settings), shared_from_this(), Connections_, DbDriverState_ ); session->Start(TDuration::Zero()); return std::move(session); @@ -59,7 +59,7 @@ std::shared_ptr<ISimpleBlockingWriteSession> TPersQueueClient::TImpl::CreateSimp } auto session = std::make_shared<TSimpleBlockingWriteSession>( - alteredSettings, shared_from_this(), Connections_, DbDriverState_ + alteredSettings, shared_from_this(), Connections_, DbDriverState_ ); return std::move(session); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h index 495735ef5a..57e9dd43bf 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h @@ -170,14 +170,14 @@ public: auto promise = NThreading::NewPromise<TDescribeTopicResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::PersQueue::V1::DescribeTopicResult result; if (any) { any->UnpackTo(&result); } - TDescribeTopicResult val(TStatus(std::move(status)), result); + TDescribeTopicResult val(TStatus(std::move(status)), result); promise.SetValue(std::move(val)); }; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp index 3f7d2e8fca..e7dd0a87e5 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp @@ -40,8 +40,8 @@ bool HasNullCounters(TReaderCounters& counters); class TErrorHandler : public IErrorHandler { public: - TErrorHandler(std::weak_ptr<TReadSession> session) - : Session(std::move(session)) + TErrorHandler(std::weak_ptr<TReadSession> session) + : Session(std::move(session)) { } @@ -91,8 +91,8 @@ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TReadSession::MakeClus return req; } -void TReadSession::Start() { - ErrorHandler = MakeIntrusive<TErrorHandler>(weak_from_this()); +void TReadSession::Start() { + ErrorHandler = MakeIntrusive<TErrorHandler>(weak_from_this()); EventsQueue = std::make_shared<TReadSessionEventsQueue>(Settings, weak_from_this()); if (!ValidateSettings()) { @@ -150,7 +150,7 @@ void TReadSession::StartClusterDiscovery() { if (any) { any->UnpackTo(&result); } - TStatus st(std::move(status)); + TStatus st(std::move(status)); selfShared->OnClusterDiscovery(st, result); }; @@ -403,7 +403,7 @@ bool TReadSession::Close(TDuration timeout) { NYql::TIssues issues; issues.AddIssue("Session was gracefully closed"); - EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); + EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); } else { ++*Settings.Counters_->Errors; for (const auto& session : sessions) { @@ -412,7 +412,7 @@ bool TReadSession::Close(TDuration timeout) { NYql::TIssues issues; issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); - EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); + EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); } with_lock (Lock) { @@ -437,7 +437,7 @@ void TReadSession::AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& } void TReadSession::AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred) { - AbortImpl(TSessionClosedEvent(statusCode, std::move(issues)), deferred); + AbortImpl(TSessionClosedEvent(statusCode, std::move(issues)), deferred); } void TReadSession::AbortImpl(EStatus statusCode, const TString& message, TDeferredActions& deferred) { @@ -447,7 +447,7 @@ void TReadSession::AbortImpl(EStatus statusCode, const TString& message, TDeferr } void TReadSession::Abort(EStatus statusCode, NYql::TIssues&& issues) { - Abort(TSessionClosedEvent(statusCode, std::move(issues))); + Abort(TSessionClosedEvent(statusCode, std::move(issues))); } void TReadSession::Abort(EStatus statusCode, const TString& message) { @@ -1499,7 +1499,7 @@ TSingleClusterReadSessionImpl::TPartitionCookieMapping::TCookie::TPtr TSingleClu UncommittedOffsetToCookie.erase(cookieIt); return cookie; } else { - ThrowFatalError(TStringBuilder() << "Invalid offset " << offset << ". Partition stream id: " << partitionStreamId << Endl); + ThrowFatalError(TStringBuilder() << "Invalid offset " << offset << ". Partition stream id: " << partitionStreamId << Endl); } // If offset wasn't found, there might be already hard released partition. // This situation is OK. @@ -2221,7 +2221,7 @@ void TDeferredActions::DeferAbortSession(const IErrorHandler::TPtr& errorHandler } void TDeferredActions::DeferAbortSession(const IErrorHandler::TPtr& errorHandler, EStatus statusCode, NYql::TIssues&& issues) { - DeferAbortSession(errorHandler, TSessionClosedEvent(statusCode, std::move(issues))); + DeferAbortSession(errorHandler, TSessionClosedEvent(statusCode, std::move(issues))); } void TDeferredActions::DeferAbortSession(const IErrorHandler::TPtr& errorHandler, EStatus statusCode, const TString& message) { @@ -2231,7 +2231,7 @@ void TDeferredActions::DeferAbortSession(const IErrorHandler::TPtr& errorHandler } void TDeferredActions::DeferAbortSession(const IErrorHandler::TPtr& errorHandler, TPlainStatus&& status) { - DeferAbortSession(errorHandler, TSessionClosedEvent(std::move(status))); + DeferAbortSession(errorHandler, TSessionClosedEvent(std::move(status))); } void TDeferredActions::DeferReconnection(std::shared_ptr<TSingleClusterReadSessionImpl> session, const IErrorHandler::TPtr& errorHandler, TPlainStatus&& status) { @@ -2489,9 +2489,9 @@ void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent::TM void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, TDisjointIntervalTree<ui64>& offsetSet, ui64 startOffset, ui64 endOffset) { if (offsetSet.Intersects(startOffset, endOffset)) { - ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval [" - << startOffset << "; " << endOffset - << ") for partition stream with id " << partitionStream->GetPartitionStreamId()); + ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval [" + << startOffset << "; " << endOffset + << ") for partition stream with id " << partitionStream->GetPartitionStreamId()); } else { offsetSet.InsertInterval(startOffset, endOffset); } @@ -2506,8 +2506,8 @@ void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, Y_ASSERT(partitionStream); auto& offsetSet = Offsets[partitionStream]; if (offsetSet.Has(offset)) { - ThrowFatalError(TStringBuilder() << "Commit set already has offset " << offset - << " for partition stream with id " << partitionStream->GetPartitionStreamId()); + ThrowFatalError(TStringBuilder() << "Commit set already has offset " << offset + << " for partition stream with id " << partitionStream->GetPartitionStreamId()); } else { offsetSet.Insert(offset); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h index f30bd8eb7c..b5b874954f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h @@ -31,7 +31,7 @@ struct IErrorHandler : public TThrRefBase { virtual void AbortSession(TSessionClosedEvent&& closeEvent) = 0; void AbortSession(EStatus statusCode, NYql::TIssues&& issues) { - AbortSession(TSessionClosedEvent(statusCode, std::move(issues))); + AbortSession(TSessionClosedEvent(statusCode, std::move(issues))); } void AbortSession(EStatus statusCode, const TString& message) { @@ -41,7 +41,7 @@ struct IErrorHandler : public TThrRefBase { } void AbortSession(TPlainStatus&& status) { - AbortSession(TSessionClosedEvent(std::move(status))); + AbortSession(TSessionClosedEvent(std::move(status))); } }; @@ -492,7 +492,7 @@ public: bool AddToCommitRanges(const ui64 startOffset, const ui64 endOffset, bool rangesMode) { if (ClientCommits.Intersects(startOffset, endOffset) || startOffset < MaxCommittedOffset) { - ThrowFatalError(TStringBuilder() << "Invalid offset range [" << startOffset << ", " << endOffset << ") : range must start from " + ThrowFatalError(TStringBuilder() << "Invalid offset range [" << startOffset << ", " << endOffset << ") : range must start from " << MaxCommittedOffset << " or has some offsets that are committed already. Partition stream id: " << PartitionStreamId << Endl); return false; } @@ -1020,7 +1020,7 @@ public: ~TReadSession(); - void Start(); + void Start(); NThreading::TFuture<void> WaitEvent() override; TVector<TReadSessionEvent::TEvent> GetEvents(bool block, TMaybe<size_t> maxEventsCount, size_t maxByteSize) override; @@ -1039,20 +1039,20 @@ public: void AddTopic(const TTopicReadSettings& topicReadSettings) /*override*/ { Y_UNUSED(topicReadSettings); // TODO: implement. - ThrowFatalError("Method \"AddTopic\" is not implemented"); + ThrowFatalError("Method \"AddTopic\" is not implemented"); } void RemoveTopic(const TString& path) /*override*/ { Y_UNUSED(path); // TODO: implement. - ThrowFatalError("Method \"RemoveTopic\" is not implemented"); + ThrowFatalError("Method \"RemoveTopic\" is not implemented"); } void RemoveTopic(const TString& path, const TVector<ui64>& partitionGruops) /*override*/ { Y_UNUSED(path); Y_UNUSED(partitionGruops); // TODO: implement. - ThrowFatalError("Method \"RemoveTopic\" is not implemented"); + ThrowFatalError("Method \"RemoveTopic\" is not implemented"); } void StopReadingData() override; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp index 30269bd7d0..af4e08b979 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp @@ -133,7 +133,7 @@ void TWriteSession::DoCdsRequest(TDuration delay) { if (any) { any->UnpackTo(&result); } - TStatus st(std::move(status)); + TStatus st(std::move(status)); if (auto sharedThis = weakThis.lock()) { sharedThis->OnCdsResponse(st, result); } @@ -267,7 +267,7 @@ NThreading::TFuture<ui64> TWriteSession::GetInitSeqNo() { if (Settings.ValidateSeqNo_) { if (AutoSeqNoMode.Defined() && *AutoSeqNoMode) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call GetInitSeqNo in Auto SeqNo mode"; - ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode"); + ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode"); } else AutoSeqNoMode = false; @@ -303,7 +303,7 @@ ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) { if (seqNo.Defined()) { if (*AutoSeqNoMode) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode"; - ThrowFatalError( + ThrowFatalError( "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" ); } else { @@ -311,7 +311,7 @@ ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) { } } else if (!(*AutoSeqNoMode)) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode"; - ThrowFatalError( + ThrowFatalError( "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" ); } @@ -1185,7 +1185,7 @@ void TWriteSession::AbortImpl() { void TWriteSession::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close"; - EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues))); + EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues))); AbortImpl(); } @@ -1197,7 +1197,7 @@ void TWriteSession::CloseImpl(EStatus statusCode, const TString& message) { void TWriteSession::CloseImpl(TPlainStatus&& status) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close"; - EventsQueue->Close(TSessionClosedEvent(std::move(status))); + EventsQueue->Close(TSessionClosedEvent(std::move(status))); AbortImpl(); } @@ -1228,7 +1228,7 @@ TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( alteredSettings.EventHandlers_.SessionClosedHandler_ = [this](const TSessionClosedEvent& event) {this->HandleClosed(event); }; Writer = std::make_shared<TWriteSession>( - alteredSettings, client, connections, dbDriverState + alteredSettings, client, connections, dbDriverState ); Writer->Start(TDuration::Max()); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compression_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compression_ut.cpp index 2ca10fdf3f..0603c709f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compression_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compression_ut.cpp @@ -1,165 +1,165 @@ -#include "ut_utils.h" - -namespace NYdb::NPersQueue::NTests { - -Y_UNIT_TEST_SUITE(Compression) { - TVector<TString> GetTestMessages(ECodec codec = ECodec::RAW) { - static const TVector<THashMap<ECodec, TString>> TEST_MESSAGES = { - { - {ECodec::RAW, "Alice and Bob"}, - {ECodec::GZIP, TString("\x1F\x8B\x08\x0\x0\x0\x0\x0\x0\x3\x73\xCC\xC9\x4C\x4E\x55\x48\xCC\x4B\x51\x70\xCA\x4F\x2\x0\x2C\xE7\x84\x5D\x0D\x0\x0\x0", 33)}, - {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x69\x0\x0\x41\x6C\x69\x63\x65\x20\x61\x6E\x64\x20\x42\x6F\x62", 22)} - }, - { - {ECodec::RAW, "Yandex.Cloud"}, - {ECodec::GZIP, TString("\x1F\x8B\x8\x0\x0\x0\x0\x0\x0\x3\x8B\x4C\xCC\x4B\x49\xAD\xD0\x73\xCE\xC9\x2F\x4D\x1\x0\x79\x91\x69\xCA\xC\x0\x0\x0", 32)}, - {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x61\x0\x0\x59\x61\x6E\x64\x65\x78\x2E\x43\x6C\x6F\x75\x64", 21)} - } - }; - TVector<TString> messages; - for (auto& m : TEST_MESSAGES) { - messages.push_back(m.at(codec)); - } - return messages; - } - - void AlterTopic(TPersQueueYdbSdkTestSetup& setup) { - std::shared_ptr<grpc::Channel> channel; - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> stub; - - { - channel = grpc::CreateChannel("localhost:" + ToString(setup.GetGrpcPort()), grpc::InsecureChannelCredentials()); - stub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); - } - - Ydb::PersQueue::V1::AlterTopicRequest request; - request.set_path(TStringBuilder() << "/Root/PQ/rt3.dc1--" << setup.GetTestTopic()); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); - props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); - props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); - auto rr = props->add_read_rules(); - rr->set_consumer_name(setup.GetTestClient()); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); - rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); - rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); - rr->set_version(1); - - Ydb::PersQueue::V1::AlterTopicResponse response; - grpc::ClientContext rcontext; - auto status = stub->AlterTopic(&rcontext, request, &response); - UNIT_ASSERT(status.ok()); - Ydb::PersQueue::V1::AlterTopicResult result; - response.operation().result().UnpackTo(&result); - Cerr << "Alter topic response: " << response << "\nAlter result: " << result << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - - void Write(TPersQueueYdbSdkTestSetup& setup, const TVector<TString>& messages, ECodec codec) { - auto& client = setup.GetPersQueueClient(); - TWriteSessionSettings writeSettings = setup.GetWriteSessionSettings(); - writeSettings.Codec(codec); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - - for (auto& message : messages) { - auto result = session->Write(message); - UNIT_ASSERT(result); - } - session->Close(); - } - - void Read( - TPersQueueYdbSdkTestSetup& setup, - const TVector<TString> messages, - const TVector<ECodec> codecs, - bool decompress - ) { - Cerr << Endl << "Start read" << Endl << Endl; - auto readSettings = setup.GetReadSessionSettings(); - readSettings.Decompress(decompress); - NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>(); - auto totalReceived = 0u; - readSettings.EventHandlers_.SimpleDataHandlers( - [&](const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { - Cerr << Endl << Endl << Endl << "Got messages" << Endl << Endl; - if (decompress) { - UNIT_ASSERT_NO_EXCEPTION(ev.GetMessages()); - UNIT_ASSERT_EXCEPTION(ev.GetCompressedMessages(), yexception); - for (auto& message : ev.GetMessages()) { - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); - ++totalReceived; - } - } else { - UNIT_ASSERT_EXCEPTION(ev.GetMessages(), yexception); - UNIT_ASSERT_NO_EXCEPTION(ev.GetCompressedMessages()); - for (auto& message : ev.GetCompressedMessages()) { - UNIT_ASSERT_VALUES_EQUAL(message.GetCodec(), codecs[totalReceived]); - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); - ++totalReceived; - } - } - Cerr << Endl << "totalReceived: " << totalReceived << " wait: " << messages.size() << Endl << Endl; - if (totalReceived == messages.size()) - checkedPromise.SetValue(); - } - ); - auto& client = setup.GetPersQueueClient(); - auto readSession = client.CreateReadSession(readSettings); - checkedPromise.GetFuture().GetValueSync(); - } - - void WriteWithOneCodec(TPersQueueYdbSdkTestSetup& setup, ECodec codec) { - AlterTopic(setup); // add zstd support - - auto messages = GetTestMessages(); - Write(setup, messages, codec); - Read(setup, messages, TVector<ECodec>(messages.size(), ECodec::RAW), true); - Read(setup, GetTestMessages(codec), TVector<ECodec>(messages.size(), codec), false); - } - - Y_UNIT_TEST(WriteRAW) { - TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); - WriteWithOneCodec(setup, ECodec::RAW); - } - - Y_UNIT_TEST(WriteGZIP) { - TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); - WriteWithOneCodec(setup, ECodec::GZIP); - } - - Y_UNIT_TEST(WriteZSTD) { - TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); - WriteWithOneCodec(setup, ECodec::ZSTD); - } - - Y_UNIT_TEST(WriteWithMixedCodecs) { - TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); - AlterTopic(setup); // add zstd support - - auto messages = GetTestMessages(); - TVector<TString> originalMessages; - TVector<TString> targetMessages; - TVector<ECodec> targetCodecs; - - auto addToTarget = [&](ECodec codec) { - originalMessages.insert(originalMessages.end(), messages.begin(), messages.end()); - for (auto& m : GetTestMessages(codec)) { - targetMessages.push_back(m); - targetCodecs.push_back(codec); - } - Write(setup, messages, codec); - }; - - addToTarget(ECodec::RAW); - addToTarget(ECodec::GZIP); - addToTarget(ECodec::ZSTD); - - Read(setup, originalMessages, TVector<ECodec>(originalMessages.size(), ECodec::RAW), true); - Read(setup, targetMessages, targetCodecs, false); - } -} -}; +#include "ut_utils.h" + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(Compression) { + TVector<TString> GetTestMessages(ECodec codec = ECodec::RAW) { + static const TVector<THashMap<ECodec, TString>> TEST_MESSAGES = { + { + {ECodec::RAW, "Alice and Bob"}, + {ECodec::GZIP, TString("\x1F\x8B\x08\x0\x0\x0\x0\x0\x0\x3\x73\xCC\xC9\x4C\x4E\x55\x48\xCC\x4B\x51\x70\xCA\x4F\x2\x0\x2C\xE7\x84\x5D\x0D\x0\x0\x0", 33)}, + {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x69\x0\x0\x41\x6C\x69\x63\x65\x20\x61\x6E\x64\x20\x42\x6F\x62", 22)} + }, + { + {ECodec::RAW, "Yandex.Cloud"}, + {ECodec::GZIP, TString("\x1F\x8B\x8\x0\x0\x0\x0\x0\x0\x3\x8B\x4C\xCC\x4B\x49\xAD\xD0\x73\xCE\xC9\x2F\x4D\x1\x0\x79\x91\x69\xCA\xC\x0\x0\x0", 32)}, + {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x61\x0\x0\x59\x61\x6E\x64\x65\x78\x2E\x43\x6C\x6F\x75\x64", 21)} + } + }; + TVector<TString> messages; + for (auto& m : TEST_MESSAGES) { + messages.push_back(m.at(codec)); + } + return messages; + } + + void AlterTopic(TPersQueueYdbSdkTestSetup& setup) { + std::shared_ptr<grpc::Channel> channel; + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> stub; + + { + channel = grpc::CreateChannel("localhost:" + ToString(setup.GetGrpcPort()), grpc::InsecureChannelCredentials()); + stub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + + Ydb::PersQueue::V1::AlterTopicRequest request; + request.set_path(TStringBuilder() << "/Root/PQ/rt3.dc1--" << setup.GetTestTopic()); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); + auto rr = props->add_read_rules(); + rr->set_consumer_name(setup.GetTestClient()); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); + rr->set_version(1); + + Ydb::PersQueue::V1::AlterTopicResponse response; + grpc::ClientContext rcontext; + auto status = stub->AlterTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + Ydb::PersQueue::V1::AlterTopicResult result; + response.operation().result().UnpackTo(&result); + Cerr << "Alter topic response: " << response << "\nAlter result: " << result << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + + void Write(TPersQueueYdbSdkTestSetup& setup, const TVector<TString>& messages, ECodec codec) { + auto& client = setup.GetPersQueueClient(); + TWriteSessionSettings writeSettings = setup.GetWriteSessionSettings(); + writeSettings.Codec(codec); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto& message : messages) { + auto result = session->Write(message); + UNIT_ASSERT(result); + } + session->Close(); + } + + void Read( + TPersQueueYdbSdkTestSetup& setup, + const TVector<TString> messages, + const TVector<ECodec> codecs, + bool decompress + ) { + Cerr << Endl << "Start read" << Endl << Endl; + auto readSettings = setup.GetReadSessionSettings(); + readSettings.Decompress(decompress); + NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>(); + auto totalReceived = 0u; + readSettings.EventHandlers_.SimpleDataHandlers( + [&](const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { + Cerr << Endl << Endl << Endl << "Got messages" << Endl << Endl; + if (decompress) { + UNIT_ASSERT_NO_EXCEPTION(ev.GetMessages()); + UNIT_ASSERT_EXCEPTION(ev.GetCompressedMessages(), yexception); + for (auto& message : ev.GetMessages()) { + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); + ++totalReceived; + } + } else { + UNIT_ASSERT_EXCEPTION(ev.GetMessages(), yexception); + UNIT_ASSERT_NO_EXCEPTION(ev.GetCompressedMessages()); + for (auto& message : ev.GetCompressedMessages()) { + UNIT_ASSERT_VALUES_EQUAL(message.GetCodec(), codecs[totalReceived]); + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); + ++totalReceived; + } + } + Cerr << Endl << "totalReceived: " << totalReceived << " wait: " << messages.size() << Endl << Endl; + if (totalReceived == messages.size()) + checkedPromise.SetValue(); + } + ); + auto& client = setup.GetPersQueueClient(); + auto readSession = client.CreateReadSession(readSettings); + checkedPromise.GetFuture().GetValueSync(); + } + + void WriteWithOneCodec(TPersQueueYdbSdkTestSetup& setup, ECodec codec) { + AlterTopic(setup); // add zstd support + + auto messages = GetTestMessages(); + Write(setup, messages, codec); + Read(setup, messages, TVector<ECodec>(messages.size(), ECodec::RAW), true); + Read(setup, GetTestMessages(codec), TVector<ECodec>(messages.size(), codec), false); + } + + Y_UNIT_TEST(WriteRAW) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::RAW); + } + + Y_UNIT_TEST(WriteGZIP) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::GZIP); + } + + Y_UNIT_TEST(WriteZSTD) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::ZSTD); + } + + Y_UNIT_TEST(WriteWithMixedCodecs) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + AlterTopic(setup); // add zstd support + + auto messages = GetTestMessages(); + TVector<TString> originalMessages; + TVector<TString> targetMessages; + TVector<ECodec> targetCodecs; + + auto addToTarget = [&](ECodec codec) { + originalMessages.insert(originalMessages.end(), messages.begin(), messages.end()); + for (auto& m : GetTestMessages(codec)) { + targetMessages.push_back(m); + targetCodecs.push_back(codec); + } + Write(setup, messages, codec); + }; + + addToTarget(ECodec::RAW); + addToTarget(ECodec::GZIP); + addToTarget(ECodec::ZSTD); + + Read(setup, originalMessages, TVector<ECodec>(originalMessages.size(), ECodec::RAW), true); + Read(setup, targetMessages, targetCodecs, false); + } +} +}; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp index eb530ef893..9b2367e3f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp @@ -327,7 +327,7 @@ struct TMockReadSessionProcessor : public TMockProcessorFactory<Ydb::PersQueue:: void Cancel() override { } - void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { + void ReadInitialMetadata(std::unordered_multimap<TString, TString>* metadata, TReadCallback callback) override { Y_UNUSED(metadata); Y_UNUSED(callback); UNIT_ASSERT_C(false, "This method is not expected to be called"); @@ -1618,7 +1618,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { // First time. dataEvent.GetMessages()[0].Commit(); - UNIT_ASSERT_EXCEPTION(dataEvent.GetMessages()[0].Commit(), NYdb::TContractViolation); + UNIT_ASSERT_EXCEPTION(dataEvent.GetMessages()[0].Commit(), NYdb::TContractViolation); } Y_UNIT_TEST(DataReceivedCallbackReal) { diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp index 64fa2066d2..bea557812c 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp @@ -43,7 +43,7 @@ TDescribeResourceResult::THierarchicalDrrProps::THierarchicalDrrProps(const Ydb: class TRateLimiterClient::TImpl : public TClientImplCommon<TRateLimiterClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl> connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) + : TClientImplCommon(std::move(connections), settings) { } @@ -112,7 +112,7 @@ public: auto promise = NThreading::NewPromise<TListResourcesResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TVector<TString> list; if (any) { @@ -124,7 +124,7 @@ public: } } - TListResourcesResult val(TStatus(std::move(status)), std::move(list)); + TListResourcesResult val(TStatus(std::move(status)), std::move(list)); promise.SetValue(std::move(val)); }; @@ -147,14 +147,14 @@ public: auto promise = NThreading::NewPromise<TDescribeResourceResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { Ydb::RateLimiter::DescribeResourceResult result; if (any) { any->UnpackTo(&result); } - TDescribeResourceResult val(TStatus(std::move(status)), result); + TDescribeResourceResult val(TStatus(std::move(status)), result); promise.SetValue(std::move(val)); }; diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.cpp b/ydb/public/sdk/cpp/client/ydb_result/result.cpp index 41ff4052eb..dc7eb09c0a 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/result.cpp @@ -1,17 +1,17 @@ #include "result.h" - + #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> #include <ydb/public/api/protos/ydb_common.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> - + #include <util/generic/map.h> #include <util/string/builder.h> #include <google/protobuf/text_format.h> - + namespace NYdb { - + TString TColumn::ToString() const { TString result; TStringOutput out(result); @@ -33,44 +33,44 @@ bool operator!=(const TColumn& col1, const TColumn& col2) { return !(col1 == col2); } -class TResultSet::TImpl { -public: +class TResultSet::TImpl { +public: TImpl(const Ydb::ResultSet& proto) : ProtoResultSet_(proto) - { + { Init(); - } - + } + TImpl(Ydb::ResultSet&& proto) : ProtoResultSet_(std::move(proto)) { Init(); - } - + } + void Init() { ColumnsMeta_.reserve(ProtoResultSet_.columns_size()); for (auto& meta : ProtoResultSet_.columns()) { ColumnsMeta_.push_back(TColumn(meta.name(), TType(meta.type()))); - } - } - + } + } + public: const Ydb::ResultSet ProtoResultSet_; TVector<TColumn> ColumnsMeta_; }; - + //////////////////////////////////////////////////////////////////////////////// - + TResultSet::TResultSet(const Ydb::ResultSet& proto) : Impl_(new TResultSet::TImpl(proto)) {} - + TResultSet::TResultSet(Ydb::ResultSet&& proto) : Impl_(new TResultSet::TImpl(std::move(proto))) {} - + size_t TResultSet::ColumnsCount() const { return Impl_->ColumnsMeta_.size(); } - + size_t TResultSet::RowsCount() const { return Impl_->ProtoResultSet_.rows_size(); } @@ -82,16 +82,16 @@ bool TResultSet::Truncated() const { const TVector<TColumn>& TResultSet::GetColumnsMeta() const { return Impl_->ColumnsMeta_; } - + const Ydb::ResultSet& TResultSet::GetProto() const { return Impl_->ProtoResultSet_; } - + //////////////////////////////////////////////////////////////////////////////// - + class TResultSetParser::TImpl { public: - TImpl(const TResultSet& resultSet) + TImpl(const TResultSet& resultSet) : ResultSet_(resultSet) { ColumnParsers.reserve(resultSet.ColumnsCount()); @@ -101,13 +101,13 @@ public: auto& column = columnsMeta[i]; ColumnIndexMap[column.Name] = i; ColumnParsers.emplace_back<TValueParser>(column.Type); - } - } - + } + } + size_t ColumnsCount() const { return ResultSet_.ColumnsCount(); - } - + } + size_t RowsCount() const { return ResultSet_.RowsCount(); } @@ -115,17 +115,17 @@ public: bool TryNextRow() { if (RowIndex_ == ResultSet_.RowsCount()) { return false; - } - + } + auto& row = ResultSet_.GetProto().rows()[RowIndex_]; for (size_t i = 0; i < ColumnsCount(); ++i) { ColumnParsers[i].Reset(row.items(i)); - } - + } + RowIndex_++; return true; - } - + } + ssize_t ColumnIndex(const TString& columnName) { auto idx = ColumnIndexMap.FindPtr(columnName); return idx ? static_cast<ssize_t>(*idx) : -1; @@ -134,11 +134,11 @@ public: TValueParser& ColumnParser(size_t columnIndex) { if (columnIndex >= ColumnParsers.size()) { FatalError(TStringBuilder() << "Column index out of bounds: " << columnIndex); - } - + } + return ColumnParsers[columnIndex]; - } - + } + TValueParser& ColumnParser(const TString& columnName) { auto idx = ColumnIndexMap.FindPtr(columnName); if (!idx) { @@ -148,80 +148,80 @@ public: return ColumnParser(*idx); } - TValue GetValue(size_t columnIndex) const { - if (columnIndex >= ColumnParsers.size()) { - FatalError(TStringBuilder() << "Column index out of bounds: " << columnIndex); - } - - if (RowIndex_ == 0) { - FatalError(TStringBuilder() << "Row position is undefined"); - } - - const auto& row = ResultSet_.GetProto().rows()[RowIndex_ - 1]; - const auto& valueType = ResultSet_.GetColumnsMeta()[columnIndex].Type; - - return TValue(valueType, row.items(columnIndex)); - } - - TValue GetValue(const TString& columnName) const { - auto idx = ColumnIndexMap.FindPtr(columnName); - if (!idx) { - FatalError(TStringBuilder() << "Unknown column: " << columnName); - } - - return GetValue(*idx); - } - + TValue GetValue(size_t columnIndex) const { + if (columnIndex >= ColumnParsers.size()) { + FatalError(TStringBuilder() << "Column index out of bounds: " << columnIndex); + } + + if (RowIndex_ == 0) { + FatalError(TStringBuilder() << "Row position is undefined"); + } + + const auto& row = ResultSet_.GetProto().rows()[RowIndex_ - 1]; + const auto& valueType = ResultSet_.GetColumnsMeta()[columnIndex].Type; + + return TValue(valueType, row.items(columnIndex)); + } + + TValue GetValue(const TString& columnName) const { + auto idx = ColumnIndexMap.FindPtr(columnName); + if (!idx) { + FatalError(TStringBuilder() << "Unknown column: " << columnName); + } + + return GetValue(*idx); + } + private: void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TResultSetParser: " << msg); - } - -private: + ThrowFatalError(TStringBuilder() << "TResultSetParser: " << msg); + } + +private: TResultSet ResultSet_; TMap<TString, size_t> ColumnIndexMap; TVector<TValueParser> ColumnParsers; size_t RowIndex_ = 0; -}; - +}; + //////////////////////////////////////////////////////////////////////////////// - + TResultSetParser::TResultSetParser(TResultSetParser&&) = default; TResultSetParser::~TResultSetParser() = default; TResultSetParser::TResultSetParser(const TResultSet& resultSet) - : Impl_(new TImpl(resultSet)) {} - + : Impl_(new TImpl(resultSet)) {} + size_t TResultSetParser::ColumnsCount() const { return Impl_->ColumnsCount(); -} - +} + size_t TResultSetParser::RowsCount() const { return Impl_->RowsCount(); } bool TResultSetParser::TryNextRow() { return Impl_->TryNextRow(); -} - +} + ssize_t TResultSetParser::ColumnIndex(const TString& columnName) { return Impl_->ColumnIndex(columnName); } TValueParser& TResultSetParser::ColumnParser(size_t columnIndex) { return Impl_->ColumnParser(columnIndex); -} - +} + TValueParser& TResultSetParser::ColumnParser(const TString& columnName) { return Impl_->ColumnParser(columnName); } -TValue TResultSetParser::GetValue(size_t columnIndex) const { - return Impl_->GetValue(columnIndex); -} - -TValue TResultSetParser::GetValue(const TString& columnName) const { - return Impl_->GetValue(columnName); -} - +TValue TResultSetParser::GetValue(size_t columnIndex) const { + return Impl_->GetValue(columnIndex); +} + +TValue TResultSetParser::GetValue(const TString& columnName) const { + return Impl_->GetValue(columnName); +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.h b/ydb/public/sdk/cpp/client/ydb_result/result.h index 1da6e7becf..287c8078be 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.h +++ b/ydb/public/sdk/cpp/client/ydb_result/result.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_value/value.h> - -#include <util/generic/vector.h> -#include <util/generic/string.h> - + +#include <util/generic/vector.h> +#include <util/generic/string.h> + namespace Ydb { - class ResultSet; -} - + class ResultSet; +} + namespace NYdb { class TProtoAccessor; -struct TColumn { - TString Name; - TType Type; +struct TColumn { + TString Name; + TType Type; TColumn(const TString& name, const TType& type) : Name(name) @@ -23,8 +23,8 @@ struct TColumn { TString ToString() const; void Out(IOutputStream& o) const; -}; - +}; + bool operator==(const TColumn& col1, const TColumn& col2); bool operator!=(const TColumn& col1, const TColumn& col2); @@ -32,13 +32,13 @@ bool operator!=(const TColumn& col1, const TColumn& col2); class TResultSet { friend class TResultSetParser; friend class NYdb::TProtoAccessor; -public: +public: TResultSet(const Ydb::ResultSet& proto); TResultSet(Ydb::ResultSet&& proto); - + //! Returns number of columns size_t ColumnsCount() const; - + //! Returns number of rows in result set (which is partial in case of stream operations) size_t RowsCount() const; @@ -47,26 +47,26 @@ public: //! Returns meta information (name, type) for columns const TVector<TColumn>& GetColumnsMeta() const; - + private: const Ydb::ResultSet& GetProto() const; - + private: class TImpl; std::shared_ptr<TImpl> Impl_; }; - + //! Note: TResultSetParser - mutable object, iteration thougth it changes internal state class TResultSetParser : public TMoveOnly { public: TResultSetParser(TResultSetParser&&); TResultSetParser(const TResultSet& resultSet); - + ~TResultSetParser(); //! Returns number of columns size_t ColumnsCount() const; - + //! Returns number of rows size_t RowsCount() const; @@ -74,7 +74,7 @@ public: //! On success TryNextRow will reset all column parsers to the values in next row. //! Column parsers are invalid before the first TryNextRow call. bool TryNextRow(); - + //! Returns index for column with specified name. //! If there is no column with such name, then -1 is returned. ssize_t ColumnIndex(const TString& columnName); @@ -82,30 +82,30 @@ public: //! Returns column value parser for column with specified index. //! State of the parser is preserved until next TryNextRow call. TValueParser& ColumnParser(size_t columnIndex); - + //! Returns column value parser for column with specified name. //! State of the parser is preserved until next TryNextRow call. TValueParser& ColumnParser(const TString& columnName); - //! Returns TValue for column with specified index. - //! TValue will have copy of coresponding data so this method - //! is less effective compare with - //! direct TValueParser constructed by ColumnParser call - TValue GetValue(size_t columnIndex) const; - - //! Returns TValue for column with specified name. - //! TValue will have copy of coresponding data so this method - //! is less effective compare with - //! direct TValueParser constructed by ColumnParser call - TValue GetValue(const TString& columnName) const; - -private: + //! Returns TValue for column with specified index. + //! TValue will have copy of coresponding data so this method + //! is less effective compare with + //! direct TValueParser constructed by ColumnParser call + TValue GetValue(size_t columnIndex) const; + + //! Returns TValue for column with specified name. + //! TValue will have copy of coresponding data so this method + //! is less effective compare with + //! direct TValueParser constructed by ColumnParser call + TValue GetValue(const TString& columnName) const; + +private: class TImpl; std::unique_ptr<TImpl> Impl_; -}; - -using TResultSets = TVector<TResultSet>; - +}; + +using TResultSets = TVector<TResultSet>; + } // namespace NYdb Y_DECLARE_OUT_SPEC(inline, NYdb::TColumn, o, x) { diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp index 1fc426978c..31f5535b96 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp @@ -13,43 +13,43 @@ namespace NYdb { namespace NScheme { using namespace NThreading; -using namespace Ydb::Scheme; - -static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) { - switch (entry) { - case ::Ydb::Scheme::Entry::DIRECTORY: - return ESchemeEntryType::Directory; - case ::Ydb::Scheme::Entry::TABLE: - return ESchemeEntryType::Table; - case ::Ydb::Scheme::Entry::PERS_QUEUE_GROUP: - return ESchemeEntryType::PqGroup; - case ::Ydb::Scheme::Entry::DATABASE: - return ESchemeEntryType::SubDomain; - case ::Ydb::Scheme::Entry::RTMR_VOLUME: - return ESchemeEntryType::RtmrVolume; - case ::Ydb::Scheme::Entry::BLOCK_STORE_VOLUME: - return ESchemeEntryType::BlockStoreVolume; - case ::Ydb::Scheme::Entry::COORDINATION_NODE: - return ESchemeEntryType::CoordinationNode; +using namespace Ydb::Scheme; + +static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) { + switch (entry) { + case ::Ydb::Scheme::Entry::DIRECTORY: + return ESchemeEntryType::Directory; + case ::Ydb::Scheme::Entry::TABLE: + return ESchemeEntryType::Table; + case ::Ydb::Scheme::Entry::PERS_QUEUE_GROUP: + return ESchemeEntryType::PqGroup; + case ::Ydb::Scheme::Entry::DATABASE: + return ESchemeEntryType::SubDomain; + case ::Ydb::Scheme::Entry::RTMR_VOLUME: + return ESchemeEntryType::RtmrVolume; + case ::Ydb::Scheme::Entry::BLOCK_STORE_VOLUME: + return ESchemeEntryType::BlockStoreVolume; + case ::Ydb::Scheme::Entry::COORDINATION_NODE: + return ESchemeEntryType::CoordinationNode; case ::Ydb::Scheme::Entry::SEQUENCE: return ESchemeEntryType::Sequence; case ::Ydb::Scheme::Entry::REPLICATION: return ESchemeEntryType::Replication; - default: - return ESchemeEntryType::Unknown; - } -} - -class TSchemeClient::TImpl : public TClientImplCommon<TSchemeClient::TImpl> { + default: + return ESchemeEntryType::Unknown; + } +} + +class TSchemeClient::TImpl : public TClientImplCommon<TSchemeClient::TImpl> { public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) {} + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) {} TAsyncStatus MakeDirectory(const TString& path, const TMakeDirectorySettings& settings) { auto request = MakeOperationRequest<Ydb::Scheme::MakeDirectoryRequest>(settings); request.set_path(path); - return RunSimple<Ydb::Scheme::V1::SchemeService, MakeDirectoryRequest, MakeDirectoryResponse>( + return RunSimple<Ydb::Scheme::V1::SchemeService, MakeDirectoryRequest, MakeDirectoryResponse>( std::move(request), &Ydb::Scheme::V1::SchemeService::Stub::AsyncMakeDirectory, TRpcRequestSettings::Make(settings), @@ -60,7 +60,7 @@ public: auto request = MakeOperationRequest<Ydb::Scheme::RemoveDirectoryRequest>(settings); request.set_path(path); - return RunSimple<Ydb::Scheme::V1::SchemeService, RemoveDirectoryRequest, RemoveDirectoryResponse>( + return RunSimple<Ydb::Scheme::V1::SchemeService, RemoveDirectoryRequest, RemoveDirectoryResponse>( std::move(request), &Ydb::Scheme::V1::SchemeService::Stub::AsyncRemoveDirectory, TRpcRequestSettings::Make(settings), @@ -73,124 +73,124 @@ public: auto promise = NThreading::NewPromise<TDescribePathResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { TSchemeEntry entry; if (any) { - DescribePathResult result; + DescribePathResult result; any->UnpackTo(&result); entry.Name = result.self().name(); entry.Owner = result.self().owner(); - entry.Type = ConvertProtoEntryType(result.self().type()); + entry.Type = ConvertProtoEntryType(result.self().type()); entry.SizeBytes = result.self().size_bytes(); - PermissionToSchemeEntry(result.self().effective_permissions(), &entry.EffectivePermissions); - PermissionToSchemeEntry(result.self().permissions(), &entry.Permissions); + PermissionToSchemeEntry(result.self().effective_permissions(), &entry.EffectivePermissions); + PermissionToSchemeEntry(result.self().permissions(), &entry.Permissions); } TDescribePathResult val(std::move(entry), - TStatus(std::move(status))); + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; - Connections_->RunDeferred<Ydb::Scheme::V1::SchemeService, DescribePathRequest, DescribePathResponse>( + Connections_->RunDeferred<Ydb::Scheme::V1::SchemeService, DescribePathRequest, DescribePathResponse>( std::move(request), extractor, - &Ydb::Scheme::V1::SchemeService::Stub::AsyncDescribePath, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Scheme::V1::SchemeService::Stub::AsyncDescribePath, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); return promise.GetFuture(); } - TAsyncListDirectoryResult ListDirectory(const TString& path, const TListDirectorySettings& settings) { + TAsyncListDirectoryResult ListDirectory(const TString& path, const TListDirectorySettings& settings) { auto request = MakeOperationRequest<Ydb::Scheme::ListDirectoryRequest>(settings); request.set_path(path); - auto promise = NThreading::NewPromise<TListDirectoryResult>(); + auto promise = NThreading::NewPromise<TListDirectoryResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { TSchemeEntry entry; TVector<TSchemeEntry> children; if (any) { - ListDirectoryResult result; + ListDirectoryResult result; any->UnpackTo(&result); entry.Name = result.self().name(); entry.Owner = result.self().owner(); - entry.Type = ConvertProtoEntryType(result.self().type()); + entry.Type = ConvertProtoEntryType(result.self().type()); for (const auto& child : result.children()) { TSchemeEntry tmp; tmp.Name = child.name(); tmp.Owner = child.owner(); - tmp.Type = ConvertProtoEntryType(child.type()); + tmp.Type = ConvertProtoEntryType(child.type()); children.push_back(tmp); } } - TListDirectoryResult val(std::move(children), std::move(entry), - TStatus(std::move(status))); + TListDirectoryResult val(std::move(children), std::move(entry), + TStatus(std::move(status))); promise.SetValue(std::move(val)); }; - Connections_->RunDeferred<Ydb::Scheme::V1::SchemeService, ListDirectoryRequest, ListDirectoryResponse>( + Connections_->RunDeferred<Ydb::Scheme::V1::SchemeService, ListDirectoryRequest, ListDirectoryResponse>( std::move(request), extractor, - &Ydb::Scheme::V1::SchemeService::Stub::AsyncListDirectory, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Scheme::V1::SchemeService::Stub::AsyncListDirectory, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); return promise.GetFuture(); } - - void PermissionsToRequest(const TPermissions& permissions, Permissions* to) { - to->set_subject(permissions.Subject); - for (const auto& perm : permissions.PermissionNames) { - to->add_permission_names(perm); - } - } - + + void PermissionsToRequest(const TPermissions& permissions, Permissions* to) { + to->set_subject(permissions.Subject); + for (const auto& perm : permissions.PermissionNames) { + to->add_permission_names(perm); + } + } + TAsyncStatus ModifyPermissions(const TString& path, const TModifyPermissionsSettings& settings) { auto request = MakeOperationRequest<Ydb::Scheme::ModifyPermissionsRequest>(settings); - request.set_path(path); + request.set_path(path); if (settings.ClearAcl_) { - request.set_clear_permissions(true); - } - + request.set_clear_permissions(true); + } + for (const auto& action : settings.Actions_) { - auto protoAction = request.add_actions(); - switch (action.first) { - case EModifyPermissionsAction::Chown: { - protoAction->set_change_owner(action.second.Subject); - } - break; - case EModifyPermissionsAction::Grant: { - PermissionsToRequest(action.second, protoAction->mutable_grant()); - } - break; - case EModifyPermissionsAction::Revoke: { - PermissionsToRequest(action.second, protoAction->mutable_revoke()); - } - break; - case EModifyPermissionsAction::Set: { - PermissionsToRequest(action.second, protoAction->mutable_set()); - } - break; - } - } - - return RunSimple<Ydb::Scheme::V1::SchemeService, ModifyPermissionsRequest, ModifyPermissionsResponse>( + auto protoAction = request.add_actions(); + switch (action.first) { + case EModifyPermissionsAction::Chown: { + protoAction->set_change_owner(action.second.Subject); + } + break; + case EModifyPermissionsAction::Grant: { + PermissionsToRequest(action.second, protoAction->mutable_grant()); + } + break; + case EModifyPermissionsAction::Revoke: { + PermissionsToRequest(action.second, protoAction->mutable_revoke()); + } + break; + case EModifyPermissionsAction::Set: { + PermissionsToRequest(action.second, protoAction->mutable_set()); + } + break; + } + } + + return RunSimple<Ydb::Scheme::V1::SchemeService, ModifyPermissionsRequest, ModifyPermissionsResponse>( std::move(request), &Ydb::Scheme::V1::SchemeService::Stub::AsyncModifyPermissions, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); - } - + } + }; //////////////////////////////////////////////////////////////////////////////// @@ -206,20 +206,20 @@ TSchemeEntry TDescribePathResult::GetEntry() const { //////////////////////////////////////////////////////////////////////////////// -TListDirectoryResult::TListDirectoryResult(TVector<TSchemeEntry>&& children, TSchemeEntry&& self, +TListDirectoryResult::TListDirectoryResult(TVector<TSchemeEntry>&& children, TSchemeEntry&& self, TStatus&& status) : TDescribePathResult(std::move(self), std::move(status)) , Children_(std::move(children)) {} -TVector<TSchemeEntry> TListDirectoryResult::GetChildren() const { - CheckStatusOk("TListDirectoryResult::GetChildren"); +TVector<TSchemeEntry> TListDirectoryResult::GetChildren() const { + CheckStatusOk("TListDirectoryResult::GetChildren"); return Children_; } //////////////////////////////////////////////////////////////////////////////// -TSchemeClient::TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +TSchemeClient::TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) {} TAsyncStatus TSchemeClient::MakeDirectory(const TString& path, const TMakeDirectorySettings& settings) { @@ -235,16 +235,16 @@ TAsyncDescribePathResult TSchemeClient::DescribePath(const TString& path, const } TAsyncListDirectoryResult TSchemeClient::ListDirectory(const TString& path, - const TListDirectorySettings& settings) + const TListDirectorySettings& settings) { - return Impl_->ListDirectory(path, settings); + return Impl_->ListDirectory(path, settings); } TAsyncStatus TSchemeClient::ModifyPermissions(const TString& path, - const TModifyPermissionsSettings& data) -{ - return Impl_->ModifyPermissions(path, data); -} - + const TModifyPermissionsSettings& data) +{ + return Impl_->ModifyPermissions(path, data); +} + } // namespace NScheme } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h index fb83214940..9d762b14c6 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h @@ -5,49 +5,49 @@ namespace NYdb { namespace NScheme { -//////////////////////////////////////////////////////////////////////////////// - -struct TPermissions { - TPermissions(const TString& subject) - : Subject(subject) - {} - TPermissions(const TString& subject, const TVector<TString>& names) - : Subject(subject) - , PermissionNames(names) - {} - TString Subject; - TVector<TString> PermissionNames; -}; - -enum class ESchemeEntryType : i32 { - Unknown = -1, - Directory = 1, - Table = 2, - PqGroup = 3, - SubDomain = 4, - RtmrVolume = 5, - BlockStoreVolume = 6, +//////////////////////////////////////////////////////////////////////////////// + +struct TPermissions { + TPermissions(const TString& subject) + : Subject(subject) + {} + TPermissions(const TString& subject, const TVector<TString>& names) + : Subject(subject) + , PermissionNames(names) + {} + TString Subject; + TVector<TString> PermissionNames; +}; + +enum class ESchemeEntryType : i32 { + Unknown = -1, + Directory = 1, + Table = 2, + PqGroup = 3, + SubDomain = 4, + RtmrVolume = 5, + BlockStoreVolume = 6, CoordinationNode = 7, Sequence = 15, Replication = 16, -}; - +}; + struct TSchemeEntry { TString Name; TString Owner; - ESchemeEntryType Type; - TVector<TPermissions> EffectivePermissions; - TVector<TPermissions> Permissions; + ESchemeEntryType Type; + TVector<TPermissions> EffectivePermissions; + TVector<TPermissions> Permissions; ui64 SizeBytes = 0; }; //////////////////////////////////////////////////////////////////////////////// class TDescribePathResult; -class TListDirectoryResult; +class TListDirectoryResult; using TAsyncDescribePathResult = NThreading::TFuture<TDescribePathResult>; -using TAsyncListDirectoryResult = NThreading::TFuture<TListDirectoryResult>; +using TAsyncListDirectoryResult = NThreading::TFuture<TListDirectoryResult>; //////////////////////////////////////////////////////////////////////////////// @@ -59,47 +59,47 @@ struct TDescribePathSettings : public TOperationRequestSettings<TDescribePathSet struct TListDirectorySettings : public TOperationRequestSettings<TListDirectorySettings> {}; -enum class EModifyPermissionsAction { - Grant, - Revoke, - Set, - Chown -}; - +enum class EModifyPermissionsAction { + Grant, + Revoke, + Set, + Chown +}; + struct TModifyPermissionsSettings : public TOperationRequestSettings<TModifyPermissionsSettings> { - TModifyPermissionsSettings& AddGrantPermissions(const TPermissions& permissions) { - AddAction(EModifyPermissionsAction::Grant, permissions); - return *this; - } - TModifyPermissionsSettings& AddRevokePermissions(const TPermissions& permissions) { - AddAction(EModifyPermissionsAction::Revoke, permissions); - return *this; - } - TModifyPermissionsSettings& AddSetPermissions(const TPermissions& permissions) { - AddAction(EModifyPermissionsAction::Set, permissions); - return *this; - } - TModifyPermissionsSettings& AddChangeOwner(const TString& owner) { - AddAction(EModifyPermissionsAction::Chown, TPermissions(owner)); - return *this; - } - TModifyPermissionsSettings& AddClearAcl() { - ClearAcl_ = true; - return *this; - } - - TVector<std::pair<EModifyPermissionsAction, TPermissions>> Actions_; - bool ClearAcl_ = false; - void AddAction(EModifyPermissionsAction action, const TPermissions& permissions) { - Actions_.emplace_back(std::pair<EModifyPermissionsAction, TPermissions>{action, permissions}); - } -}; - + TModifyPermissionsSettings& AddGrantPermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Grant, permissions); + return *this; + } + TModifyPermissionsSettings& AddRevokePermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Revoke, permissions); + return *this; + } + TModifyPermissionsSettings& AddSetPermissions(const TPermissions& permissions) { + AddAction(EModifyPermissionsAction::Set, permissions); + return *this; + } + TModifyPermissionsSettings& AddChangeOwner(const TString& owner) { + AddAction(EModifyPermissionsAction::Chown, TPermissions(owner)); + return *this; + } + TModifyPermissionsSettings& AddClearAcl() { + ClearAcl_ = true; + return *this; + } + + TVector<std::pair<EModifyPermissionsAction, TPermissions>> Actions_; + bool ClearAcl_ = false; + void AddAction(EModifyPermissionsAction action, const TPermissions& permissions) { + Actions_.emplace_back(std::pair<EModifyPermissionsAction, TPermissions>{action, permissions}); + } +}; + class TSchemeClient { class TImpl; public: - TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + TSchemeClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); TAsyncStatus MakeDirectory(const TString& path, const TMakeDirectorySettings& settings = TMakeDirectorySettings()); @@ -110,12 +110,12 @@ public: TAsyncDescribePathResult DescribePath(const TString& path, const TDescribePathSettings& settings = TDescribePathSettings()); - TAsyncListDirectoryResult ListDirectory(const TString& path, - const TListDirectorySettings& settings = TListDirectorySettings()); + TAsyncListDirectoryResult ListDirectory(const TString& path, + const TListDirectorySettings& settings = TListDirectorySettings()); + + TAsyncStatus ModifyPermissions(const TString& path, + const TModifyPermissionsSettings& data); - TAsyncStatus ModifyPermissions(const TString& path, - const TModifyPermissionsSettings& data); - private: std::shared_ptr<TImpl> Impl_; }; @@ -131,7 +131,7 @@ private: TSchemeEntry Entry_; }; -class TListDirectoryResult : public TDescribePathResult { +class TListDirectoryResult : public TDescribePathResult { public: TListDirectoryResult(TVector<TSchemeEntry>&& children, TSchemeEntry&& self, TStatus&& status); TVector<TSchemeEntry> GetChildren() const; diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h index b072603911..1c33edc333 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h @@ -1,40 +1,40 @@ -#pragma once +#pragma once #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h> #include <ydb/public/api/protos/ydb_table.pb.h> - + #include <library/cpp/cache/cache.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> + +#include <functional> -#include <functional> - namespace NYdb { namespace NTable { //////////////////////////////////////////////////////////////////////////////// -using TSessionInspectorFn = std::function<void(TAsyncCreateSessionResult future)>; - -class TSession::TImpl : public TEndpointObj { +using TSessionInspectorFn = std::function<void(TAsyncCreateSessionResult future)>; + +class TSession::TImpl : public TEndpointObj { friend class TTableClient; friend class TSession; -#ifdef YDB_IMPL_TABLE_CLIENT_SESSION_UT -public: -#endif +#ifdef YDB_IMPL_TABLE_CLIENT_SESSION_UT +public: +#endif TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize); -public: - enum EState { - S_STANDALONE, - S_IDLE, - S_BROKEN, - S_ACTIVE, +public: + enum EState { + S_STANDALONE, + S_IDLE, + S_BROKEN, + S_ACTIVE, S_DISCONNECTED, S_CLOSING - }; + }; struct TDataQueryInfo { TString QueryId; @@ -48,58 +48,58 @@ public: , ParameterTypes(parameterTypes) {} }; public: - ~TImpl(); - - const TString& GetId() const; - const TString& GetEndpoint() const; - void MarkBroken(); + ~TImpl(); + + const TString& GetId() const; + const TString& GetEndpoint() const; + void MarkBroken(); void MarkAsClosing(); - void MarkStandalone(); - void MarkActive(); - void MarkIdle(); - void MarkDisconnected(); - EState GetState() const; - void SetNeedUpdateActiveCounter(bool flag); - bool NeedUpdateActiveCounter() const; - void InvalidateQueryInCache(const TString& key); - void InvalidateQueryCache(); - TMaybe<TDataQueryInfo> GetQueryFromCache(const TString& query, bool allowMigration); - void AddQueryToCache(const TDataQuery& query); - void ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast); - void ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast); - TInstant GetTimeToTouchFast() const; - TInstant GetTimeInPastFast() const; - - // SetTimeInterval/GetTimeInterval, are not atomic! - void SetTimeInterval(TDuration interval); - TDuration GetTimeInterval() const; - - const TLRUCache<TString, TDataQueryInfo>& GetQueryCacheUnsafe() const; - + void MarkStandalone(); + void MarkActive(); + void MarkIdle(); + void MarkDisconnected(); + EState GetState() const; + void SetNeedUpdateActiveCounter(bool flag); + bool NeedUpdateActiveCounter() const; + void InvalidateQueryInCache(const TString& key); + void InvalidateQueryCache(); + TMaybe<TDataQueryInfo> GetQueryFromCache(const TString& query, bool allowMigration); + void AddQueryToCache(const TDataQuery& query); + void ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast); + void ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast); + TInstant GetTimeToTouchFast() const; + TInstant GetTimeInPastFast() const; + + // SetTimeInterval/GetTimeInterval, are not atomic! + void SetTimeInterval(TDuration interval); + TDuration GetTimeInterval() const; + + const TLRUCache<TString, TDataQueryInfo>& GetQueryCacheUnsafe() const; + static std::function<void(TSession::TImpl*)> GetSmartDeleter(std::shared_ptr<TTableClient::TImpl> client); - static TSessionInspectorFn GetSessionInspector( - NThreading::TPromise<TCreateSessionResult>& promise, - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings, - ui32 counter, bool needUpdateActiveSessionCounter); - -private: - const TString SessionId_; - const TString Endpoint_; - EState State_; + static TSessionInspectorFn GetSessionInspector( + NThreading::TPromise<TCreateSessionResult>& promise, + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings, + ui32 counter, bool needUpdateActiveSessionCounter); + +private: + const TString SessionId_; + const TString Endpoint_; + EState State_; bool UseQueryCache_; TLRUCache<TString, TDataQueryInfo> QueryCache_; - TAdaptiveLock Lock_; - TInstant TimeToTouch_; - TInstant TimeInPast_; - // Is used to implement progressive timeout for settler keep alive call - TDuration TimeInterval_; - // Indicate session was in active state, but state was changed (need to decrement active session counter) - // TODO: suboptimal because need lock for atomic change from interceptor - // Rewrite with bit field - bool NeedUpdateActiveCounter_; -}; - + TAdaptiveLock Lock_; + TInstant TimeToTouch_; + TInstant TimeInPast_; + // Is used to implement progressive timeout for settler keep alive call + TDuration TimeInterval_; + // Indicate session was in active state, but state was changed (need to decrement active session counter) + // TODO: suboptimal because need lock for atomic change from interceptor + // Rewrite with bit field + bool NeedUpdateActiveCounter_; +}; + } // namespace NTable } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp index bde453b34f..1484553c85 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp @@ -1,176 +1,176 @@ #include "request_migrator.h" - -#include <vector> -#include <numeric> -#include <cmath> - -namespace NYdb { - -namespace NMath { - -TStats CalcCV(const std::vector<size_t>& in) { - if (in.empty()) - return {0, 0.0}; - - if (in.size() == 1) - return {0, static_cast<float>(in[0])}; - - const size_t sum = std::accumulate(in.begin(), in.end(), 0); - if (!sum) - return {0, 0.0}; - - const float mean = sum / static_cast<float>(in.size()); - - double tmp = 0; - for (float t : in) { - t -= mean; - tmp += t * t; - } - - auto cv = static_cast<ui64>(llround(100.0f * sqrt(tmp / static_cast<float>(in.size() - 1)) / mean)); - return {cv, mean}; -} - -} // namespace NMath - -namespace NTable { - -constexpr TDuration MIGRATION_PREPARE_CLIENT_TIMEOUT = TDuration::Seconds(5); - -NThreading::TFuture<ui64> TRequestMigrator::SetHost(const TString& host, TSession targetSession) { + +#include <vector> +#include <numeric> +#include <cmath> + +namespace NYdb { + +namespace NMath { + +TStats CalcCV(const std::vector<size_t>& in) { + if (in.empty()) + return {0, 0.0}; + + if (in.size() == 1) + return {0, static_cast<float>(in[0])}; + + const size_t sum = std::accumulate(in.begin(), in.end(), 0); + if (!sum) + return {0, 0.0}; + + const float mean = sum / static_cast<float>(in.size()); + + double tmp = 0; + for (float t : in) { + t -= mean; + tmp += t * t; + } + + auto cv = static_cast<ui64>(llround(100.0f * sqrt(tmp / static_cast<float>(in.size() - 1)) / mean)); + return {cv, mean}; +} + +} // namespace NMath + +namespace NTable { + +constexpr TDuration MIGRATION_PREPARE_CLIENT_TIMEOUT = TDuration::Seconds(5); + +NThreading::TFuture<ui64> TRequestMigrator::SetHost(const TString& host, TSession targetSession) { std::lock_guard lock(Lock_); - CurHost_ = host; - TargetSession_.reset(new TSession(targetSession)); - if (!host) - return {}; - - Promise_ = NThreading::NewPromise<ui64>(); - return Promise_.GetFuture(); -} - -NThreading::TFuture<ui64> TRequestMigrator::SetHost(const TString& host) { + CurHost_ = host; + TargetSession_.reset(new TSession(targetSession)); + if (!host) + return {}; + + Promise_ = NThreading::NewPromise<ui64>(); + return Promise_.GetFuture(); +} + +NThreading::TFuture<ui64> TRequestMigrator::SetHost(const TString& host) { std::lock_guard lock(Lock_); - CurHost_ = host; - TargetSession_.reset(); - if (!host) - return {}; - - Promise_ = NThreading::NewPromise<ui64>(); - return Promise_.GetFuture(); -} - -NThreading::TFuture<ui64> TRequestMigrator::PrepareQuery(size_t id) { - const auto& query = Queries_[id]; - const auto settings = TPrepareDataQuerySettings() - .ClientTimeout(MIGRATION_PREPARE_CLIENT_TIMEOUT); - - return TargetSession_->PrepareDataQuery(query, settings) - .Apply([id, this](TAsyncPrepareQueryResult future) { - auto result = future.ExtractValue(); - if (!result.IsSuccess()) { - // In case of error SessionStatusInterception should change session state - return NThreading::MakeFuture<ui64>(id); - } else { - auto nextId = id + 1; - if (nextId == Queries_.size()) { - return NThreading::MakeFuture<ui64>(nextId); - } else { - return PrepareQuery(nextId); - } - } - }); -} - -std::function<void()> TRequestMigrator::CreateMigrationTask() { - return [this]() { - auto finishMigrationCb = [this](ui64 n) { - // Make copy to prevent race with set new promise - auto promise = Promise_; - // Release target session - TargetSession_.reset(); - promise.SetValue(n); - }; - - if (Queries_.empty() || !TargetSession_) { - finishMigrationCb(0); - } else { - PrepareQuery(0).Subscribe([finishMigrationCb](NThreading::TFuture<ui64> future) { - ui64 result = future.ExtractValue(); - finishMigrationCb(result); - }); - } - }; -} - -bool TRequestMigrator::IsOurSession(TSession::TImpl* session) const { - if (!CurHost_) - return false; - - if (session->GetEndpoint() != CurHost_) - return false; - - return true; -} - -bool TRequestMigrator::Reset() { + CurHost_ = host; + TargetSession_.reset(); + if (!host) + return {}; + + Promise_ = NThreading::NewPromise<ui64>(); + return Promise_.GetFuture(); +} + +NThreading::TFuture<ui64> TRequestMigrator::PrepareQuery(size_t id) { + const auto& query = Queries_[id]; + const auto settings = TPrepareDataQuerySettings() + .ClientTimeout(MIGRATION_PREPARE_CLIENT_TIMEOUT); + + return TargetSession_->PrepareDataQuery(query, settings) + .Apply([id, this](TAsyncPrepareQueryResult future) { + auto result = future.ExtractValue(); + if (!result.IsSuccess()) { + // In case of error SessionStatusInterception should change session state + return NThreading::MakeFuture<ui64>(id); + } else { + auto nextId = id + 1; + if (nextId == Queries_.size()) { + return NThreading::MakeFuture<ui64>(nextId); + } else { + return PrepareQuery(nextId); + } + } + }); +} + +std::function<void()> TRequestMigrator::CreateMigrationTask() { + return [this]() { + auto finishMigrationCb = [this](ui64 n) { + // Make copy to prevent race with set new promise + auto promise = Promise_; + // Release target session + TargetSession_.reset(); + promise.SetValue(n); + }; + + if (Queries_.empty() || !TargetSession_) { + finishMigrationCb(0); + } else { + PrepareQuery(0).Subscribe([finishMigrationCb](NThreading::TFuture<ui64> future) { + ui64 result = future.ExtractValue(); + finishMigrationCb(result); + }); + } + }; +} + +bool TRequestMigrator::IsOurSession(TSession::TImpl* session) const { + if (!CurHost_) + return false; + + if (session->GetEndpoint() != CurHost_) + return false; + + return true; +} + +bool TRequestMigrator::Reset() { if (Lock_.try_lock()) { - if (CurHost_) { - CurHost_.clear(); - TargetSession_.reset(); - Promise_.SetValue(0); + if (CurHost_) { + CurHost_.clear(); + TargetSession_.reset(); + Promise_.SetValue(0); Lock_.unlock(); - return true; - } else { + return true; + } else { Lock_.unlock(); - return false; - } - } else { - return false; - } -} - -bool TRequestMigrator::DoCheckAndMigrate(TSession::TImpl* session, std::shared_ptr<IMigratorClient> client) { - if (session->GetEndpoint().empty()) - return false; - + return false; + } + } else { + return false; + } +} + +bool TRequestMigrator::DoCheckAndMigrate(TSession::TImpl* session, std::shared_ptr<IMigratorClient> client) { + if (session->GetEndpoint().empty()) + return false; + if (Lock_.try_lock()) { - if (IsOurSession(session)) { - Queries_.clear(); - - const auto& queryCache = session->GetQueryCacheUnsafe(); - for (auto it = queryCache.Begin(); it != queryCache.End(); ++it) { - Queries_.emplace_back(it.Key()); - } - - // Force unlink session from ObjRegistry (if the session has been linked) - // this allow to solve logical race between host scan task and real session dtor - // which can cause double request migration - session->Unlink(); - - Finished_ = Promise_.GetFuture().Apply([](const NThreading::TFuture<ui64>&) { - return NThreading::MakeFuture(); - }); - - client->ScheduleTaskUnsafe(CreateMigrationTask(), TDuration()); - - // Clear host to prevent multiple migrations - CurHost_.clear(); + if (IsOurSession(session)) { + Queries_.clear(); + + const auto& queryCache = session->GetQueryCacheUnsafe(); + for (auto it = queryCache.Begin(); it != queryCache.End(); ++it) { + Queries_.emplace_back(it.Key()); + } + + // Force unlink session from ObjRegistry (if the session has been linked) + // this allow to solve logical race between host scan task and real session dtor + // which can cause double request migration + session->Unlink(); + + Finished_ = Promise_.GetFuture().Apply([](const NThreading::TFuture<ui64>&) { + return NThreading::MakeFuture(); + }); + + client->ScheduleTaskUnsafe(CreateMigrationTask(), TDuration()); + + // Clear host to prevent multiple migrations + CurHost_.clear(); Lock_.unlock(); - return true; - } else { + return true; + } else { Lock_.unlock(); - return false; - } - } else { - return false; - } -} - -void TRequestMigrator::Wait() const { + return false; + } + } else { + return false; + } +} + +void TRequestMigrator::Wait() const { std::lock_guard lock(Lock_); - if (Finished_.Initialized()) - Finished_.GetValueSync(); -} - -} // namespace NTable -} // namespace NYdb + if (Finished_.Initialized()) + Finished_.GetValueSync(); +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h index e9c2500d9f..81b89392be 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h @@ -1,80 +1,80 @@ -#pragma once - +#pragma once + #include "client_session.h" - + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <library/cpp/threading/future/future.h> - -#include <util/generic/string.h> - + +#include <util/generic/string.h> + #include <mutex> -#include <memory> - -namespace NYdb { - -namespace NMath { - -struct TStats { - const ui64 Cv; - const float Mean; -}; - -TStats CalcCV(const std::vector<size_t>&); - -} // namespace NMath - -namespace NTable { - -// TableClientImpl interface for migrator -// Migrator should be able -// - PrepareDataQuery -// - Schedule some cb to be executed -class IMigratorClient { -public: - virtual ~IMigratorClient() = default; - virtual TAsyncPrepareQueryResult PrepareDataQuery(const TSession& session, const TString& query, - const TPrepareDataQuerySettings& settings) = 0; - virtual void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) = 0; -}; - -class TRequestMigrator { -public: - // Set host to migrate session from. - // If the target session set requests will be reprepared on the new session in background. - // Real migration will be performed after DoCheckAndMigrate call - // IMPORTANT: SetHost methods are not reentrant. Moreover new call of this methods allowed only if - // future returned from previous call was set. - // The value of future means number of requests migrated - NThreading::TFuture<ui64> SetHost(const TString& host, TSession targetSession); - NThreading::TFuture<ui64> SetHost(const TString& host); - - // Checks and perform migration if the session suitable to be removed from host. - // Prepared requests will be migrated if target session was set along with host. - // Returns false if session is not suitable or unable to get lock to start migration - // Returns true if session is suitable in this case Unlink methos on the session is called - // This methos is thread safe. - bool DoCheckAndMigrate(TSession::TImpl* session, std::shared_ptr<IMigratorClient> client); - - // Wait current migration to be finished (if has one) - void Wait() const; - - // Reset migrator to initiall state if migration was not started and returns true - // Returns false if migration was started - bool Reset(); -private: - std::function<void()> CreateMigrationTask(); - bool IsOurSession(TSession::TImpl* session) const; - NThreading::TFuture<ui64> PrepareQuery(size_t id); - - TString CurHost_; - std::vector<TString> Queries_; - std::unique_ptr<TSession> TargetSession_; - +#include <memory> + +namespace NYdb { + +namespace NMath { + +struct TStats { + const ui64 Cv; + const float Mean; +}; + +TStats CalcCV(const std::vector<size_t>&); + +} // namespace NMath + +namespace NTable { + +// TableClientImpl interface for migrator +// Migrator should be able +// - PrepareDataQuery +// - Schedule some cb to be executed +class IMigratorClient { +public: + virtual ~IMigratorClient() = default; + virtual TAsyncPrepareQueryResult PrepareDataQuery(const TSession& session, const TString& query, + const TPrepareDataQuerySettings& settings) = 0; + virtual void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) = 0; +}; + +class TRequestMigrator { +public: + // Set host to migrate session from. + // If the target session set requests will be reprepared on the new session in background. + // Real migration will be performed after DoCheckAndMigrate call + // IMPORTANT: SetHost methods are not reentrant. Moreover new call of this methods allowed only if + // future returned from previous call was set. + // The value of future means number of requests migrated + NThreading::TFuture<ui64> SetHost(const TString& host, TSession targetSession); + NThreading::TFuture<ui64> SetHost(const TString& host); + + // Checks and perform migration if the session suitable to be removed from host. + // Prepared requests will be migrated if target session was set along with host. + // Returns false if session is not suitable or unable to get lock to start migration + // Returns true if session is suitable in this case Unlink methos on the session is called + // This methos is thread safe. + bool DoCheckAndMigrate(TSession::TImpl* session, std::shared_ptr<IMigratorClient> client); + + // Wait current migration to be finished (if has one) + void Wait() const; + + // Reset migrator to initiall state if migration was not started and returns true + // Returns false if migration was started + bool Reset(); +private: + std::function<void()> CreateMigrationTask(); + bool IsOurSession(TSession::TImpl* session) const; + NThreading::TFuture<ui64> PrepareQuery(size_t id); + + TString CurHost_; + std::vector<TString> Queries_; + std::unique_ptr<TSession> TargetSession_; + mutable std::mutex Lock_; - NThreading::TPromise<ui64> Promise_; - NThreading::TFuture<void> Finished_; -}; - -} // namespace NTable -} // namespace NYdb + NThreading::TPromise<ui64> Promise_; + NThreading::TFuture<void> Finished_; +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator_ut.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator_ut.cpp index 238b0a5bc8..714890a324 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator_ut.cpp @@ -1,231 +1,231 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/system/thread.h> -#include <util/thread/pool.h> - + +#include <util/system/thread.h> +#include <util/thread/pool.h> + #include <ydb/public/api/protos/ydb_value.pb.h> - -#define YDB_IMPL_TABLE_CLIENT_SESSION_UT 1 - + +#define YDB_IMPL_TABLE_CLIENT_SESSION_UT 1 + #include <ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h> #include <ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h> - -namespace NYdb { - -using namespace NTable; - -Y_UNIT_TEST_SUITE(MathTest) { - using namespace NMath; - Y_UNIT_TEST(CVCalc) { - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>()).Cv, 0); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{1}).Cv, 0); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{10, 20}).Cv, 47); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{11, 11, 11, 11}).Cv, 0); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{11, 11, 10, 12}).Cv, 7); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{0, 0, 0, 0}).Cv, 0); - UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{1, 4, 0, 0, 1, 0, 1, 1, 0}).Cv, 143); - } -} - -class TSessionLiveEmulator : public TThread { -public: - TSessionLiveEmulator(const TString& hostname, std::function<bool(TSession::TImpl*)> cb) - : TThread(&ThreadProc, this) - , Hostname_(hostname) - , Cb_(cb) - { + +namespace NYdb { + +using namespace NTable; + +Y_UNIT_TEST_SUITE(MathTest) { + using namespace NMath; + Y_UNIT_TEST(CVCalc) { + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>()).Cv, 0); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{1}).Cv, 0); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{10, 20}).Cv, 47); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{11, 11, 11, 11}).Cv, 0); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{11, 11, 10, 12}).Cv, 7); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{0, 0, 0, 0}).Cv, 0); + UNIT_ASSERT_VALUES_EQUAL(CalcCV(TVector<size_t>{1, 4, 0, 0, 1, 0, 1, 1, 0}).Cv, 143); + } +} + +class TSessionLiveEmulator : public TThread { +public: + TSessionLiveEmulator(const TString& hostname, std::function<bool(TSession::TImpl*)> cb) + : TThread(&ThreadProc, this) + , Hostname_(hostname) + , Cb_(cb) + { Cont_.store(true); - } - - static void* ThreadProc(void* _this) { - SetCurrentThreadName("TSessionLiveEmulator"); - static_cast<TSessionLiveEmulator*>(_this)->Exec(); - return nullptr; - } - - void Exec() { + } + + static void* ThreadProc(void* _this) { + SetCurrentThreadName("TSessionLiveEmulator"); + static_cast<TSessionLiveEmulator*>(_this)->Exec(); + return nullptr; + } + + void Exec() { while (Cont_.load()) { auto rowSession = new TSession::TImpl("someSessionId", Hostname_, true, 100); - if (Cb_(rowSession)) { - Processed_++; - } - delete rowSession; - } - } - - void Stop() { + if (Cb_(rowSession)) { + Processed_++; + } + delete rowSession; + } + } + + void Stop() { Cont_.store(false); - } - - ui64 GetProcessed() const { - return Processed_; - } - - const TString& GetHostname() const { - return Hostname_; - } - -private: + } + + ui64 GetProcessed() const { + return Processed_; + } + + const TString& GetHostname() const { + return Hostname_; + } + +private: std::atomic_bool Cont_; - const TString Hostname_; - std::function<bool(TSession::TImpl*)> Cb_; - ui64 Processed_ = 0; -}; - -class TMigratorClient : public IMigratorClient { -public: - TMigratorClient() + const TString Hostname_; + std::function<bool(TSession::TImpl*)> Cb_; + ui64 Processed_ = 0; +}; + +class TMigratorClient : public IMigratorClient { +public: + TMigratorClient() : Pool_(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false))) - { + { Scheduled_.store(0); - } - - void Stop() { - Pool_->Stop(); - } - - void Start() { - Pool_->Start(2, 1000); - } - - virtual TAsyncPrepareQueryResult PrepareDataQuery(const TSession& session, const TString& query, - const TPrepareDataQuerySettings& settings) override { - Y_UNUSED(session); - Y_UNUSED(query); - Y_UNUSED(settings); - Y_VERIFY(false); - return {}; - } - - void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) override { - Y_VERIFY(!timeout); + } + + void Stop() { + Pool_->Stop(); + } + + void Start() { + Pool_->Start(2, 1000); + } + + virtual TAsyncPrepareQueryResult PrepareDataQuery(const TSession& session, const TString& query, + const TPrepareDataQuerySettings& settings) override { + Y_UNUSED(session); + Y_UNUSED(query); + Y_UNUSED(settings); + Y_VERIFY(false); + return {}; + } + + void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) override { + Y_VERIFY(!timeout); ++Scheduled_; - Y_VERIFY(Pool_->AddFunc(std::move(fn))); - } - - size_t GetScheduledCount() const { + Y_VERIFY(Pool_->AddFunc(std::move(fn))); + } + + size_t GetScheduledCount() const { return Scheduled_.load(); - } -private: - std::unique_ptr<IThreadPool> Pool_; + } +private: + std::unique_ptr<IThreadPool> Pool_; std::atomic_int Scheduled_; -}; - - -Y_UNIT_TEST_SUITE(RequestMigratorTest) { - Y_UNIT_TEST(ThreadingNoMatch) { - TRequestMigrator migrator; - - std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); - - auto cb = [&migrator, client](TSession::TImpl* session) { - return migrator.DoCheckAndMigrate(session, client); - }; - - TSessionLiveEmulator liveEmulator[2] = {{"host0", cb}, {"host1", cb}}; - for (int i = 0; i < 2; i++) { - liveEmulator[i].Start(); - } - - Sleep(TDuration::MilliSeconds(100)); - for (int i = 0; i < 2; i++) { - liveEmulator[i].Stop(); - liveEmulator[i].Join(); - UNIT_ASSERT_VALUES_EQUAL(liveEmulator[i].GetProcessed(), 0); - } - } - - Y_UNIT_TEST(ShutDown) { - - std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); - - client->Start(); - - const int attempts = 100; - for (int i = 0; i < attempts; i++) { - TRequestMigrator migrator; - - migrator.SetHost("host1"); - +}; + + +Y_UNIT_TEST_SUITE(RequestMigratorTest) { + Y_UNIT_TEST(ThreadingNoMatch) { + TRequestMigrator migrator; + + std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); + + auto cb = [&migrator, client](TSession::TImpl* session) { + return migrator.DoCheckAndMigrate(session, client); + }; + + TSessionLiveEmulator liveEmulator[2] = {{"host0", cb}, {"host1", cb}}; + for (int i = 0; i < 2; i++) { + liveEmulator[i].Start(); + } + + Sleep(TDuration::MilliSeconds(100)); + for (int i = 0; i < 2; i++) { + liveEmulator[i].Stop(); + liveEmulator[i].Join(); + UNIT_ASSERT_VALUES_EQUAL(liveEmulator[i].GetProcessed(), 0); + } + } + + Y_UNIT_TEST(ShutDown) { + + std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); + + client->Start(); + + const int attempts = 100; + for (int i = 0; i < attempts; i++) { + TRequestMigrator migrator; + + migrator.SetHost("host1"); + auto session = new TSession::TImpl("someSessionId", "host1", true, 100); - UNIT_ASSERT(migrator.DoCheckAndMigrate(session, client)); - delete session; - - migrator.Wait(); - } - - UNIT_ASSERT_VALUES_EQUAL(client->GetScheduledCount(), attempts); - - client->Stop(); - } - - void DoThreadingMatchTest(bool withReset) { - TRequestMigrator migrator; - - std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); - - auto cb = [&migrator, client](TSession::TImpl* session) { - return migrator.DoCheckAndMigrate(session, client); - }; - - TSessionLiveEmulator liveEmulator[2] = {{"host0", cb}, {"host1", cb}}; - - client->Start(); - - // Start emulation of session live: create it and destroy in multiple threads. - for (int i = 0; i < 2; i++) { - liveEmulator[i].Start(); - } - - ui64 proposed = 0; - { - // Emulate balancing scan. - NThreading::TFuture<ui64> readyToMigrate; - for (int i = 0; i < 1000; i++) { - if (!withReset) - Sleep(TDuration::MilliSeconds(1)); - if (readyToMigrate.Initialized() && !readyToMigrate.HasValue()) { - if (withReset) { - if (migrator.Reset()) { - proposed--; - } - } - continue; - } - if (readyToMigrate.Initialized()) - UNIT_ASSERT_VALUES_EQUAL(readyToMigrate.GetValue(), 0); - proposed++; - readyToMigrate = migrator.SetHost("host1"); - } - if (readyToMigrate.Initialized()) { - // whait the last proposed host to be processed - while(!readyToMigrate.HasValue()) { - Sleep(TDuration::MilliSeconds(1)); - } - } - } - - for (int i = 0; i < 2; i++) { - liveEmulator[i].Stop(); - liveEmulator[i].Join(); - } - - client->Stop(); - - UNIT_ASSERT_VALUES_EQUAL(liveEmulator[0].GetProcessed(), 0); - UNIT_ASSERT_VALUES_EQUAL(liveEmulator[1].GetProcessed(), proposed); - } - - Y_UNIT_TEST(ThreadingMatch) { - DoThreadingMatchTest(false); - } - - Y_UNIT_TEST(ThreadingMatchWithReset) { - DoThreadingMatchTest(true); - } - - -} - -} + UNIT_ASSERT(migrator.DoCheckAndMigrate(session, client)); + delete session; + + migrator.Wait(); + } + + UNIT_ASSERT_VALUES_EQUAL(client->GetScheduledCount(), attempts); + + client->Stop(); + } + + void DoThreadingMatchTest(bool withReset) { + TRequestMigrator migrator; + + std::shared_ptr<TMigratorClient> client = std::make_shared<TMigratorClient>(); + + auto cb = [&migrator, client](TSession::TImpl* session) { + return migrator.DoCheckAndMigrate(session, client); + }; + + TSessionLiveEmulator liveEmulator[2] = {{"host0", cb}, {"host1", cb}}; + + client->Start(); + + // Start emulation of session live: create it and destroy in multiple threads. + for (int i = 0; i < 2; i++) { + liveEmulator[i].Start(); + } + + ui64 proposed = 0; + { + // Emulate balancing scan. + NThreading::TFuture<ui64> readyToMigrate; + for (int i = 0; i < 1000; i++) { + if (!withReset) + Sleep(TDuration::MilliSeconds(1)); + if (readyToMigrate.Initialized() && !readyToMigrate.HasValue()) { + if (withReset) { + if (migrator.Reset()) { + proposed--; + } + } + continue; + } + if (readyToMigrate.Initialized()) + UNIT_ASSERT_VALUES_EQUAL(readyToMigrate.GetValue(), 0); + proposed++; + readyToMigrate = migrator.SetHost("host1"); + } + if (readyToMigrate.Initialized()) { + // whait the last proposed host to be processed + while(!readyToMigrate.HasValue()) { + Sleep(TDuration::MilliSeconds(1)); + } + } + } + + for (int i = 0; i < 2; i++) { + liveEmulator[i].Stop(); + liveEmulator[i].Join(); + } + + client->Stop(); + + UNIT_ASSERT_VALUES_EQUAL(liveEmulator[0].GetProcessed(), 0); + UNIT_ASSERT_VALUES_EQUAL(liveEmulator[1].GetProcessed(), proposed); + } + + Y_UNIT_TEST(ThreadingMatch) { + DoThreadingMatchTest(false); + } + + Y_UNIT_TEST(ThreadingMatchWithReset) { + DoThreadingMatchTest(true); + } + + +} + +} diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 0eb81ad99d..610fec3884 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -17,29 +17,29 @@ #include <library/cpp/cache/cache.h> -#include <util/generic/map.h> -#include <util/random/random.h> +#include <util/generic/map.h> +#include <util/random/random.h> #include <util/string/join.h> -#include <unordered_map> - +#include <unordered_map> + namespace NYdb { namespace NTable { using namespace NThreading; -//How often run session pool keep alive check -constexpr TDuration PERIODIC_ACTION_INTERVAL = TDuration::Seconds(5); -//How often run settler keep alive check -constexpr TDuration SETTLER_PERIODIC_ACTION_INTERVAL = TDuration::Seconds(1); -//How ofter run host scan to perform session balancing -constexpr TDuration HOSTSCAN_PERIODIC_ACTION_INTERVAL = TDuration::Seconds(2); -constexpr TDuration MIGRATION_GET_SESSION_CLIENT_TIMEOUT = TDuration::Seconds(2); -constexpr ui64 KEEP_ALIVE_RANDOM_FRACTION = 4; -constexpr TDuration KEEP_ALIVE_CLIENT_TIMEOUT = TDuration::Seconds(5); -constexpr ui64 PERIODIC_ACTION_BATCH_SIZE = 10; //Max number of tasks to perform during one interval +//How often run session pool keep alive check +constexpr TDuration PERIODIC_ACTION_INTERVAL = TDuration::Seconds(5); +//How often run settler keep alive check +constexpr TDuration SETTLER_PERIODIC_ACTION_INTERVAL = TDuration::Seconds(1); +//How ofter run host scan to perform session balancing +constexpr TDuration HOSTSCAN_PERIODIC_ACTION_INTERVAL = TDuration::Seconds(2); +constexpr TDuration MIGRATION_GET_SESSION_CLIENT_TIMEOUT = TDuration::Seconds(2); +constexpr ui64 KEEP_ALIVE_RANDOM_FRACTION = 4; +constexpr TDuration KEEP_ALIVE_CLIENT_TIMEOUT = TDuration::Seconds(5); +constexpr ui64 PERIODIC_ACTION_BATCH_SIZE = 10; //Max number of tasks to perform during one interval constexpr ui32 MAX_BACKOFF_DURATION_MS = TDuration::Hours(1).MilliSeconds(); - + //////////////////////////////////////////////////////////////////////////////// class TStorageSettings::TImpl { @@ -155,22 +155,22 @@ TMaybe<bool> TColumnFamilyDescription::GetKeepInMemory() const { } } -TBuildIndexOperation::TBuildIndexOperation(TStatus &&status, Ydb::Operations::Operation &&operation) - : TOperation(std::move(status), std::move(operation)) -{ - Ydb::Table::IndexBuildMetadata metadata; - GetProto().metadata().UnpackTo(&metadata); - Metadata_.State = static_cast<EBuildIndexState>(metadata.state()); - Metadata_.Progress = metadata.progress(); - const auto& desc = metadata.description(); - Metadata_.Path = desc.path(); +TBuildIndexOperation::TBuildIndexOperation(TStatus &&status, Ydb::Operations::Operation &&operation) + : TOperation(std::move(status), std::move(operation)) +{ + Ydb::Table::IndexBuildMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + Metadata_.State = static_cast<EBuildIndexState>(metadata.state()); + Metadata_.Progress = metadata.progress(); + const auto& desc = metadata.description(); + Metadata_.Path = desc.path(); Metadata_.Desctiption = TProtoAccessor::FromProto(desc.index()); -} - -const TBuildIndexOperation::TMetadata& TBuildIndexOperation::Metadata() const { - return Metadata_; -} - +} + +const TBuildIndexOperation::TMetadata& TBuildIndexOperation::Metadata() const { + return Metadata_; +} + //////////////////////////////////////////////////////////////////////////////// class TPartitioningSettings::TImpl { @@ -233,20 +233,20 @@ ui64 TPartitioningSettings::GetMaxPartitionsCount() const { //////////////////////////////////////////////////////////////////////////////// -struct TTableStats { - ui64 Rows = 0; - ui64 Size = 0; - ui64 Partitions = 0; - TInstant ModificationTime; - TInstant CreationTime; -}; - -static TInstant ProtobufTimestampToTInstant(const NProtoBuf::Timestamp& timestamp) { - ui64 lastModificationUs = timestamp.seconds() * 1000000; - lastModificationUs += timestamp.nanos() / 1000; - return TInstant::MicroSeconds(lastModificationUs); -} - +struct TTableStats { + ui64 Rows = 0; + ui64 Size = 0; + ui64 Partitions = 0; + TInstant ModificationTime; + TInstant CreationTime; +}; + +static TInstant ProtobufTimestampToTInstant(const NProtoBuf::Timestamp& timestamp) { + ui64 lastModificationUs = timestamp.seconds() * 1000000; + lastModificationUs += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(lastModificationUs); +} + class TTableDescription::TImpl { using EUnit = TValueSinceUnixEpochModeSettings::EUnit; @@ -271,7 +271,7 @@ class TTableDescription::TImpl { Indexes_.reserve(proto.indexesSize()); for (const auto& index : proto.indexes()) { Indexes_.emplace_back(TProtoAccessor::FromProto(index)); - } + } // ttl settings switch (proto.ttl_settings().mode_case()) { @@ -345,33 +345,33 @@ public: Owner_ = Proto_.self().owner(); PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); - + TMaybe<TValue> leftValue; for (const auto& bound : Proto_.shard_key_bounds()) { TMaybe<TKeyBound> fromBound = leftValue ? TKeyBound::Inclusive(*leftValue) : TMaybe<TKeyBound>(); - - TValue value(TType(bound.type()), bound.value()); - const TKeyBound& toBound = TKeyBound::Exclusive(value); - - Ranges_.emplace_back(TKeyRange(fromBound, toBound)); - leftValue = value; - } + + TValue value(TType(bound.type()), bound.value()); + const TKeyBound& toBound = TKeyBound::Exclusive(value); + + Ranges_.emplace_back(TKeyRange(fromBound, toBound)); + leftValue = value; + } for (const auto& shardStats : Proto_.table_stats().partition_stats()) { - PartitionStats_.emplace_back( - TPartitionStats{shardStats.rows_estimate(), shardStats.store_size()} - ); - } - + PartitionStats_.emplace_back( + TPartitionStats{shardStats.rows_estimate(), shardStats.store_size()} + ); + } + TableStats.Rows = Proto_.table_stats().rows_estimate(); TableStats.Size = Proto_.table_stats().store_size(); TableStats.Partitions = Proto_.table_stats().partitions(); - + TableStats.ModificationTime = ProtobufTimestampToTInstant(Proto_.table_stats().modification_time()); TableStats.CreationTime = ProtobufTimestampToTInstant(Proto_.table_stats().creation_time()); - + if (describeSettings.WithKeyShardBoundary_) { Ranges_.emplace_back(TKeyRange( leftValue ? TKeyBound::Inclusive(*leftValue) : TMaybe<TKeyBound>(), @@ -425,12 +425,12 @@ public: void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns)); - } - + } + void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); - } - + } + void SetTtlSettings(TTtlSettings&& settings) { TtlSettings_ = std::move(settings); } @@ -493,38 +493,38 @@ public: return Columns_; } - const TVector<TIndexDescription>& GetIndexDescriptions() const { - return Indexes_; - } - + const TVector<TIndexDescription>& GetIndexDescriptions() const { + return Indexes_; + } + const TMaybe<TTtlSettings>& GetTtlSettings() const { return TtlSettings_; } - const TString& GetOwner() const { - return Owner_; - } - + const TString& GetOwner() const { + return Owner_; + } + const TVector<NScheme::TPermissions>& GetPermissions() const { return Permissions_; } - const TVector<NScheme::TPermissions>& GetEffectivePermissions() const { - return EffectivePermissions_; - } - - const TVector<TKeyRange>& GetKeyRanges() const { - return Ranges_; - } - - const TVector<TPartitionStats>& GetPartitionStats() const { - return PartitionStats_; - } - - const TTableStats& GetTableStats() const { - return TableStats; - } - + const TVector<NScheme::TPermissions>& GetEffectivePermissions() const { + return EffectivePermissions_; + } + + const TVector<TKeyRange>& GetKeyRanges() const { + return Ranges_; + } + + const TVector<TPartitionStats>& GetPartitionStats() const { + return PartitionStats_; + } + + const TTableStats& GetTableStats() const { + return TableStats; + } + bool HasStorageSettings() const { return HasStorageSettings_; } @@ -574,14 +574,14 @@ private: TStorageSettings StorageSettings_; TVector<TString> PrimaryKey_; TVector<TTableColumn> Columns_; - TVector<TIndexDescription> Indexes_; + TVector<TIndexDescription> Indexes_; TMaybe<TTtlSettings> TtlSettings_; - TString Owner_; + TString Owner_; TVector<NScheme::TPermissions> Permissions_; - TVector<NScheme::TPermissions> EffectivePermissions_; - TVector<TKeyRange> Ranges_; - TVector<TPartitionStats> PartitionStats_; - TTableStats TableStats; + TVector<NScheme::TPermissions> EffectivePermissions_; + TVector<TKeyRange> Ranges_; + TVector<TPartitionStats> PartitionStats_; + TTableStats TableStats; TVector<TColumnFamilyDescription> ColumnFamilies_; THashMap<TString, TString> Attributes_; TString CompactionPolicy_; @@ -628,30 +628,30 @@ TVector<TTableColumn> TTableDescription::GetTableColumns() const { return Impl_->GetColumns(); } -TVector<TIndexDescription> TTableDescription::GetIndexDescriptions() const { - return Impl_->GetIndexDescriptions(); -} - +TVector<TIndexDescription> TTableDescription::GetIndexDescriptions() const { + return Impl_->GetIndexDescriptions(); +} + TMaybe<TTtlSettings> TTableDescription::GetTtlSettings() const { return Impl_->GetTtlSettings(); } const TString& TTableDescription::GetOwner() const { - return Impl_->GetOwner(); -} - + return Impl_->GetOwner(); +} + const TVector<NScheme::TPermissions>& TTableDescription::GetPermissions() const { return Impl_->GetPermissions(); } const TVector<NScheme::TPermissions>& TTableDescription::GetEffectivePermissions() const { - return Impl_->GetEffectivePermissions(); -} - -const TVector<TKeyRange>& TTableDescription::GetKeyRanges() const { - return Impl_->GetKeyRanges(); -} - + return Impl_->GetEffectivePermissions(); +} + +const TVector<TKeyRange>& TTableDescription::GetKeyRanges() const { + return Impl_->GetKeyRanges(); +} + void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family) { Impl_->AddColumn(name, type, family); } @@ -670,12 +670,12 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); -} - +} + void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); -} - +} + void TTableDescription::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); } @@ -744,30 +744,30 @@ void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mod Impl_->SetReadReplicasSettings(mode, readReplicasCount); } -const TVector<TPartitionStats>& TTableDescription::GetPartitionStats() const { - return Impl_->GetPartitionStats(); -} - -TInstant TTableDescription::GetModificationTime() const { - return Impl_->GetTableStats().ModificationTime; -} - -TInstant TTableDescription::GetCreationTime() const { - return Impl_->GetTableStats().CreationTime; -} - -ui64 TTableDescription::GetTableSize() const { - return Impl_->GetTableStats().Size; -} - -ui64 TTableDescription::GetTableRows() const { - return Impl_->GetTableStats().Rows; -} - -ui64 TTableDescription::GetPartitionsCount() const { - return Impl_->GetTableStats().Partitions; -} - +const TVector<TPartitionStats>& TTableDescription::GetPartitionStats() const { + return Impl_->GetPartitionStats(); +} + +TInstant TTableDescription::GetModificationTime() const { + return Impl_->GetTableStats().ModificationTime; +} + +TInstant TTableDescription::GetCreationTime() const { + return Impl_->GetTableStats().CreationTime; +} + +ui64 TTableDescription::GetTableSize() const { + return Impl_->GetTableStats().Size; +} + +ui64 TTableDescription::GetTableRows() const { + return Impl_->GetTableStats().Rows; +} + +ui64 TTableDescription::GetPartitionsCount() const { + return Impl_->GetTableStats().Partitions; +} + const TStorageSettings& TTableDescription::GetStorageSettings() const { return Impl_->GetStorageSettings(); } @@ -1006,15 +1006,15 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const EPrim } TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDecimalType& type, const TString& family) { - auto columnType = TTypeBuilder() - .BeginOptional() - .Decimal(type) - .EndOptional() - .Build(); + auto columnType = TTypeBuilder() + .BeginOptional() + .Decimal(type) + .EndOptional() + .Build(); TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); - return *this; -} - + return *this; +} + TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family) { auto columnType = TTypeBuilder() .Primitive(type) @@ -1045,19 +1045,19 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); - return *this; -} - + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns); - return *this; -} - + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn) { TableDescription_.AddSecondaryIndex(indexName, type, TVector<TString>{indexColumn}); - return *this; -} - + return *this; +} + TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); } @@ -1173,70 +1173,70 @@ TTableDescription TTableBuilder::Build() { //////////////////////////////////////////////////////////////////////////////// -class TTablePartIterator::TReaderImpl { -public: - using TSelf = TTablePartIterator::TReaderImpl; - using TResponse = Ydb::Table::ReadTableResponse; +class TTablePartIterator::TReaderImpl { +public: + using TSelf = TTablePartIterator::TReaderImpl; + using TResponse = Ydb::Table::ReadTableResponse; using TStreamProcessorPtr = NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; using TGRpcStatus = NGrpc::TGrpcStatus; - using TBatchReadResult = std::pair<TResponse, TGRpcStatus>; - - TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) - : StreamProcessor_(streamProcessor) - , Finished_(false) - , Endpoint_(endpoint) - {} - - ~TReaderImpl() { - StreamProcessor_->Cancel(); - } - - bool IsFinished() { - return Finished_; - } - + using TBatchReadResult = std::pair<TResponse, TGRpcStatus>; + + TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) + {} + + ~TReaderImpl() { + StreamProcessor_->Cancel(); + } + + bool IsFinished() { + return Finished_; + } + TAsyncSimpleStreamPart<TResultSet> ReadNext(std::shared_ptr<TSelf> self) { auto promise = NThreading::NewPromise<TSimpleStreamPart<TResultSet>>(); - // Capture self - guarantee no dtor call during the read - auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { - if (!grpcStatus.Ok()) { - self->Finished_ = true; - promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), - TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); - } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); - EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); - promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), - TStatus(clientStatus, std::move(issues))}); - } - }; - StreamProcessor_->Read(&Response_, readCb); - return promise.GetFuture(); - } -private: - TStreamProcessorPtr StreamProcessor_; - TResponse Response_; - bool Finished_; - TString Endpoint_; -}; - -TTablePartIterator::TTablePartIterator( - std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status) - : TStatus(std::move(status)) - , ReaderImpl_(impl) -{} - + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + if (!grpcStatus.Ok()) { + self->Finished_ = true; + promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), + TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); + } else { + NYql::TIssues issues; + NYql::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); + promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), + TStatus(clientStatus, std::move(issues))}); + } + }; + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); + } +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + TString Endpoint_; +}; + +TTablePartIterator::TTablePartIterator( + std::shared_ptr<TReaderImpl> impl, + TPlainStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) +{} + TAsyncSimpleStreamPart<TResultSet> TTablePartIterator::ReadNext() { - if (ReaderImpl_->IsFinished()) - RaiseError("Attempt to perform read on invalid or finished stream"); - return ReaderImpl_->ReadNext(ReaderImpl_); -} - -//////////////////////////////////////////////////////////////////////////////// - + if (ReaderImpl_->IsFinished()) + RaiseError("Attempt to perform read on invalid or finished stream"); + return ReaderImpl_->ReadNext(ReaderImpl_); +} + +//////////////////////////////////////////////////////////////////////////////// + class TScanQueryPartIterator::TReaderImpl { public: using TSelf = TScanQueryPartIterator::TReaderImpl; @@ -1246,7 +1246,7 @@ public: using TGRpcStatus = NGrpc::TGrpcStatus; using TBatchReadResult = std::pair<TResponse, TGRpcStatus>; - TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) + TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) : StreamProcessor_(streamProcessor) , Finished_(false) , Endpoint_(endpoint) @@ -1266,14 +1266,14 @@ public: auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { if (!grpcStatus.Ok()) { self->Finished_ = true; - promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); + promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); } else { NYql::TIssues issues; NYql::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); - // TODO: Add headers for streaming calls. - TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; - TStatus status{std::move(plainStatus)}; + // TODO: Add headers for streaming calls. + TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; + TStatus status{std::move(plainStatus)}; TMaybe<TQueryStats> queryStats; if (self->Response_.result().has_query_stats()) { @@ -1300,8 +1300,8 @@ private: TScanQueryPartIterator::TScanQueryPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status) - : TStatus(std::move(status)) + TPlainStatus&& status) + : TStatus(std::move(status)) , ReaderImpl_(impl) {} @@ -1313,78 +1313,78 @@ TAsyncScanQueryPart TScanQueryPartIterator::ReadNext() { //////////////////////////////////////////////////////////////////////////////// -class TSessionPoolImpl { - typedef TAsyncCreateSessionResult - (*TAwareSessonProvider) - (std::shared_ptr<TTableClient::TImpl> client, const TCreateSessionSettings& settings); -public: - using TKeepAliveCmd = std::function<void(TSession session)>; +class TSessionPoolImpl { + typedef TAsyncCreateSessionResult + (*TAwareSessonProvider) + (std::shared_ptr<TTableClient::TImpl> client, const TCreateSessionSettings& settings); +public: + using TKeepAliveCmd = std::function<void(TSession session)>; using TDeletePredicate = std::function<bool(TSession::TImpl* session, TTableClient::TImpl* client, size_t sessionsCount)>; - TSessionPoolImpl(ui32 maxActiveSessions); - // TAwareSessonProvider: - // function is called if session pool is empty, - // this is used for additional total session count limitation - TAsyncCreateSessionResult GetSession( - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings, - TAwareSessonProvider sessionProvider); - // Returns true if session was extracted from session pool and dropped via smart deleter - // Returns false if session for given endpoint was not found - // NOTE: O(n) under session pool lock, should not be used often - bool DropSessionOnEndpoint(std::shared_ptr<TTableClient::TImpl> client, const TString& endpoint); - // Returns true if session returned to pool successfully - bool ReturnSession(TSession::TImpl* impl, bool active); + TSessionPoolImpl(ui32 maxActiveSessions); + // TAwareSessonProvider: + // function is called if session pool is empty, + // this is used for additional total session count limitation + TAsyncCreateSessionResult GetSession( + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings, + TAwareSessonProvider sessionProvider); + // Returns true if session was extracted from session pool and dropped via smart deleter + // Returns false if session for given endpoint was not found + // NOTE: O(n) under session pool lock, should not be used often + bool DropSessionOnEndpoint(std::shared_ptr<TTableClient::TImpl> client, const TString& endpoint); + // Returns true if session returned to pool successfully + bool ReturnSession(TSession::TImpl* impl, bool active); TPeriodicCb CreatePeriodicTask(std::weak_ptr<TTableClient::TImpl> weakClient, TKeepAliveCmd&& cmd, TDeletePredicate&& predicate); - i64 GetActiveSessions() const; - i64 GetActiveSessionsLimit() const; - i64 GetCurrentPoolSize() const; - void DecrementActiveCounter(); - - void Drain(std::function<bool(std::unique_ptr<TSession::TImpl>&&)> cb, bool close); + i64 GetActiveSessions() const; + i64 GetActiveSessionsLimit() const; + i64 GetCurrentPoolSize() const; + void DecrementActiveCounter(); + + void Drain(std::function<bool(std::unique_ptr<TSession::TImpl>&&)> cb, bool close); void SetStatCollector(NSdkStats::TStatCollector::TSessionPoolStatCollector collector); - - static void CreateFakeSession(NThreading::TPromise<TCreateSessionResult>& promise, - std::shared_ptr<TTableClient::TImpl> client); -private: + + static void CreateFakeSession(NThreading::TPromise<TCreateSessionResult>& promise, + std::shared_ptr<TTableClient::TImpl> client); +private: void UpdateStats(); mutable std::mutex Mtx_; - TMultiMap<TInstant, std::unique_ptr<TSession::TImpl>> Sessions_; - bool Closed_; - i64 ActiveSessions_; - const ui32 MaxActiveSessions_; + TMultiMap<TInstant, std::unique_ptr<TSession::TImpl>> Sessions_; + bool Closed_; + i64 ActiveSessions_; + const ui32 MaxActiveSessions_; NSdkStats::TSessionCounter ActiveSessionsCounter_; NSdkStats::TSessionCounter InPoolSessionsCounter_; - NSdkStats::TAtomicCounter<NMonitoring::TRate> FakeSessionsCounter_; -}; - -static TDuration RandomizeThreshold(TDuration duration) { - TDuration::TValue value = duration.GetValue(); - if (KEEP_ALIVE_RANDOM_FRACTION) { - const i64 randomLimit = value / KEEP_ALIVE_RANDOM_FRACTION; - if (randomLimit < 2) - return duration; - value += static_cast<i64>(RandomNumber<ui64>(randomLimit)); - } - return TDuration::FromValue(value); -} - -static TDuration GetMinTimeToTouch(const TSessionPoolSettings& settings) { - return Min(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); -} - -static TDuration GetMaxTimeToTouch(const TSessionPoolSettings& settings) { - return Max(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); -} - -static TStatus GetStatus(const TOperation& operation) { - return operation.Status(); -} - -static TStatus GetStatus(const TStatus& status) { - return status; -} - + NSdkStats::TAtomicCounter<NMonitoring::TRate> FakeSessionsCounter_; +}; + +static TDuration RandomizeThreshold(TDuration duration) { + TDuration::TValue value = duration.GetValue(); + if (KEEP_ALIVE_RANDOM_FRACTION) { + const i64 randomLimit = value / KEEP_ALIVE_RANDOM_FRACTION; + if (randomLimit < 2) + return duration; + value += static_cast<i64>(RandomNumber<ui64>(randomLimit)); + } + return TDuration::FromValue(value); +} + +static TDuration GetMinTimeToTouch(const TSessionPoolSettings& settings) { + return Min(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); +} + +static TDuration GetMaxTimeToTouch(const TSessionPoolSettings& settings) { + return Max(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); +} + +static TStatus GetStatus(const TOperation& operation) { + return operation.Status(); +} + +static TStatus GetStatus(const TStatus& status) { + return status; +} + static bool IsSessionCloseRequested(const TStatus& status) { const auto& meta = status.GetResponseMetadata(); auto hints = meta.equal_range(NYdb::YDB_SERVER_HINTS); @@ -1400,49 +1400,49 @@ static bool IsSessionCloseRequested(const TStatus& status) { template<typename TResponse> NThreading::TFuture<TResponse> InjectSessionStatusInterception( std::shared_ptr<TSession::TImpl>& impl, NThreading::TFuture<TResponse> asyncResponse, - bool updateTimeout, - TDuration timeout, + bool updateTimeout, + TDuration timeout, std::function<void(const TResponse&, TSession::TImpl&)> cb = {}) { auto promise = NThreading::NewPromise<TResponse>(); - asyncResponse.Subscribe([impl, promise, cb, updateTimeout, timeout](NThreading::TFuture<TResponse> future) mutable { + asyncResponse.Subscribe([impl, promise, cb, updateTimeout, timeout](NThreading::TFuture<TResponse> future) mutable { Y_VERIFY(future.HasValue()); - - // TResponse can hold refcounted user provided data (TSession for example) - // and we do not want to have copy of it (for example it can cause delay dtor call) - // so using move semantic here is mandatory. - // Also we must reset captured shared pointer to session impl - TResponse value = std::move(future.ExtractValue()); - - const TStatus& status = GetStatus(value); - // Exclude CLIENT_RESOURCE_EXHAUSTED from transport errors which can cause to session disconnect - // since we have guarantee this request wasn't been started to execute. - - if (status.IsTransportError() && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED) { - // Mark disconnected - the session will be returnet to settler - impl->MarkDisconnected(); - } else if (status.GetStatus() == EStatus::SESSION_BUSY) { - impl->MarkDisconnected(); - } else if (status.GetStatus() == EStatus::BAD_SESSION) { + + // TResponse can hold refcounted user provided data (TSession for example) + // and we do not want to have copy of it (for example it can cause delay dtor call) + // so using move semantic here is mandatory. + // Also we must reset captured shared pointer to session impl + TResponse value = std::move(future.ExtractValue()); + + const TStatus& status = GetStatus(value); + // Exclude CLIENT_RESOURCE_EXHAUSTED from transport errors which can cause to session disconnect + // since we have guarantee this request wasn't been started to execute. + + if (status.IsTransportError() && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED) { + // Mark disconnected - the session will be returnet to settler + impl->MarkDisconnected(); + } else if (status.GetStatus() == EStatus::SESSION_BUSY) { + impl->MarkDisconnected(); + } else if (status.GetStatus() == EStatus::BAD_SESSION) { impl->MarkBroken(); } else if (IsSessionCloseRequested(status)) { impl->MarkAsClosing(); - } else { - // NOTE: About GetState and lock - // Simultanious call multiple requests on the same session make no sence, due to server limitation. - // But user can perform this call, right now we do not protect session from this, it may cause - // raise on session state if respoise is not success. - // It should not be a problem - in case of this race we close session - // or put it in to settler. - if (updateTimeout && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED) { - impl->ScheduleTimeToTouch(RandomizeThreshold(timeout), impl->GetState() == TSession::TImpl::EState::S_ACTIVE); - } + } else { + // NOTE: About GetState and lock + // Simultanious call multiple requests on the same session make no sence, due to server limitation. + // But user can perform this call, right now we do not protect session from this, it may cause + // raise on session state if respoise is not success. + // It should not be a problem - in case of this race we close session + // or put it in to settler. + if (updateTimeout && status.GetStatus() != EStatus::CLIENT_RESOURCE_EXHAUSTED) { + impl->ScheduleTimeToTouch(RandomizeThreshold(timeout), impl->GetState() == TSession::TImpl::EState::S_ACTIVE); + } } if (cb) { cb(value, *impl); } - impl.reset(); - promise.SetValue(std::move(value)); + impl.reset(); + promise.SetValue(std::move(value)); }); return promise.GetFuture(); } @@ -1457,20 +1457,20 @@ static ui32 CalcBackoffTime(const TBackoffSettings& settings, ui32 retryNumber) double durationMs = round(maxDuration.MilliSeconds() * uncertaintyMultiplier); return std::max(std::min(durationMs, (double)MAX_BACKOFF_DURATION_MS), 0.0); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TTableClient::TImpl: public TClientImplCommon<TTableClient::TImpl>, public IMigratorClient { +} + +//////////////////////////////////////////////////////////////////////////////// + +class TTableClient::TImpl: public TClientImplCommon<TTableClient::TImpl>, public IMigratorClient { public: - using TReadTableStreamProcessorPtr = TTablePartIterator::TReaderImpl::TStreamProcessorPtr; + using TReadTableStreamProcessorPtr = TTablePartIterator::TReaderImpl::TStreamProcessorPtr; using TScanQueryProcessorPtr = TScanQueryPartIterator::TReaderImpl::TStreamProcessorPtr; TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) + : TClientImplCommon(std::move(connections), settings) , Settings_(settings) - , SessionPool_(Settings_.SessionPoolSettings_.MaxActiveSessions_) - , SettlerPool_(0) + , SessionPool_(Settings_.SessionPoolSettings_.MaxActiveSessions_) + , SettlerPool_(0) { if (!DbDriverState_->StatCollector.IsCollecting()) { return; @@ -1485,78 +1485,78 @@ public: )); } - ~TImpl() { - RequestMigrator_.Wait(); - - if (Connections_->GetDrainOnDtors()) { - Drain().Wait(); - } - } - - bool LinkObjToEndpoint(const TString& endpoint, TEndpointObj* obj, const void* tag) { - return DbDriverState_->EndpointPool.LinkObjToEndpoint(endpoint, obj, tag); - } - - void InitStopper() { - std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); - auto cb = [weak]() mutable { - auto strong = weak.lock(); - if (!strong) { - auto promise = NThreading::NewPromise<void>(); - promise.SetException("no more client"); - return promise.GetFuture(); - } + ~TImpl() { + RequestMigrator_.Wait(); + + if (Connections_->GetDrainOnDtors()) { + Drain().Wait(); + } + } + + bool LinkObjToEndpoint(const TString& endpoint, TEndpointObj* obj, const void* tag) { + return DbDriverState_->EndpointPool.LinkObjToEndpoint(endpoint, obj, tag); + } + + void InitStopper() { + std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); + auto cb = [weak]() mutable { + auto strong = weak.lock(); + if (!strong) { + auto promise = NThreading::NewPromise<void>(); + promise.SetException("no more client"); + return promise.GetFuture(); + } return strong->Drain(); - }; + }; DbDriverState_->AddCb(std::move(cb), TDbDriverState::ENotifyType::STOP); - } - - NThreading::TFuture<void> Drain() { - TVector<std::unique_ptr<TSession::TImpl>> sessions; - // No realocations under lock - sessions.reserve(Settings_.SessionPoolSettings_.MaxActiveSessions_); - auto drainer = [&sessions](std::unique_ptr<TSession::TImpl>&& impl) mutable { - sessions.push_back(std::move(impl)); - return true; - }; - SessionPool_.Drain(drainer, true); - SettlerPool_.Drain(drainer, true); - TVector<TAsyncStatus> closeResults; - for (auto& s : sessions) { - if (s->GetId()) { - closeResults.push_back(CloseInternal(s.get())); - } - } - sessions.clear(); + } + + NThreading::TFuture<void> Drain() { + TVector<std::unique_ptr<TSession::TImpl>> sessions; + // No realocations under lock + sessions.reserve(Settings_.SessionPoolSettings_.MaxActiveSessions_); + auto drainer = [&sessions](std::unique_ptr<TSession::TImpl>&& impl) mutable { + sessions.push_back(std::move(impl)); + return true; + }; + SessionPool_.Drain(drainer, true); + SettlerPool_.Drain(drainer, true); + TVector<TAsyncStatus> closeResults; + for (auto& s : sessions) { + if (s->GetId()) { + closeResults.push_back(CloseInternal(s.get())); + } + } + sessions.clear(); return NThreading::WaitExceptionOrAll(closeResults); - } - - NThreading::TFuture<void> Stop() { - return Drain(); - } - - void ScheduleTask(const std::function<void()>& fn, TDuration timeout) { - std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); - auto cbGuard = [weak, fn]() { - auto strongClient = weak.lock(); - if (strongClient) { - fn(); - } - }; - Connections_->ScheduleOneTimeTask(std::move(cbGuard), timeout); - } - - void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) { - Connections_->ScheduleOneTimeTask(std::move(fn), timeout); - } - + } + + NThreading::TFuture<void> Stop() { + return Drain(); + } + + void ScheduleTask(const std::function<void()>& fn, TDuration timeout) { + std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); + auto cbGuard = [weak, fn]() { + auto strongClient = weak.lock(); + if (strongClient) { + fn(); + } + }; + Connections_->ScheduleOneTimeTask(std::move(cbGuard), timeout); + } + + void ScheduleTaskUnsafe(std::function<void()>&& fn, TDuration timeout) { + Connections_->ScheduleOneTimeTask(std::move(fn), timeout); + } + void AsyncBackoff(const TBackoffSettings& settings, ui32 retryNumber, const std::function<void()>& fn) { - auto durationMs = CalcBackoffTime(settings, retryNumber); - ScheduleTask(fn, TDuration::MilliSeconds(durationMs)); - } - - void StartPeriodicSessionPoolTask() { + auto durationMs = CalcBackoffTime(settings, retryNumber); + ScheduleTask(fn, TDuration::MilliSeconds(durationMs)); + } + + void StartPeriodicSessionPoolTask() { auto deletePredicate = [](TSession::TImpl* session, TTableClient::TImpl* client, size_t sessionsCount) { @@ -1572,362 +1572,362 @@ public: return false; }; - auto keepAliveCmd = [](TSession session) { - Y_VERIFY(session.GetId()); - - const auto sessionPoolSettings = session.Client_->Settings_.SessionPoolSettings_; - const auto spentTime = session.SessionImpl_->GetTimeToTouchFast() - session.SessionImpl_->GetTimeInPastFast(); - - const auto maxTimeToTouch = GetMaxTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); - const auto minTimeToTouch = GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); - - auto calcTimeToNextTouch = [maxTimeToTouch, minTimeToTouch] (const TDuration spent) { - auto timeToNextTouch = minTimeToTouch; - if (maxTimeToTouch > spent) { - auto t = maxTimeToTouch - spent; - timeToNextTouch = Min(t, minTimeToTouch); - } - return timeToNextTouch; - }; - - if (spentTime >= sessionPoolSettings.KeepAliveIdleThreshold_) { - - // Handle of session status will be done inside InjectSessionStatusInterception routine. - // We just need to reschedule time to next call because InjectSessionStatusInterception doesn't - // update timeInPast for calls from internal keep alive routine - session.KeepAlive(KeepAliveSettings) - .Subscribe([spentTime, session, maxTimeToTouch, calcTimeToNextTouch](TAsyncKeepAliveResult asyncResult) { - if (!asyncResult.GetValue().IsSuccess()) - return; - - if (spentTime >= maxTimeToTouch) { - auto timeToNextTouch = calcTimeToNextTouch(spentTime); - session.SessionImpl_->ScheduleTimeToTouchFast(timeToNextTouch, true); - } - }); - return; - } - - auto timeToNextTouch = calcTimeToNextTouch(spentTime); - session.SessionImpl_->ScheduleTimeToTouchFast( - RandomizeThreshold(timeToNextTouch), - spentTime >= maxTimeToTouch - ); - }; - - std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); - Connections_->AddPeriodicTask( - SessionPool_.CreatePeriodicTask( - weak, + auto keepAliveCmd = [](TSession session) { + Y_VERIFY(session.GetId()); + + const auto sessionPoolSettings = session.Client_->Settings_.SessionPoolSettings_; + const auto spentTime = session.SessionImpl_->GetTimeToTouchFast() - session.SessionImpl_->GetTimeInPastFast(); + + const auto maxTimeToTouch = GetMaxTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); + const auto minTimeToTouch = GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_); + + auto calcTimeToNextTouch = [maxTimeToTouch, minTimeToTouch] (const TDuration spent) { + auto timeToNextTouch = minTimeToTouch; + if (maxTimeToTouch > spent) { + auto t = maxTimeToTouch - spent; + timeToNextTouch = Min(t, minTimeToTouch); + } + return timeToNextTouch; + }; + + if (spentTime >= sessionPoolSettings.KeepAliveIdleThreshold_) { + + // Handle of session status will be done inside InjectSessionStatusInterception routine. + // We just need to reschedule time to next call because InjectSessionStatusInterception doesn't + // update timeInPast for calls from internal keep alive routine + session.KeepAlive(KeepAliveSettings) + .Subscribe([spentTime, session, maxTimeToTouch, calcTimeToNextTouch](TAsyncKeepAliveResult asyncResult) { + if (!asyncResult.GetValue().IsSuccess()) + return; + + if (spentTime >= maxTimeToTouch) { + auto timeToNextTouch = calcTimeToNextTouch(spentTime); + session.SessionImpl_->ScheduleTimeToTouchFast(timeToNextTouch, true); + } + }); + return; + } + + auto timeToNextTouch = calcTimeToNextTouch(spentTime); + session.SessionImpl_->ScheduleTimeToTouchFast( + RandomizeThreshold(timeToNextTouch), + spentTime >= maxTimeToTouch + ); + }; + + std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); + Connections_->AddPeriodicTask( + SessionPool_.CreatePeriodicTask( + weak, std::move(keepAliveCmd), std::move(deletePredicate) - ), PERIODIC_ACTION_INTERVAL); - } - - static void DoSettlerKeepAlive(TSession&& session) { - auto curThreshold = session.SessionImpl_->GetTimeInterval(); - if (!curThreshold) { - session.SessionImpl_->SetTimeInterval(TDuration::Seconds(1)); - } else { - session.SessionImpl_->SetTimeInterval(TDuration::FromValue(curThreshold.GetValue() * 2)); - } - - // Bypass InjectSessionStatusInterception to simplify logic - std::shared_ptr<TTableClient::TImpl> clientImpl = session.Client_; - clientImpl->KeepAlive(session.SessionImpl_.get(), KeepAliveSettings) - .Subscribe([session = std::move(session)](TAsyncKeepAliveResult asyncResult) { - - auto result = asyncResult.ExtractValue(); - Y_VERIFY(session.SessionImpl_->GetState() == TSession::TImpl::EState::S_DISCONNECTED); - - if (!result.IsTransportError()) { - if (result.IsSuccess()) { - switch (result.GetSessionStatus()) { - case ESessionStatus::Ready: - // Ready state, mark Idle to return to session pool after dtor call - session.SessionImpl_->MarkIdle(); - session.SessionImpl_->SetTimeInterval(TDuration::Zero()); - session.SessionImpl_->ScheduleTimeToTouchFast( - RandomizeThreshold( - GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_) - ), - true - ); - break; - case ESessionStatus::Unspecified: - // Old server, we have no information about server status - close it - session.SessionImpl_->MarkBroken(); - break; - case ESessionStatus::Busy: - // Session should still be in S_DISCONNECTED state - break; - default: - Y_VERIFY(false, "unimplemented"); - } - } else { - session.SessionImpl_->MarkBroken(); - } - } - if (session.SessionImpl_->GetState() == TSession::TImpl::EState::S_DISCONNECTED) { - session.SessionImpl_->ScheduleTimeToTouchFast( - RandomizeThreshold(session.SessionImpl_->GetTimeInterval()), - false - ); - } - }); - } - - void StartPeriodicSettlerTask() { - + ), PERIODIC_ACTION_INTERVAL); + } + + static void DoSettlerKeepAlive(TSession&& session) { + auto curThreshold = session.SessionImpl_->GetTimeInterval(); + if (!curThreshold) { + session.SessionImpl_->SetTimeInterval(TDuration::Seconds(1)); + } else { + session.SessionImpl_->SetTimeInterval(TDuration::FromValue(curThreshold.GetValue() * 2)); + } + + // Bypass InjectSessionStatusInterception to simplify logic + std::shared_ptr<TTableClient::TImpl> clientImpl = session.Client_; + clientImpl->KeepAlive(session.SessionImpl_.get(), KeepAliveSettings) + .Subscribe([session = std::move(session)](TAsyncKeepAliveResult asyncResult) { + + auto result = asyncResult.ExtractValue(); + Y_VERIFY(session.SessionImpl_->GetState() == TSession::TImpl::EState::S_DISCONNECTED); + + if (!result.IsTransportError()) { + if (result.IsSuccess()) { + switch (result.GetSessionStatus()) { + case ESessionStatus::Ready: + // Ready state, mark Idle to return to session pool after dtor call + session.SessionImpl_->MarkIdle(); + session.SessionImpl_->SetTimeInterval(TDuration::Zero()); + session.SessionImpl_->ScheduleTimeToTouchFast( + RandomizeThreshold( + GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_) + ), + true + ); + break; + case ESessionStatus::Unspecified: + // Old server, we have no information about server status - close it + session.SessionImpl_->MarkBroken(); + break; + case ESessionStatus::Busy: + // Session should still be in S_DISCONNECTED state + break; + default: + Y_VERIFY(false, "unimplemented"); + } + } else { + session.SessionImpl_->MarkBroken(); + } + } + if (session.SessionImpl_->GetState() == TSession::TImpl::EState::S_DISCONNECTED) { + session.SessionImpl_->ScheduleTimeToTouchFast( + RandomizeThreshold(session.SessionImpl_->GetTimeInterval()), + false + ); + } + }); + } + + void StartPeriodicSettlerTask() { + auto deletePredicate = [](TSession::TImpl* , TTableClient::TImpl* , size_t) { return false; }; - auto ttl = Settings_.SettlerSessionPoolTTL_; - auto keepAliveCmd = [ttl](TSession session) { - Y_VERIFY(session.GetId()); - Y_VERIFY(!session.SessionImpl_->NeedUpdateActiveCounter()); - // Here we can change session associated variables without additional locking - - if ((session.SessionImpl_->GetTimeInterval().Seconds() * 2 - 1) < ttl) { - DoSettlerKeepAlive(std::move(session)); - } else { - session.SessionImpl_->MarkBroken(); - } - }; - std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); - Connections_->AddPeriodicTask( - SettlerPool_.CreatePeriodicTask( - weak, + auto ttl = Settings_.SettlerSessionPoolTTL_; + auto keepAliveCmd = [ttl](TSession session) { + Y_VERIFY(session.GetId()); + Y_VERIFY(!session.SessionImpl_->NeedUpdateActiveCounter()); + // Here we can change session associated variables without additional locking + + if ((session.SessionImpl_->GetTimeInterval().Seconds() * 2 - 1) < ttl) { + DoSettlerKeepAlive(std::move(session)); + } else { + session.SessionImpl_->MarkBroken(); + } + }; + std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); + Connections_->AddPeriodicTask( + SettlerPool_.CreatePeriodicTask( + weak, std::move(keepAliveCmd), std::move(deletePredicate) - ), SETTLER_PERIODIC_ACTION_INTERVAL); - } - - static TString ScanForeignLocations(std::shared_ptr<TTableClient::TImpl> client) { - size_t max = 0; - TString result; - - auto cb = [&result, &max](const TString& host, const IObjRegistryHandle& handle) { - const auto sz = handle.Size(); - if (sz > max) { - result = host; - max = sz; - } - }; - - client->DbDriverState_->ForEachForeignEndpoint(cb, client.get()); - - return result; - } - - static std::pair<TString, size_t> ScanLocation(std::shared_ptr<TTableClient::TImpl> client, - std::unordered_map<TString, size_t>& sessions, bool allNodes) - { - std::pair<TString, size_t> result = {TString(), 0}; - - auto cb = [&result, &sessions](const TString& host, const IObjRegistryHandle& handle) { - const auto sz = handle.Size(); - sessions.insert({host, sz}); - if (sz > result.second) { - result.first = host; - result.second = sz; - } - }; - - if (allNodes) { - client->DbDriverState_->ForEachEndpoint(cb, client.get()); - } else { - client->DbDriverState_->ForEachLocalEndpoint(cb, client.get()); - } - - return result; - } - - static NThreading::TFuture<ui64> StartRequestMigration(std::shared_ptr<TTableClient::TImpl> client, - const TString& oldEndpoint, const size_t maxSessions, const std::unordered_map<TString, size_t>& hostMap) - { - const auto settings = TCreateSessionSettings() - .ClientTimeout(MIGRATION_GET_SESSION_CLIENT_TIMEOUT); - - auto newEndpoint = client->DbDriverState_->GetEndpoint(); - - // New host is same - if (newEndpoint == oldEndpoint) - return NThreading::MakeFuture<ui64>(0); - - // New host unknown - auto it = hostMap.find(newEndpoint); - if (it == hostMap.end()) - return NThreading::MakeFuture<ui64>(0); - - // Number of sessions on proposed host will bw grater after creation new session - // skip such host - if (it->second >= maxSessions) - return NThreading::MakeFuture<ui64>(0); - - return client->CreateSession(settings, false, newEndpoint) - .Apply([oldEndpoint, client, maxSessions](TAsyncCreateSessionResult future) { - auto sessionResult = future.ExtractValue(); - - if (!sessionResult.IsSuccess()) - return NThreading::MakeFuture<ui64>(0); - - auto session = sessionResult.GetSession(); - - // Mark idle to prevent ActiveSessionCounter change - session.SessionImpl_->MarkIdle(); - - // Got session, but the host just has been removed, we can`t trust this session - // TODO: probably change CreateSession to return error in this case - if (!session.SessionImpl_->ObjectRegistred()) { - session.SessionImpl_->MarkBroken(); - return NThreading::MakeFuture<ui64>(0); - } - - auto sessionsOnProposedHost = session.SessionImpl_->ObjectCount(); - - if (sessionResult.GetEndpoint() != oldEndpoint && sessionsOnProposedHost <= maxSessions) { - return client->RequestMigrator_.SetHost(oldEndpoint, session); - } else { - session.SessionImpl_->MarkBroken(); - return NThreading::MakeFuture<ui64>(0); - } - }); - } - - static NMath::TStats CalcCV(const std::unordered_map<TString, size_t>& in) { - TVector<size_t> t; - t.reserve(in.size()); - std::transform(in.begin(), in.end(), std::back_inserter(t), [](const std::pair<TString, size_t>& pair) { - return pair.second; - }); - return NMath::CalcCV(t); - } - - void StartPeriodicHostScanTask() { - std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); - - // The future in completed when we have finished current migrate task - // and ready to accept new one - TFuture<ui64> readyToMigrate; - std::pair<TString, size_t> winner = {TString(), 0}; - - auto periodicCb = [weak, readyToMigrate, winner](NYql::TIssues&&, EStatus status) mutable -> bool { - - if (status != EStatus::SUCCESS) { - return false; - } - - auto strongClient = weak.lock(); - if (!strongClient) { - return false; - } else { - TRequestMigrator& migrator = strongClient->RequestMigrator_; - // Another migration has started but not finished yet - if (readyToMigrate.Initialized() && !readyToMigrate.HasValue()) { - // There is a host we want to find session on but session was not returned from client - if (winner.first) { - // Try to find sutiable session in session pool - if (!strongClient->SessionPool_.DropSessionOnEndpoint(strongClient, winner.first)) { - // No session in session pool, reset migrator and start from scratch - migrator.Reset(); - } - } - return true; - } - - const auto balancingPolicy = strongClient->DbDriverState_->GetBalancingPolicy(); - - // Try to find any host at foreign locations if prefer local dc - const TString foreignHost = (balancingPolicy == EBalancingPolicy::UsePreferableLocation) ? - ScanForeignLocations(strongClient) : TString(); - - std::unordered_map<TString, size_t> hostMap; - - winner = ScanLocation(strongClient, hostMap, - balancingPolicy == EBalancingPolicy::UseAllNodes); - - bool forceMigrate = false; - - // There is host in foreign locations - if (foreignHost) { - // But no hosts at local - if (hostMap.empty()) { - Y_VERIFY(!winner.second); - // Scan whole cluster - we have no local dc - winner = ScanLocation(strongClient, hostMap, true); - } else { - // We have local and foreign hosts, so force migration to local one - forceMigrate = true; - // Just replace source - winner.first = foreignHost; - winner.second++; - } - } - - const auto minCv = strongClient->Settings_.MinSessionCV_; - - const auto stats = CalcCV(hostMap); - - strongClient->DbDriverState_->StatCollector.SetSessionCV(stats.Cv); - - // Just scan to update monitoring counter ^^ - // Balancing feature is disabled. - if (!minCv) - return true; - - if (hostMap.size() < 2) - return true; - - - // Migrate last session only if move from foreign to local - if (!forceMigrate && winner.second < 2) - return true; - - // Add counter to update monitoring - if (readyToMigrate.Initialized()) { - strongClient->RequestMigrated.Add(readyToMigrate.GetValue()); - } - - if (stats.Cv > minCv || forceMigrate) { - if (!strongClient->Settings_.AllowRequestMigration_) { - readyToMigrate = migrator.SetHost(winner.first); - } else { - // Max number of session on target host to allow migrate requests to. - // We should not try to migrate if number of sessions on target host equal - // or less than number of session on source host by one. - // And apply some heuristic: do not try to migrate on host where number of sessions - // grater than mean. - // - // If force migration flag is set we should increase the limit - // to allow migration if no sessions on local cluster - size_t maxSessions = 0; - if (forceMigrate) { - maxSessions = (size_t)ceil(stats.Mean) + 1; - } else { - maxSessions = std::min((size_t)ceil(stats.Mean), winner.second - 1); - } - readyToMigrate = StartRequestMigration(strongClient, winner.first, maxSessions, hostMap); - } - } else { - readyToMigrate = migrator.SetHost(TString()); - } - return true; - } - }; - - Connections_->AddPeriodicTask(std::move(periodicCb), HOSTSCAN_PERIODIC_ACTION_INTERVAL); - } - - TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings) { - return SessionPool_.GetSession(shared_from_this(), settings, &SettlerAwareSessonProvider); - } - - i64 GetActiveSessionCount() const { - return SessionPool_.GetActiveSessions(); - } - + ), SETTLER_PERIODIC_ACTION_INTERVAL); + } + + static TString ScanForeignLocations(std::shared_ptr<TTableClient::TImpl> client) { + size_t max = 0; + TString result; + + auto cb = [&result, &max](const TString& host, const IObjRegistryHandle& handle) { + const auto sz = handle.Size(); + if (sz > max) { + result = host; + max = sz; + } + }; + + client->DbDriverState_->ForEachForeignEndpoint(cb, client.get()); + + return result; + } + + static std::pair<TString, size_t> ScanLocation(std::shared_ptr<TTableClient::TImpl> client, + std::unordered_map<TString, size_t>& sessions, bool allNodes) + { + std::pair<TString, size_t> result = {TString(), 0}; + + auto cb = [&result, &sessions](const TString& host, const IObjRegistryHandle& handle) { + const auto sz = handle.Size(); + sessions.insert({host, sz}); + if (sz > result.second) { + result.first = host; + result.second = sz; + } + }; + + if (allNodes) { + client->DbDriverState_->ForEachEndpoint(cb, client.get()); + } else { + client->DbDriverState_->ForEachLocalEndpoint(cb, client.get()); + } + + return result; + } + + static NThreading::TFuture<ui64> StartRequestMigration(std::shared_ptr<TTableClient::TImpl> client, + const TString& oldEndpoint, const size_t maxSessions, const std::unordered_map<TString, size_t>& hostMap) + { + const auto settings = TCreateSessionSettings() + .ClientTimeout(MIGRATION_GET_SESSION_CLIENT_TIMEOUT); + + auto newEndpoint = client->DbDriverState_->GetEndpoint(); + + // New host is same + if (newEndpoint == oldEndpoint) + return NThreading::MakeFuture<ui64>(0); + + // New host unknown + auto it = hostMap.find(newEndpoint); + if (it == hostMap.end()) + return NThreading::MakeFuture<ui64>(0); + + // Number of sessions on proposed host will bw grater after creation new session + // skip such host + if (it->second >= maxSessions) + return NThreading::MakeFuture<ui64>(0); + + return client->CreateSession(settings, false, newEndpoint) + .Apply([oldEndpoint, client, maxSessions](TAsyncCreateSessionResult future) { + auto sessionResult = future.ExtractValue(); + + if (!sessionResult.IsSuccess()) + return NThreading::MakeFuture<ui64>(0); + + auto session = sessionResult.GetSession(); + + // Mark idle to prevent ActiveSessionCounter change + session.SessionImpl_->MarkIdle(); + + // Got session, but the host just has been removed, we can`t trust this session + // TODO: probably change CreateSession to return error in this case + if (!session.SessionImpl_->ObjectRegistred()) { + session.SessionImpl_->MarkBroken(); + return NThreading::MakeFuture<ui64>(0); + } + + auto sessionsOnProposedHost = session.SessionImpl_->ObjectCount(); + + if (sessionResult.GetEndpoint() != oldEndpoint && sessionsOnProposedHost <= maxSessions) { + return client->RequestMigrator_.SetHost(oldEndpoint, session); + } else { + session.SessionImpl_->MarkBroken(); + return NThreading::MakeFuture<ui64>(0); + } + }); + } + + static NMath::TStats CalcCV(const std::unordered_map<TString, size_t>& in) { + TVector<size_t> t; + t.reserve(in.size()); + std::transform(in.begin(), in.end(), std::back_inserter(t), [](const std::pair<TString, size_t>& pair) { + return pair.second; + }); + return NMath::CalcCV(t); + } + + void StartPeriodicHostScanTask() { + std::weak_ptr<TTableClient::TImpl> weak = shared_from_this(); + + // The future in completed when we have finished current migrate task + // and ready to accept new one + TFuture<ui64> readyToMigrate; + std::pair<TString, size_t> winner = {TString(), 0}; + + auto periodicCb = [weak, readyToMigrate, winner](NYql::TIssues&&, EStatus status) mutable -> bool { + + if (status != EStatus::SUCCESS) { + return false; + } + + auto strongClient = weak.lock(); + if (!strongClient) { + return false; + } else { + TRequestMigrator& migrator = strongClient->RequestMigrator_; + // Another migration has started but not finished yet + if (readyToMigrate.Initialized() && !readyToMigrate.HasValue()) { + // There is a host we want to find session on but session was not returned from client + if (winner.first) { + // Try to find sutiable session in session pool + if (!strongClient->SessionPool_.DropSessionOnEndpoint(strongClient, winner.first)) { + // No session in session pool, reset migrator and start from scratch + migrator.Reset(); + } + } + return true; + } + + const auto balancingPolicy = strongClient->DbDriverState_->GetBalancingPolicy(); + + // Try to find any host at foreign locations if prefer local dc + const TString foreignHost = (balancingPolicy == EBalancingPolicy::UsePreferableLocation) ? + ScanForeignLocations(strongClient) : TString(); + + std::unordered_map<TString, size_t> hostMap; + + winner = ScanLocation(strongClient, hostMap, + balancingPolicy == EBalancingPolicy::UseAllNodes); + + bool forceMigrate = false; + + // There is host in foreign locations + if (foreignHost) { + // But no hosts at local + if (hostMap.empty()) { + Y_VERIFY(!winner.second); + // Scan whole cluster - we have no local dc + winner = ScanLocation(strongClient, hostMap, true); + } else { + // We have local and foreign hosts, so force migration to local one + forceMigrate = true; + // Just replace source + winner.first = foreignHost; + winner.second++; + } + } + + const auto minCv = strongClient->Settings_.MinSessionCV_; + + const auto stats = CalcCV(hostMap); + + strongClient->DbDriverState_->StatCollector.SetSessionCV(stats.Cv); + + // Just scan to update monitoring counter ^^ + // Balancing feature is disabled. + if (!minCv) + return true; + + if (hostMap.size() < 2) + return true; + + + // Migrate last session only if move from foreign to local + if (!forceMigrate && winner.second < 2) + return true; + + // Add counter to update monitoring + if (readyToMigrate.Initialized()) { + strongClient->RequestMigrated.Add(readyToMigrate.GetValue()); + } + + if (stats.Cv > minCv || forceMigrate) { + if (!strongClient->Settings_.AllowRequestMigration_) { + readyToMigrate = migrator.SetHost(winner.first); + } else { + // Max number of session on target host to allow migrate requests to. + // We should not try to migrate if number of sessions on target host equal + // or less than number of session on source host by one. + // And apply some heuristic: do not try to migrate on host where number of sessions + // grater than mean. + // + // If force migration flag is set we should increase the limit + // to allow migration if no sessions on local cluster + size_t maxSessions = 0; + if (forceMigrate) { + maxSessions = (size_t)ceil(stats.Mean) + 1; + } else { + maxSessions = std::min((size_t)ceil(stats.Mean), winner.second - 1); + } + readyToMigrate = StartRequestMigration(strongClient, winner.first, maxSessions, hostMap); + } + } else { + readyToMigrate = migrator.SetHost(TString()); + } + return true; + } + }; + + Connections_->AddPeriodicTask(std::move(periodicCb), HOSTSCAN_PERIODIC_ACTION_INTERVAL); + } + + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings) { + return SessionPool_.GetSession(shared_from_this(), settings, &SettlerAwareSessonProvider); + } + + i64 GetActiveSessionCount() const { + return SessionPool_.GetActiveSessions(); + } + i64 GetActiveSessionsLimit() const { return SessionPool_.GetActiveSessionsLimit(); } @@ -1936,93 +1936,93 @@ public: return SessionPool_.GetCurrentPoolSize(); } - TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings, bool standalone, - TString preferedLocation = TString()) - { + TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings, bool standalone, + TString preferedLocation = TString()) + { auto request = MakeOperationRequest<Ydb::Table::CreateSessionRequest>(settings); auto createSessionPromise = NewPromise<TCreateSessionResult>(); - auto self = shared_from_this(); + auto self = shared_from_this(); - auto createSessionExtractor = [createSessionPromise, self, standalone] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Table::CreateSessionResult result; + auto createSessionExtractor = [createSessionPromise, self, standalone] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::CreateSessionResult result; if (any) { any->UnpackTo(&result); } - auto session = TSession(self, result.session_id(), status.Endpoint); - if (status.Ok()) { - if (standalone) { - session.SessionImpl_->MarkStandalone(); - } else { - session.SessionImpl_->MarkActive(); - } - self->DbDriverState_->StatCollector.IncSessionsOnHost(status.Endpoint); - } else { - // We do not use SessionStatusInterception for CreateSession request - session.SessionImpl_->MarkBroken(); - } - TCreateSessionResult val(TStatus(std::move(status)), std::move(session)); + auto session = TSession(self, result.session_id(), status.Endpoint); + if (status.Ok()) { + if (standalone) { + session.SessionImpl_->MarkStandalone(); + } else { + session.SessionImpl_->MarkActive(); + } + self->DbDriverState_->StatCollector.IncSessionsOnHost(status.Endpoint); + } else { + // We do not use SessionStatusInterception for CreateSession request + session.SessionImpl_->MarkBroken(); + } + TCreateSessionResult val(TStatus(std::move(status)), std::move(session)); createSessionPromise.SetValue(std::move(val)); }; - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::CreateSessionRequest, Ydb::Table::CreateSessionResponse>( + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::CreateSessionRequest, Ydb::Table::CreateSessionResponse>( std::move(request), createSessionExtractor, - &Ydb::Table::V1::TableService::Stub::AsyncCreateSession, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Table::V1::TableService::Stub::AsyncCreateSession, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - preferedLocation); + settings.ClientTimeout_, + preferedLocation); std::weak_ptr<TDbDriverState> state = DbDriverState_; - return createSessionPromise.GetFuture(); + return createSessionPromise.GetFuture(); } - TAsyncKeepAliveResult KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings) { + TAsyncKeepAliveResult KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::KeepAliveRequest>(settings); - request.set_session_id(session->GetId()); - - auto keepAliveResultPromise = NewPromise<TKeepAliveResult>(); - auto self = shared_from_this(); - - auto keepAliveExtractor = [keepAliveResultPromise, self] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Table::KeepAliveResult result; - ESessionStatus sessionStatus = ESessionStatus::Unspecified; - if (any) { - any->UnpackTo(&result); - - switch (result.session_status()) { - case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_READY: - sessionStatus = ESessionStatus::Ready; - break; - case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_BUSY: - sessionStatus = ESessionStatus::Busy; - break; - default: - sessionStatus = ESessionStatus::Unspecified; - } - } - TKeepAliveResult val(TStatus(std::move(status)), sessionStatus); - keepAliveResultPromise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::KeepAliveRequest, Ydb::Table::KeepAliveResponse>( + request.set_session_id(session->GetId()); + + auto keepAliveResultPromise = NewPromise<TKeepAliveResult>(); + auto self = shared_from_this(); + + auto keepAliveExtractor = [keepAliveResultPromise, self] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::KeepAliveResult result; + ESessionStatus sessionStatus = ESessionStatus::Unspecified; + if (any) { + any->UnpackTo(&result); + + switch (result.session_status()) { + case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_READY: + sessionStatus = ESessionStatus::Ready; + break; + case Ydb::Table::KeepAliveResult_SessionStatus_SESSION_STATUS_BUSY: + sessionStatus = ESessionStatus::Busy; + break; + default: + sessionStatus = ESessionStatus::Unspecified; + } + } + TKeepAliveResult val(TStatus(std::move(status)), sessionStatus); + keepAliveResultPromise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::KeepAliveRequest, Ydb::Table::KeepAliveResponse>( std::move(request), - keepAliveExtractor, + keepAliveExtractor, &Ydb::Table::V1::TableService::Stub::AsyncKeepAlive, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session->GetEndpoint()); - - return keepAliveResultPromise.GetFuture(); - } - + settings.ClientTimeout_, + session->GetEndpoint()); + + return keepAliveResultPromise.GetFuture(); + } + TFuture<TStatus> CreateTable(Ydb::Table::CreateTableRequest&& request, const TCreateTableSettings& settings) { return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::CreateTableRequest,Ydb::Table::CreateTableResponse>( @@ -2041,18 +2041,18 @@ public: settings.ClientTimeout_); } - TAsyncOperation AlterTableLong(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings) - { - using Ydb::Table::V1::TableService; - using Ydb::Table::AlterTableRequest; - using Ydb::Table::AlterTableResponse; - return RunOperation<TableService, AlterTableRequest, AlterTableResponse, TOperation>( - std::move(request), - &Ydb::Table::V1::TableService::Stub::AsyncAlterTable, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - } - + TAsyncOperation AlterTableLong(Ydb::Table::AlterTableRequest&& request, const TAlterTableSettings& settings) + { + using Ydb::Table::V1::TableService; + using Ydb::Table::AlterTableRequest; + using Ydb::Table::AlterTableResponse; + return RunOperation<TableService, AlterTableRequest, AlterTableResponse, TOperation>( + std::move(request), + &Ydb::Table::V1::TableService::Stub::AsyncAlterTable, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + } + TFuture<TStatus> CopyTable(const TString& sessionId, const TString& src, const TString& dst, const TCopyTableSettings& settings) { @@ -2098,52 +2098,52 @@ public: settings.ClientTimeout_); } - TAsyncDescribeTableResult DescribeTable(const TString& sessionId, const TString& path, const TDescribeTableSettings& settings) { + TAsyncDescribeTableResult DescribeTable(const TString& sessionId, const TString& path, const TDescribeTableSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::DescribeTableRequest>(settings); request.set_session_id(sessionId); request.set_path(path); - if (settings.WithKeyShardBoundary_) { - request.set_include_shard_key_bounds(true); - } - - if (settings.WithTableStatistics_) { - request.set_include_table_stats(true); - } - - if (settings.WithPartitionStatistics_) { - request.set_include_partition_stats(true); - } - + if (settings.WithKeyShardBoundary_) { + request.set_include_shard_key_bounds(true); + } + + if (settings.WithTableStatistics_) { + request.set_include_table_stats(true); + } + + if (settings.WithPartitionStatistics_) { + request.set_include_partition_stats(true); + } + auto promise = NewPromise<TDescribeTableResult>(); - auto extractor = [promise, settings] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Table::DescribeTableResult result; + auto extractor = [promise, settings] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Table::DescribeTableResult result; if (any) { any->UnpackTo(&result); } - TDescribeTableResult describeTableResult(TStatus(std::move(status)), + TDescribeTableResult describeTableResult(TStatus(std::move(status)), std::move(result), settings); promise.SetValue(std::move(describeTableResult)); }; - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::DescribeTableRequest, Ydb::Table::DescribeTableResponse>( + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::DescribeTableRequest, Ydb::Table::DescribeTableResponse>( std::move(request), extractor, - &Ydb::Table::V1::TableService::Stub::AsyncDescribeTable, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Table::V1::TableService::Stub::AsyncDescribeTable, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); return promise.GetFuture(); } - template<typename TParamsType> + template<typename TParamsType> TAsyncDataQueryResult ExecuteDataQuery(TSession& session, const TString& query, const TTxControl& txControl, - TParamsType params, const TExecDataQuerySettings& settings) + TParamsType params, const TExecDataQuerySettings& settings) { - auto maybeQuery = session.SessionImpl_->GetQueryFromCache(query, Settings_.AllowRequestMigration_); + auto maybeQuery = session.SessionImpl_->GetQueryFromCache(query, Settings_.AllowRequestMigration_); if (maybeQuery) { TDataQuery dataQuery(session, query, maybeQuery->QueryId, maybeQuery->ParameterTypes); return ExecuteDataQuery(session, dataQuery, txControl, params, settings, true); @@ -2152,27 +2152,27 @@ public: CacheMissCounter.Inc(); return InjectSessionStatusInterception(session.SessionImpl_, - ExecuteDataQueryInternal(session, query, txControl, params, settings, false), - true, GetMinTimeToTouch(Settings_.SessionPoolSettings_)); + ExecuteDataQueryInternal(session, query, txControl, params, settings, false), + true, GetMinTimeToTouch(Settings_.SessionPoolSettings_)); } - template<typename TParamsType> + template<typename TParamsType> TAsyncDataQueryResult ExecuteDataQuery(TSession& session, const TDataQuery& dataQuery, const TTxControl& txControl, - TParamsType params, const TExecDataQuerySettings& settings, + TParamsType params, const TExecDataQuerySettings& settings, bool fromCache) { - TString queryKey = dataQuery.Impl_->GetTextHash(); - auto cb = [queryKey](const TDataQueryResult& result, TSession::TImpl& session) { + TString queryKey = dataQuery.Impl_->GetTextHash(); + auto cb = [queryKey](const TDataQueryResult& result, TSession::TImpl& session) { if (result.GetStatus() == EStatus::NOT_FOUND) { - session.InvalidateQueryInCache(queryKey); + session.InvalidateQueryInCache(queryKey); } }; - return InjectSessionStatusInterception<TDataQueryResult>( + return InjectSessionStatusInterception<TDataQueryResult>( session.SessionImpl_, session.Client_->ExecuteDataQueryInternal(session, dataQuery, txControl, params, settings, fromCache), - true, - GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_), + true, + GetMinTimeToTouch(session.Client_->Settings_.SessionPoolSettings_), cb); } @@ -2180,28 +2180,28 @@ public: const TPrepareDataQuerySettings& settings) { auto request = MakeOperationRequest<Ydb::Table::PrepareDataQueryRequest>(settings); - request.set_session_id(session.GetId()); + request.set_session_id(session.GetId()); request.set_yql_text(query); auto promise = NewPromise<TPrepareQueryResult>(); - //See ExecuteDataQueryInternal for explanation - auto sessionPtr = new TSession(session); - auto extractor = [promise, sessionPtr, query] - (google::protobuf::Any* any, TPlainStatus status) mutable { + //See ExecuteDataQueryInternal for explanation + auto sessionPtr = new TSession(session); + auto extractor = [promise, sessionPtr, query] + (google::protobuf::Any* any, TPlainStatus status) mutable { TDataQuery dataQuery(*sessionPtr, query, ""); if (any) { - Ydb::Table::PrepareQueryResult result; + Ydb::Table::PrepareQueryResult result; any->UnpackTo(&result); - if (status.Ok()) { + if (status.Ok()) { dataQuery = TDataQuery(*sessionPtr, query, result.query_id(), result.parameters_types()); - sessionPtr->SessionImpl_->AddQueryToCache(dataQuery); + sessionPtr->SessionImpl_->AddQueryToCache(dataQuery); } } - TPrepareQueryResult prepareQueryResult(TStatus(std::move(status)), + TPrepareQueryResult prepareQueryResult(TStatus(std::move(status)), dataQuery, false); delete sessionPtr; promise.SetValue(std::move(prepareQueryResult)); @@ -2209,15 +2209,15 @@ public: CollectQuerySize(query, QuerySizeHistogram); - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::PrepareDataQueryRequest, Ydb::Table::PrepareDataQueryResponse>( + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::PrepareDataQueryRequest, Ydb::Table::PrepareDataQueryResponse>( std::move(request), extractor, - &Ydb::Table::V1::TableService::Stub::AsyncPrepareDataQuery, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Table::V1::TableService::Stub::AsyncPrepareDataQuery, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->Endpoint_); + settings.ClientTimeout_, + session.SessionImpl_->Endpoint_); return promise.GetFuture(); } @@ -2229,7 +2229,7 @@ public: request.set_session_id(sessionId); request.set_yql_text(query); - return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::ExecuteSchemeQueryRequest, Ydb::Table::ExecuteSchemeQueryResponse>( + return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::ExecuteSchemeQueryRequest, Ydb::Table::ExecuteSchemeQueryResponse>( std::move(request), &Ydb::Table::V1::TableService::Stub::AsyncExecuteSchemeQuery, TRpcRequestSettings::Make(settings), @@ -2240,49 +2240,49 @@ public: const TBeginTxSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::BeginTransactionRequest>(settings); - request.set_session_id(session.GetId()); - SetTxSettings(txSettings, request.mutable_tx_settings()); + request.set_session_id(session.GetId()); + SetTxSettings(txSettings, request.mutable_tx_settings()); auto promise = NewPromise<TBeginTransactionResult>(); - auto extractor = [promise, session] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise, session] + (google::protobuf::Any* any, TPlainStatus status) mutable { TString txId; if (any) { - Ydb::Table::BeginTransactionResult result; + Ydb::Table::BeginTransactionResult result; any->UnpackTo(&result); txId = result.tx_meta().id(); } - TBeginTransactionResult beginTxResult(TStatus(std::move(status)), + TBeginTransactionResult beginTxResult(TStatus(std::move(status)), TTransaction(session, txId)); promise.SetValue(std::move(beginTxResult)); }; - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::BeginTransactionRequest, Ydb::Table::BeginTransactionResponse>( + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::BeginTransactionRequest, Ydb::Table::BeginTransactionResponse>( std::move(request), extractor, - &Ydb::Table::V1::TableService::Stub::AsyncBeginTransaction, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Table::V1::TableService::Stub::AsyncBeginTransaction, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->GetEndpoint()); + settings.ClientTimeout_, + session.SessionImpl_->GetEndpoint()); return promise.GetFuture(); } - TAsyncCommitTransactionResult CommitTransaction(const TSession& session, const TTransaction& tx, + TAsyncCommitTransactionResult CommitTransaction(const TSession& session, const TTransaction& tx, const TCommitTxSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::CommitTransactionRequest>(settings); - request.set_session_id(session.GetId()); + request.set_session_id(session.GetId()); request.set_tx_id(tx.GetId()); request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); auto promise = NewPromise<TCommitTransactionResult>(); - auto extractor = [promise] + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TMaybe<TQueryStats> queryStats; if (any) { @@ -2294,7 +2294,7 @@ public: } } - TCommitTransactionResult commitTxResult(TStatus(std::move(status)), queryStats); + TCommitTransactionResult commitTxResult(TStatus(std::move(status)), queryStats); promise.SetValue(std::move(commitTxResult)); }; @@ -2305,60 +2305,60 @@ public: DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->GetEndpoint()); + settings.ClientTimeout_, + session.SessionImpl_->GetEndpoint()); return promise.GetFuture(); } - TAsyncStatus RollbackTransaction(const TSession& session, const TTransaction& tx, + TAsyncStatus RollbackTransaction(const TSession& session, const TTransaction& tx, const TRollbackTxSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::RollbackTransactionRequest>(settings); - request.set_session_id(session.GetId()); + request.set_session_id(session.GetId()); request.set_tx_id(tx.GetId()); - return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::RollbackTransactionRequest, Ydb::Table::RollbackTransactionResponse>( + return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::RollbackTransactionRequest, Ydb::Table::RollbackTransactionResponse>( std::move(request), &Ydb::Table::V1::TableService::Stub::AsyncRollbackTransaction, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->GetEndpoint()); + settings.ClientTimeout_, + session.SessionImpl_->GetEndpoint()); } - TAsyncExplainDataQueryResult ExplainDataQuery(const TSession& session, const TString& query, + TAsyncExplainDataQueryResult ExplainDataQuery(const TSession& session, const TString& query, const TExplainDataQuerySettings& settings) { auto request = MakeOperationRequest<Ydb::Table::ExplainDataQueryRequest>(settings); - request.set_session_id(session.GetId()); + request.set_session_id(session.GetId()); request.set_yql_text(query); auto promise = NewPromise<TExplainQueryResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { TString ast; TString plan; if (any) { - Ydb::Table::ExplainQueryResult result; + Ydb::Table::ExplainQueryResult result; any->UnpackTo(&result); ast = result.query_ast(); plan = result.query_plan(); } - TExplainQueryResult val(TStatus(std::move(status)), + TExplainQueryResult val(TStatus(std::move(status)), std::move(plan), std::move(ast)); promise.SetValue(std::move(val)); }; - Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::ExplainDataQueryRequest, Ydb::Table::ExplainDataQueryResponse>( + Connections_->RunDeferred<Ydb::Table::V1::TableService, Ydb::Table::ExplainDataQueryRequest, Ydb::Table::ExplainDataQueryResponse>( std::move(request), extractor, - &Ydb::Table::V1::TableService::Stub::AsyncExplainDataQuery, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + &Ydb::Table::V1::TableService::Stub::AsyncExplainDataQuery, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->GetEndpoint()); + settings.ClientTimeout_, + session.SessionImpl_->GetEndpoint()); return promise.GetFuture(); } @@ -2368,158 +2368,158 @@ public: protoValue->mutable_value()->CopyFrom(TProtoAccessor::GetProto(value)); } - NThreading::TFuture<std::pair<TPlainStatus, TReadTableStreamProcessorPtr>> ReadTable( - const TString& sessionId, - const TString& path, - const TReadTableSettings& settings) - { + NThreading::TFuture<std::pair<TPlainStatus, TReadTableStreamProcessorPtr>> ReadTable( + const TString& sessionId, + const TString& path, + const TReadTableSettings& settings) + { auto request = MakeRequest<Ydb::Table::ReadTableRequest>(); - request.set_session_id(sessionId); - request.set_path(path); - request.set_ordered(settings.Ordered_); - if (settings.RowLimit_) { - request.set_row_limit(settings.RowLimit_.GetRef()); - } - for (const auto& col : settings.Columns_) { - request.add_columns(col); - } + request.set_session_id(sessionId); + request.set_path(path); + request.set_ordered(settings.Ordered_); + if (settings.RowLimit_) { + request.set_row_limit(settings.RowLimit_.GetRef()); + } + for (const auto& col : settings.Columns_) { + request.add_columns(col); + } if (settings.UseSnapshot_) { request.set_use_snapshot( settings.UseSnapshot_.GetRef() ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); } - - if (settings.From_) { - const auto& from = settings.From_.GetRef(); - if (from.IsInclusive()) { - SetTypedValue(request.mutable_key_range()->mutable_greater_or_equal(), from.GetValue()); - } else { - SetTypedValue(request.mutable_key_range()->mutable_greater(), from.GetValue()); - } - } - - if (settings.To_) { - const auto& to = settings.To_.GetRef(); - if (to.IsInclusive()) { - SetTypedValue(request.mutable_key_range()->mutable_less_or_equal(), to.GetValue()); - } else { - SetTypedValue(request.mutable_key_range()->mutable_less(), to.GetValue()); - } - } - - auto promise = NewPromise<std::pair<TPlainStatus, TReadTableStreamProcessorPtr>>(); - - Connections_->StartReadStream<Ydb::Table::V1::TableService, Ydb::Table::ReadTableRequest, Ydb::Table::ReadTableResponse>( + + if (settings.From_) { + const auto& from = settings.From_.GetRef(); + if (from.IsInclusive()) { + SetTypedValue(request.mutable_key_range()->mutable_greater_or_equal(), from.GetValue()); + } else { + SetTypedValue(request.mutable_key_range()->mutable_greater(), from.GetValue()); + } + } + + if (settings.To_) { + const auto& to = settings.To_.GetRef(); + if (to.IsInclusive()) { + SetTypedValue(request.mutable_key_range()->mutable_less_or_equal(), to.GetValue()); + } else { + SetTypedValue(request.mutable_key_range()->mutable_less(), to.GetValue()); + } + } + + auto promise = NewPromise<std::pair<TPlainStatus, TReadTableStreamProcessorPtr>>(); + + Connections_->StartReadStream<Ydb::Table::V1::TableService, Ydb::Table::ReadTableRequest, Ydb::Table::ReadTableResponse>( std::move(request), - [promise] (TPlainStatus status, TReadTableStreamProcessorPtr processor) mutable { - promise.SetValue(std::make_pair(status, processor)); - }, - &Ydb::Table::V1::TableService::Stub::AsyncStreamReadTable, - DbDriverState_, + [promise] (TPlainStatus status, TReadTableStreamProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Table::V1::TableService::Stub::AsyncStreamReadTable, + DbDriverState_, TRpcRequestSettings::Make(settings)); - - return promise.GetFuture(); - - } - - TAsyncStatus Close(const TSession::TImpl* sessionImpl, const TCloseSessionSettings& settings) { + + return promise.GetFuture(); + + } + + TAsyncStatus Close(const TSession::TImpl* sessionImpl, const TCloseSessionSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::DeleteSessionRequest>(settings); - request.set_session_id(sessionImpl->GetId()); - return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::DeleteSessionRequest, Ydb::Table::DeleteSessionResponse>( + request.set_session_id(sessionImpl->GetId()); + return RunSimple<Ydb::Table::V1::TableService, Ydb::Table::DeleteSessionRequest, Ydb::Table::DeleteSessionResponse>( std::move(request), &Ydb::Table::V1::TableService::Stub::AsyncDeleteSession, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - sessionImpl->GetEndpoint()); - } - - TAsyncStatus CloseInternal(const TSession::TImpl* sessionImpl) { - static const auto internalCloseSessionSettings = TCloseSessionSettings() - .ClientTimeout(TDuration::Seconds(2)); - - auto driver = Connections_; - return Close(sessionImpl, internalCloseSessionSettings) - .Apply([driver{std::move(driver)}](TAsyncStatus status) mutable - { - driver.reset(); - return status; - }); - } - - bool ReturnSession(TSession::TImpl* sessionImpl) { - Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_ACTIVE|| - sessionImpl->GetState() == TSession::TImpl::S_DISCONNECTED || - sessionImpl->GetState() == TSession::TImpl::S_IDLE); - - if (RequestMigrator_.DoCheckAndMigrate(sessionImpl, shared_from_this())) { - SessionRemovedDueBalancing.Inc(); - return false; - } - if (sessionImpl->GetState() == TSession::TImpl::S_ACTIVE || - sessionImpl->GetState() == TSession::TImpl::S_IDLE) { - bool needUpdateCounter = sessionImpl->NeedUpdateActiveCounter(); - // Also removes NeedUpdateActiveCounter flag - sessionImpl->MarkIdle(); - sessionImpl->SetTimeInterval(TDuration::Zero()); - if (!SessionPool_.ReturnSession(sessionImpl, needUpdateCounter)) { - sessionImpl->SetNeedUpdateActiveCounter(needUpdateCounter); - return false; - } - } else { - if (sessionImpl->NeedUpdateActiveCounter()) { - SessionPool_.DecrementActiveCounter(); - sessionImpl->SetNeedUpdateActiveCounter(false); - } - if (!sessionImpl->GetTimeInterval()) { - // Settler keep-alive task set time interval to non zero value always - // this value is zero if session in session pool or active state. - // So, at _this_ point we know session was returned _not_ from settler keep-alive task: - // - we can just run keep-alive task here - - // To start time counting from scratch - sessionImpl->ScheduleTimeToTouchFast(TDuration::Zero(), true); - auto client = shared_from_this(); - TSession session(client, - std::shared_ptr<TSession::TImpl>( - sessionImpl, - TSession::TImpl::GetSmartDeleter(client))); - DoSettlerKeepAlive(std::move(session)); - } else { - // Session returns from settler keep-alive task - - if (!SettlerPool_.ReturnSession(sessionImpl, false)) { - return false; - } - } - } - return true; - } - - void DeleteSession(TSession::TImpl* sessionImpl) { - if (sessionImpl->NeedUpdateActiveCounter()) { - SessionPool_.DecrementActiveCounter(); - } - - if (sessionImpl->GetId()) { - CloseInternal(sessionImpl); - DbDriverState_->StatCollector.DecSessionsOnHost(sessionImpl->Endpoint_); - } - - delete sessionImpl; - } - - ui32 GetSessionRetryLimit() const { - return Settings_.SessionPoolSettings_.RetryLimit_; - } - + settings.ClientTimeout_, + sessionImpl->GetEndpoint()); + } + + TAsyncStatus CloseInternal(const TSession::TImpl* sessionImpl) { + static const auto internalCloseSessionSettings = TCloseSessionSettings() + .ClientTimeout(TDuration::Seconds(2)); + + auto driver = Connections_; + return Close(sessionImpl, internalCloseSessionSettings) + .Apply([driver{std::move(driver)}](TAsyncStatus status) mutable + { + driver.reset(); + return status; + }); + } + + bool ReturnSession(TSession::TImpl* sessionImpl) { + Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_ACTIVE|| + sessionImpl->GetState() == TSession::TImpl::S_DISCONNECTED || + sessionImpl->GetState() == TSession::TImpl::S_IDLE); + + if (RequestMigrator_.DoCheckAndMigrate(sessionImpl, shared_from_this())) { + SessionRemovedDueBalancing.Inc(); + return false; + } + if (sessionImpl->GetState() == TSession::TImpl::S_ACTIVE || + sessionImpl->GetState() == TSession::TImpl::S_IDLE) { + bool needUpdateCounter = sessionImpl->NeedUpdateActiveCounter(); + // Also removes NeedUpdateActiveCounter flag + sessionImpl->MarkIdle(); + sessionImpl->SetTimeInterval(TDuration::Zero()); + if (!SessionPool_.ReturnSession(sessionImpl, needUpdateCounter)) { + sessionImpl->SetNeedUpdateActiveCounter(needUpdateCounter); + return false; + } + } else { + if (sessionImpl->NeedUpdateActiveCounter()) { + SessionPool_.DecrementActiveCounter(); + sessionImpl->SetNeedUpdateActiveCounter(false); + } + if (!sessionImpl->GetTimeInterval()) { + // Settler keep-alive task set time interval to non zero value always + // this value is zero if session in session pool or active state. + // So, at _this_ point we know session was returned _not_ from settler keep-alive task: + // - we can just run keep-alive task here + + // To start time counting from scratch + sessionImpl->ScheduleTimeToTouchFast(TDuration::Zero(), true); + auto client = shared_from_this(); + TSession session(client, + std::shared_ptr<TSession::TImpl>( + sessionImpl, + TSession::TImpl::GetSmartDeleter(client))); + DoSettlerKeepAlive(std::move(session)); + } else { + // Session returns from settler keep-alive task + + if (!SettlerPool_.ReturnSession(sessionImpl, false)) { + return false; + } + } + } + return true; + } + + void DeleteSession(TSession::TImpl* sessionImpl) { + if (sessionImpl->NeedUpdateActiveCounter()) { + SessionPool_.DecrementActiveCounter(); + } + + if (sessionImpl->GetId()) { + CloseInternal(sessionImpl); + DbDriverState_->StatCollector.DecSessionsOnHost(sessionImpl->Endpoint_); + } + + delete sessionImpl; + } + + ui32 GetSessionRetryLimit() const { + return Settings_.SessionPoolSettings_.RetryLimit_; + } + void SetStatCollector(const NSdkStats::TStatCollector::TClientStatCollector& collector) { CacheMissCounter.Set(collector.CacheMiss); QuerySizeHistogram.Set(collector.QuerySize); ParamsSizeHistogram.Set(collector.ParamsSize); RetryOperationStatCollector = collector.RetryOperationStatCollector; - SessionRemovedDueBalancing.Set(collector.SessionRemovedDueBalancing); - RequestMigrated.Set(collector.RequestMigrated); + SessionRemovedDueBalancing.Set(collector.SessionRemovedDueBalancing); + RequestMigrated.Set(collector.RequestMigrated); } TAsyncBulkUpsertResult BulkUpsert(const TString& table, TValue&& rows, const TBulkUpsertSettings& settings) { @@ -2532,10 +2532,10 @@ public: auto promise = NewPromise<TBulkUpsertResult>(); - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { Y_UNUSED(any); - TBulkUpsertResult val(TStatus(std::move(status))); + TBulkUpsertResult val(TStatus(std::move(status))); promise.SetValue(std::move(val)); }; @@ -2544,7 +2544,7 @@ public: extractor, &Ydb::Table::V1::TableService::Stub::AsyncBulkUpsert, DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); @@ -2635,16 +2635,16 @@ public: { auto promise = NewPromise<TScanQueryPartIterator>(); - auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, + auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, TTableClient::TImpl::TScanQueryProcessorPtr>> future) mutable { Y_ASSERT(future.HasValue()); auto pair = future.ExtractValue(); promise.SetValue(TScanQueryPartIterator( pair.second - ? std::make_shared<TScanQueryPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) + ? std::make_shared<TScanQueryPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) : nullptr, - std::move(pair.first)) + std::move(pair.first)) ); }; @@ -2652,29 +2652,29 @@ public: return promise.GetFuture(); } - static void CloseAndDeleteSession( - std::unique_ptr<TSession::TImpl>&& impl, - std::shared_ptr<TTableClient::TImpl> client); + static void CloseAndDeleteSession( + std::unique_ptr<TSession::TImpl>&& impl, + std::shared_ptr<TTableClient::TImpl> client); public: TClientSettings Settings_; private: - static void SetParams( - ::google::protobuf::Map<TString, Ydb::TypedValue>* params, - Ydb::Table::ExecuteDataQueryRequest* request) - { - if (params) { - request->mutable_parameters()->swap(*params); - } - } - - static void SetParams( - const ::google::protobuf::Map<TString, Ydb::TypedValue>& params, - Ydb::Table::ExecuteDataQueryRequest* request) - { - *request->mutable_parameters() = params; - } - + static void SetParams( + ::google::protobuf::Map<TString, Ydb::TypedValue>* params, + Ydb::Table::ExecuteDataQueryRequest* request) + { + if (params) { + request->mutable_parameters()->swap(*params); + } + } + + static void SetParams( + const ::google::protobuf::Map<TString, Ydb::TypedValue>& params, + Ydb::Table::ExecuteDataQueryRequest* request) + { + *request->mutable_parameters() = params; + } + static void CollectParams( ::google::protobuf::Map<TString, Ydb::TypedValue>* params, NSdkStats::TAtomicHistogram<NMonitoring::THistogram> histgoram) @@ -2711,9 +2711,9 @@ private: static void CollectQuerySize(const TDataQuery&, NSdkStats::TAtomicHistogram<NMonitoring::THistogram>&) {} - template <typename TQueryType, typename TParamsType> + template <typename TQueryType, typename TParamsType> TAsyncDataQueryResult ExecuteDataQueryInternal(const TSession& session, const TQueryType& query, - const TTxControl& txControl, TParamsType params, + const TTxControl& txControl, TParamsType params, const TExecDataQuerySettings& settings, bool fromCache) { auto request = MakeOperationRequest<Ydb::Table::ExecuteDataQueryRequest>(settings); @@ -2723,7 +2723,7 @@ private: if (txControl.TxId_) { txControlProto->set_tx_id(*txControl.TxId_); } else { - SetTxSettings(txControl.BeginTx_, txControlProto->mutable_begin_tx()); + SetTxSettings(txControl.BeginTx_, txControlProto->mutable_begin_tx()); } request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); @@ -2731,7 +2731,7 @@ private: SetQuery(query, request.mutable_query()); CollectQuerySize(query, QuerySizeHistogram); - SetParams(params, &request); + SetParams(params, &request); CollectParams(params, ParamsSizeHistogram); SetQueryCachePolicy(query, settings, request.mutable_query_cache_policy()); @@ -2739,17 +2739,17 @@ private: auto promise = NewPromise<TDataQueryResult>(); bool keepInCache = settings.KeepInQueryCache_ && settings.KeepInQueryCache_.GetRef(); - // We don't want to delay call of TSession dtor, so we can't capture it by copy - // otherwise we break session pool and other clients logic. - // Same problem with TDataQuery and TTransaction - // - // The fast solution is: - // - create copy of TSession out of lambda - // - capture pointer - // - call free just before SetValue call - auto sessionPtr = new TSession(session); - auto extractor = [promise, sessionPtr, query, fromCache, keepInCache] - (google::protobuf::Any* any, TPlainStatus status) mutable { + // We don't want to delay call of TSession dtor, so we can't capture it by copy + // otherwise we break session pool and other clients logic. + // Same problem with TDataQuery and TTransaction + // + // The fast solution is: + // - create copy of TSession out of lambda + // - capture pointer + // - call free just before SetValue call + auto sessionPtr = new TSession(session); + auto extractor = [promise, sessionPtr, query, fromCache, keepInCache] + (google::protobuf::Any* any, TPlainStatus status) mutable { TVector<TResultSet> res; TMaybe<TTransaction> tx; TMaybe<TDataQuery> dataQuery; @@ -2765,13 +2765,13 @@ private: } if (result.has_tx_meta()) { - tx = TTransaction(*sessionPtr, result.tx_meta().id()); + tx = TTransaction(*sessionPtr, result.tx_meta().id()); } if (result.has_query_meta()) { if (queryText) { auto& query_meta = result.query_meta(); - dataQuery = TDataQuery(*sessionPtr, *queryText, query_meta.id(), query_meta.parameters_types()); + dataQuery = TDataQuery(*sessionPtr, *queryText, query_meta.id(), query_meta.parameters_types()); } } @@ -2784,12 +2784,12 @@ private: sessionPtr->SessionImpl_->AddQueryToCache(*dataQuery); } - TDataQueryResult dataQueryResult(TStatus(std::move(status)), + TDataQueryResult dataQueryResult(TStatus(std::move(status)), std::move(res), tx, dataQuery, fromCache, queryStats); - - delete sessionPtr; - tx.Clear(); - dataQuery.Clear(); + + delete sessionPtr; + tx.Clear(); + dataQuery.Clear(); promise.SetValue(std::move(dataQueryResult)); }; @@ -2797,16 +2797,16 @@ private: std::move(request), extractor, &Ydb::Table::V1::TableService::Stub::AsyncExecuteDataQuery, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_, - session.SessionImpl_->Endpoint_); + settings.ClientTimeout_, + session.SessionImpl_->Endpoint_); return promise.GetFuture(); } - static void SetTxSettings(const TTxSettings& txSettings, Ydb::Table::TransactionSettings* proto) + static void SetTxSettings(const TTxSettings& txSettings, Ydb::Table::TransactionSettings* proto) { switch (txSettings.Mode_) { case TTxSettings::TS_SERIALIZABLE_RW: @@ -2820,18 +2820,18 @@ private: proto->mutable_stale_read_only(); break; default: - throw TContractViolation("Unexpected transaction mode."); + throw TContractViolation("Unexpected transaction mode."); } } - static void SetQuery(const TString& queryText, Ydb::Table::Query* query) { + static void SetQuery(const TString& queryText, Ydb::Table::Query* query) { query->set_yql_text(queryText); } - static void SetQuery(const TDataQuery& queryData, Ydb::Table::Query* query) { + static void SetQuery(const TDataQuery& queryData, Ydb::Table::Query* query) { query->set_id(queryData.GetId()); } - + static void SetQueryCachePolicy(const TString&, const TExecDataQuerySettings& settings, Ydb::Table::QueryCachePolicy* queryCachePolicy) { @@ -2851,196 +2851,196 @@ private: return queryData.GetText(); } - static TAsyncCreateSessionResult SettlerAwareSessonProvider( - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings); - + static TAsyncCreateSessionResult SettlerAwareSessonProvider( + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings); + public: - NSdkStats::TAtomicCounter<NMonitoring::TRate> CacheMissCounter; + NSdkStats::TAtomicCounter<NMonitoring::TRate> CacheMissCounter; NSdkStats::TStatCollector::TClientRetryOperationStatCollector RetryOperationStatCollector; NSdkStats::TAtomicHistogram<NMonitoring::THistogram> QuerySizeHistogram; NSdkStats::TAtomicHistogram<NMonitoring::THistogram> ParamsSizeHistogram; - NSdkStats::TAtomicCounter<NMonitoring::TRate> SessionRemovedDueBalancing; - NSdkStats::TAtomicCounter<NMonitoring::TRate> RequestMigrated; - + NSdkStats::TAtomicCounter<NMonitoring::TRate> SessionRemovedDueBalancing; + NSdkStats::TAtomicCounter<NMonitoring::TRate> RequestMigrated; + private: - TSessionPoolImpl SessionPool_; - TSessionPoolImpl SettlerPool_; - TRequestMigrator RequestMigrator_; - static const TKeepAliveSettings KeepAliveSettings; + TSessionPoolImpl SessionPool_; + TSessionPoolImpl SettlerPool_; + TRequestMigrator RequestMigrator_; + static const TKeepAliveSettings KeepAliveSettings; }; -const TKeepAliveSettings TTableClient::TImpl::KeepAliveSettings = TKeepAliveSettings().ClientTimeout(KEEP_ALIVE_CLIENT_TIMEOUT); - -TSessionPoolImpl::TSessionPoolImpl(ui32 maxActiveSessions) - : Closed_(false) - , ActiveSessions_(0) - , MaxActiveSessions_(maxActiveSessions) -{} - -void TTableClient::TImpl::CloseAndDeleteSession(std::unique_ptr<TSession::TImpl>&& impl, - std::shared_ptr<TTableClient::TImpl> client) { - std::shared_ptr<TSession::TImpl> deleteSession( - impl.release(), - TSession::TImpl::GetSmartDeleter(client)); - - deleteSession->MarkBroken(); -} - -TAsyncCreateSessionResult TTableClient::TImpl::SettlerAwareSessonProvider( - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings) -{ - if (client->SessionPool_.GetActiveSessionsLimit() == 0) { - return client->CreateSession(settings, false); - } - - auto sessionsInSettler = client->SettlerPool_.GetCurrentPoolSize(); - // The purpose is just protect server from huge number of sessions in case - // of transport errors, this check is a bit inaccurate - - // The alternative way is to close sessions in settler during periodic task, - // but it can cause additional latency to close. - - if (sessionsInSettler > client->SessionPool_.GetActiveSessionsLimit()) { - std::unique_ptr<TSession::TImpl> session; - auto drainer = [&session](std::unique_ptr<TSession::TImpl>&& impl) mutable { - session = std::move(impl); - return false; - }; - client->SettlerPool_.Drain(drainer, false); - if (session) { - CloseAndDeleteSession(std::move(session), client); - } - } - - return client->CreateSession(settings, false); -} - -void TSessionPoolImpl::CreateFakeSession( - NThreading::TPromise<TCreateSessionResult>& promise, - std::shared_ptr<TTableClient::TImpl> client) -{ - TSession session(client, "", ""); - // Mark broken to prevent returning to session pool - session.SessionImpl_->MarkBroken(); - TCreateSessionResult val( - TStatus( - TPlainStatus( - EStatus::CLIENT_RESOURCE_EXHAUSTED, - "Active sessions limit exceeded" - ) - ), - std::move(session) - ); - - client->ScheduleTaskUnsafe([promise, val{std::move(val)}]() mutable { - promise.SetValue(std::move(val)); - }, TDuration()); -} - -TAsyncCreateSessionResult TSessionPoolImpl::GetSession( - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings, - TAwareSessonProvider sessionProvider) -{ - auto createSessionPromise = NewPromise<TCreateSessionResult>(); - std::unique_ptr<TSession::TImpl> sessionImpl; - bool needUpdateActiveSessionCounter = false; - bool returnFakeSession = false; +const TKeepAliveSettings TTableClient::TImpl::KeepAliveSettings = TKeepAliveSettings().ClientTimeout(KEEP_ALIVE_CLIENT_TIMEOUT); + +TSessionPoolImpl::TSessionPoolImpl(ui32 maxActiveSessions) + : Closed_(false) + , ActiveSessions_(0) + , MaxActiveSessions_(maxActiveSessions) +{} + +void TTableClient::TImpl::CloseAndDeleteSession(std::unique_ptr<TSession::TImpl>&& impl, + std::shared_ptr<TTableClient::TImpl> client) { + std::shared_ptr<TSession::TImpl> deleteSession( + impl.release(), + TSession::TImpl::GetSmartDeleter(client)); + + deleteSession->MarkBroken(); +} + +TAsyncCreateSessionResult TTableClient::TImpl::SettlerAwareSessonProvider( + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings) +{ + if (client->SessionPool_.GetActiveSessionsLimit() == 0) { + return client->CreateSession(settings, false); + } + + auto sessionsInSettler = client->SettlerPool_.GetCurrentPoolSize(); + // The purpose is just protect server from huge number of sessions in case + // of transport errors, this check is a bit inaccurate + + // The alternative way is to close sessions in settler during periodic task, + // but it can cause additional latency to close. + + if (sessionsInSettler > client->SessionPool_.GetActiveSessionsLimit()) { + std::unique_ptr<TSession::TImpl> session; + auto drainer = [&session](std::unique_ptr<TSession::TImpl>&& impl) mutable { + session = std::move(impl); + return false; + }; + client->SettlerPool_.Drain(drainer, false); + if (session) { + CloseAndDeleteSession(std::move(session), client); + } + } + + return client->CreateSession(settings, false); +} + +void TSessionPoolImpl::CreateFakeSession( + NThreading::TPromise<TCreateSessionResult>& promise, + std::shared_ptr<TTableClient::TImpl> client) +{ + TSession session(client, "", ""); + // Mark broken to prevent returning to session pool + session.SessionImpl_->MarkBroken(); + TCreateSessionResult val( + TStatus( + TPlainStatus( + EStatus::CLIENT_RESOURCE_EXHAUSTED, + "Active sessions limit exceeded" + ) + ), + std::move(session) + ); + + client->ScheduleTaskUnsafe([promise, val{std::move(val)}]() mutable { + promise.SetValue(std::move(val)); + }, TDuration()); +} + +TAsyncCreateSessionResult TSessionPoolImpl::GetSession( + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings, + TAwareSessonProvider sessionProvider) +{ + auto createSessionPromise = NewPromise<TCreateSessionResult>(); + std::unique_ptr<TSession::TImpl> sessionImpl; + bool needUpdateActiveSessionCounter = false; + bool returnFakeSession = false; { std::lock_guard guard(Mtx_); - if (MaxActiveSessions_) { - if (ActiveSessions_ < MaxActiveSessions_) { - ActiveSessions_++; - needUpdateActiveSessionCounter = true; - } else { - returnFakeSession = true; - } - } else { - ActiveSessions_++; - needUpdateActiveSessionCounter = true; - } - if (!Sessions_.empty()) { - auto it = std::prev(Sessions_.end()); - sessionImpl = std::move(it->second); - Sessions_.erase(it); - } + if (MaxActiveSessions_) { + if (ActiveSessions_ < MaxActiveSessions_) { + ActiveSessions_++; + needUpdateActiveSessionCounter = true; + } else { + returnFakeSession = true; + } + } else { + ActiveSessions_++; + needUpdateActiveSessionCounter = true; + } + if (!Sessions_.empty()) { + auto it = std::prev(Sessions_.end()); + sessionImpl = std::move(it->second); + Sessions_.erase(it); + } UpdateStats(); - } - if (returnFakeSession) { + } + if (returnFakeSession) { FakeSessionsCounter_.Inc(); - CreateFakeSession(createSessionPromise, client); - return createSessionPromise.GetFuture(); - } else if (sessionImpl) { - Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_IDLE); - Y_VERIFY(!sessionImpl->GetTimeInterval()); - Y_VERIFY(needUpdateActiveSessionCounter); - sessionImpl->MarkActive(); - sessionImpl->SetNeedUpdateActiveCounter(true); - - TCreateSessionResult val(TStatus(TPlainStatus()), + CreateFakeSession(createSessionPromise, client); + return createSessionPromise.GetFuture(); + } else if (sessionImpl) { + Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_IDLE); + Y_VERIFY(!sessionImpl->GetTimeInterval()); + Y_VERIFY(needUpdateActiveSessionCounter); + sessionImpl->MarkActive(); + sessionImpl->SetNeedUpdateActiveCounter(true); + + TCreateSessionResult val(TStatus(TPlainStatus()), TSession(client, std::shared_ptr<TSession::TImpl>( sessionImpl.release(), TSession::TImpl::GetSmartDeleter(client)))); - - client->ScheduleTaskUnsafe([createSessionPromise, val{std::move(val)}]() mutable { - createSessionPromise.SetValue(std::move(val)); - }, TDuration()); - - return createSessionPromise.GetFuture(); - } else { - const auto& sessionResult = sessionProvider(client, settings); - sessionResult.Subscribe(TSession::TImpl::GetSessionInspector(createSessionPromise, client, settings, 0, needUpdateActiveSessionCounter)); - return createSessionPromise.GetFuture(); - } -} - -bool TSessionPoolImpl::DropSessionOnEndpoint(std::shared_ptr<TTableClient::TImpl> client, const TString& endpoint) { - std::unique_ptr<TSession::TImpl> sessionImpl; + + client->ScheduleTaskUnsafe([createSessionPromise, val{std::move(val)}]() mutable { + createSessionPromise.SetValue(std::move(val)); + }, TDuration()); + + return createSessionPromise.GetFuture(); + } else { + const auto& sessionResult = sessionProvider(client, settings); + sessionResult.Subscribe(TSession::TImpl::GetSessionInspector(createSessionPromise, client, settings, 0, needUpdateActiveSessionCounter)); + return createSessionPromise.GetFuture(); + } +} + +bool TSessionPoolImpl::DropSessionOnEndpoint(std::shared_ptr<TTableClient::TImpl> client, const TString& endpoint) { + std::unique_ptr<TSession::TImpl> sessionImpl; { std::lock_guard guard(Mtx_); - for (auto it = Sessions_.begin(); it != Sessions_.end(); it++) { - if (it->second->GetEndpoint() == endpoint) { - sessionImpl = std::move(it->second); - Sessions_.erase(it); - break; - } - } - } - if (!sessionImpl) - return false; - - auto deleteFn = TSession::TImpl::GetSmartDeleter(client); - deleteFn(sessionImpl.release()); - - return true; -} - -bool TSessionPoolImpl::ReturnSession(TSession::TImpl* impl, bool active) { + for (auto it = Sessions_.begin(); it != Sessions_.end(); it++) { + if (it->second->GetEndpoint() == endpoint) { + sessionImpl = std::move(it->second); + Sessions_.erase(it); + break; + } + } + } + if (!sessionImpl) + return false; + + auto deleteFn = TSession::TImpl::GetSmartDeleter(client); + deleteFn(sessionImpl.release()); + + return true; +} + +bool TSessionPoolImpl::ReturnSession(TSession::TImpl* impl, bool active) { { std::lock_guard guard(Mtx_); - if (Closed_) - return false; - Sessions_.emplace(std::make_pair(impl->GetTimeToTouchFast(), impl)); - if (active) { - Y_VERIFY(ActiveSessions_); - ActiveSessions_--; - impl->SetNeedUpdateActiveCounter(false); - } + if (Closed_) + return false; + Sessions_.emplace(std::make_pair(impl->GetTimeToTouchFast(), impl)); + if (active) { + Y_VERIFY(ActiveSessions_); + ActiveSessions_--; + impl->SetNeedUpdateActiveCounter(false); + } UpdateStats(); - } - return true; -} - -void TSessionPoolImpl::DecrementActiveCounter() { + } + return true; +} + +void TSessionPoolImpl::DecrementActiveCounter() { std::lock_guard guard(Mtx_); Y_VERIFY(ActiveSessions_); ActiveSessions_--; UpdateStats(); -} - -void TSessionPoolImpl::Drain(std::function<bool(std::unique_ptr<TSession::TImpl>&&)> cb, bool close) { +} + +void TSessionPoolImpl::Drain(std::function<bool(std::unique_ptr<TSession::TImpl>&&)> cb, bool close) { std::lock_guard guard(Mtx_); Closed_ = close; for (auto it = Sessions_.begin(); it != Sessions_.end();) { @@ -3048,129 +3048,129 @@ void TSessionPoolImpl::Drain(std::function<bool(std::unique_ptr<TSession::TImpl> it = Sessions_.erase(it); if (!cont) break; - } + } UpdateStats(); -} - -TPeriodicCb TSessionPoolImpl::CreatePeriodicTask(std::weak_ptr<TTableClient::TImpl> weakClient, +} + +TPeriodicCb TSessionPoolImpl::CreatePeriodicTask(std::weak_ptr<TTableClient::TImpl> weakClient, TKeepAliveCmd&& cmd, TDeletePredicate&& deletePredicate) -{ +{ auto periodicCb = [this, weakClient, cmd=std::move(cmd), deletePredicate=std::move(deletePredicate)](NYql::TIssues&&, EStatus status) { - if (status != EStatus::SUCCESS) { - return false; - } - - auto strongClient = weakClient.lock(); - if (!strongClient) { - // No more clients alive - no need to run periodic, - // moreover it is unsafe to touch this ptr! - return false; - } else { - auto keepAliveBatchSize = PERIODIC_ACTION_BATCH_SIZE; - TVector<std::unique_ptr<TSession::TImpl>> sessionsToTouch; - sessionsToTouch.reserve(keepAliveBatchSize); + if (status != EStatus::SUCCESS) { + return false; + } + + auto strongClient = weakClient.lock(); + if (!strongClient) { + // No more clients alive - no need to run periodic, + // moreover it is unsafe to touch this ptr! + return false; + } else { + auto keepAliveBatchSize = PERIODIC_ACTION_BATCH_SIZE; + TVector<std::unique_ptr<TSession::TImpl>> sessionsToTouch; + sessionsToTouch.reserve(keepAliveBatchSize); TVector<std::unique_ptr<TSession::TImpl>> sessionsToDelete; sessionsToDelete.reserve(keepAliveBatchSize); - auto now = TInstant::Now(); + auto now = TInstant::Now(); { std::lock_guard guard(Mtx_); - auto& sessions = Sessions_; - - auto it = sessions.begin(); - while (it != sessions.end() && keepAliveBatchSize--) { - if (now < it->second->GetTimeToTouchFast()) - break; + auto& sessions = Sessions_; + + auto it = sessions.begin(); + while (it != sessions.end() && keepAliveBatchSize--) { + if (now < it->second->GetTimeToTouchFast()) + break; if (deletePredicate(it->second.get(), strongClient.get(), sessions.size())) { sessionsToDelete.emplace_back(std::move(it->second)); } else { sessionsToTouch.emplace_back(std::move(it->second)); } - sessions.erase(it++); - } + sessions.erase(it++); + } UpdateStats(); - } - - for (auto& sessionImpl : sessionsToTouch) { - if (sessionImpl) { - Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_IDLE || - sessionImpl->GetState() == TSession::TImpl::S_DISCONNECTED); - TSession session(strongClient, std::shared_ptr<TSession::TImpl>( - sessionImpl.release(), - TSession::TImpl::GetSmartDeleter(strongClient))); - cmd(session); - } - } + } + + for (auto& sessionImpl : sessionsToTouch) { + if (sessionImpl) { + Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_IDLE || + sessionImpl->GetState() == TSession::TImpl::S_DISCONNECTED); + TSession session(strongClient, std::shared_ptr<TSession::TImpl>( + sessionImpl.release(), + TSession::TImpl::GetSmartDeleter(strongClient))); + cmd(session); + } + } for (auto& sessionImpl : sessionsToDelete) { if (sessionImpl) { Y_VERIFY(sessionImpl->GetState() == TSession::TImpl::S_IDLE || sessionImpl->GetState() == TSession::TImpl::S_DISCONNECTED); - TTableClient::TImpl::CloseAndDeleteSession(std::move(sessionImpl), strongClient); + TTableClient::TImpl::CloseAndDeleteSession(std::move(sessionImpl), strongClient); } } - } - - return true; - }; - return periodicCb; -} - -i64 TSessionPoolImpl::GetActiveSessions() const { + } + + return true; + }; + return periodicCb; +} + +i64 TSessionPoolImpl::GetActiveSessions() const { std::lock_guard guard(Mtx_); return ActiveSessions_; -} - -i64 TSessionPoolImpl::GetActiveSessionsLimit() const { - return MaxActiveSessions_; -} - -i64 TSessionPoolImpl::GetCurrentPoolSize() const { +} + +i64 TSessionPoolImpl::GetActiveSessionsLimit() const { + return MaxActiveSessions_; +} + +i64 TSessionPoolImpl::GetCurrentPoolSize() const { std::lock_guard guard(Mtx_); return Sessions_.size(); -} - -static bool IsSessionStatusRetriable(const TCreateSessionResult& res) { - switch (res.GetStatus()) { - case EStatus::OVERLOADED: - // For CreateSession request we can retry some of transport errors - // - endpoind will be pessimized and session will be created on the - // another endpoint - case EStatus::CLIENT_DEADLINE_EXCEEDED: - case EStatus::CLIENT_RESOURCE_EXHAUSTED: - case EStatus::TRANSPORT_UNAVAILABLE: - return true; - default: - return false; - } -} - -TSessionInspectorFn TSession::TImpl::GetSessionInspector( - NThreading::TPromise<TCreateSessionResult>& promise, - std::shared_ptr<TTableClient::TImpl> client, - const TCreateSessionSettings& settings, - ui32 counter, bool needUpdateActiveSessionCounter) -{ - return [promise, client, settings, counter, needUpdateActiveSessionCounter](TAsyncCreateSessionResult future) mutable { - Y_ASSERT(future.HasValue()); - auto session = future.ExtractValue(); - if (IsSessionStatusRetriable(session) && counter < client->GetSessionRetryLimit()) { - counter++; - client->CreateSession(settings, false) - .Subscribe(GetSessionInspector( - promise, - client, - settings, - counter, - needUpdateActiveSessionCounter) - ); - } else { - session.Session_.SessionImpl_->SetNeedUpdateActiveCounter(needUpdateActiveSessionCounter); - promise.SetValue(std::move(session)); - } - }; -} - +} + +static bool IsSessionStatusRetriable(const TCreateSessionResult& res) { + switch (res.GetStatus()) { + case EStatus::OVERLOADED: + // For CreateSession request we can retry some of transport errors + // - endpoind will be pessimized and session will be created on the + // another endpoint + case EStatus::CLIENT_DEADLINE_EXCEEDED: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::TRANSPORT_UNAVAILABLE: + return true; + default: + return false; + } +} + +TSessionInspectorFn TSession::TImpl::GetSessionInspector( + NThreading::TPromise<TCreateSessionResult>& promise, + std::shared_ptr<TTableClient::TImpl> client, + const TCreateSessionSettings& settings, + ui32 counter, bool needUpdateActiveSessionCounter) +{ + return [promise, client, settings, counter, needUpdateActiveSessionCounter](TAsyncCreateSessionResult future) mutable { + Y_ASSERT(future.HasValue()); + auto session = future.ExtractValue(); + if (IsSessionStatusRetriable(session) && counter < client->GetSessionRetryLimit()) { + counter++; + client->CreateSession(settings, false) + .Subscribe(GetSessionInspector( + promise, + client, + settings, + counter, + needUpdateActiveSessionCounter) + ); + } else { + session.Session_.SessionImpl_->SetNeedUpdateActiveCounter(needUpdateActiveSessionCounter); + promise.SetValue(std::move(session)); + } + }; +} + void TSessionPoolImpl::SetStatCollector(NSdkStats::TStatCollector::TSessionPoolStatCollector statCollector) { ActiveSessionsCounter_.Set(statCollector.ActiveSessions); InPoolSessionsCounter_.Set(statCollector.InPoolSessions); @@ -3183,27 +3183,27 @@ void TSessionPoolImpl::UpdateStats() { } TTableClient::TTableClient(const TDriver& driver, const TClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) { - Impl_->StartPeriodicSessionPoolTask(); - Impl_->StartPeriodicSettlerTask(); - Impl_->StartPeriodicHostScanTask(); - Impl_->InitStopper(); -} + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) { + Impl_->StartPeriodicSessionPoolTask(); + Impl_->StartPeriodicSettlerTask(); + Impl_->StartPeriodicHostScanTask(); + Impl_->InitStopper(); +} TAsyncCreateSessionResult TTableClient::CreateSession(const TCreateSessionSettings& settings) { - // Returns standalone session - return Impl_->CreateSession(settings, true); + // Returns standalone session + return Impl_->CreateSession(settings, true); } TAsyncCreateSessionResult TTableClient::GetSession(const TCreateSessionSettings& settings) { - // Returns session from session pool - return Impl_->GetSession(settings); -} - -i64 TTableClient::GetActiveSessionCount() const { - return Impl_->GetActiveSessionCount(); -} - + // Returns session from session pool + return Impl_->GetSession(settings); +} + +i64 TTableClient::GetActiveSessionCount() const { + return Impl_->GetActiveSessionCount(); +} + i64 TTableClient::GetActiveSessionsLimit() const { return Impl_->GetActiveSessionsLimit(); } @@ -3217,11 +3217,11 @@ TTableBuilder TTableClient::GetTableBuilder() { } TParamsBuilder TTableClient::GetParamsBuilder() { - return TParamsBuilder(); + return TParamsBuilder(); } TTypeBuilder TTableClient::GetTypeBuilder() { - return TTypeBuilder(); + return TTypeBuilder(); } //////////////////////////////////////////////////////////////////////////////// @@ -3236,7 +3236,7 @@ struct TRetryState { }; static void Backoff(const TBackoffSettings& settings, ui32 retryNumber) { - auto durationMs = CalcBackoffTime(settings, retryNumber); + auto durationMs = CalcBackoffTime(settings, retryNumber); Sleep(TDuration::MilliSeconds(durationMs)); } @@ -3460,7 +3460,7 @@ TStatus TTableClient::RetryOperationSyncHelper(const TOperationWrapperSyncFunc& } case EStatus::BAD_SESSION: - case EStatus::SESSION_BUSY: + case EStatus::SESSION_BUSY: retryState.Session.Clear(); break; @@ -3530,10 +3530,10 @@ TStatus TTableClient::RetryOperationSync(const TOperationSyncFunc& operation, co return RetryOperationSyncHelper(operationWrapper, settings); } -NThreading::TFuture<void> TTableClient::Stop() { - return Impl_->Stop(); -} - +NThreading::TFuture<void> TTableClient::Stop() { + return Impl_->Stop(); +} + TAsyncBulkUpsertResult TTableClient::BulkUpsert(const TString& table, TValue&& rows, const TBulkUpsertSettings& settings) { @@ -3560,27 +3560,27 @@ TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const TString& //////////////////////////////////////////////////////////////////////////////// -static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settings, Ydb::Table::TableProfile* proto) { - if (settings.PresetName_) { - proto->set_preset_name(settings.PresetName_.GetRef()); - } - if (settings.ExecutionPolicy_) { - proto->mutable_execution_policy()->set_preset_name(settings.ExecutionPolicy_.GetRef()); - } - if (settings.CompactionPolicy_) { - proto->mutable_compaction_policy()->set_preset_name(settings.CompactionPolicy_.GetRef()); - } - if (settings.PartitioningPolicy_) { - const auto& policy = settings.PartitioningPolicy_.GetRef(); - if (policy.PresetName_) { - proto->mutable_partitioning_policy()->set_preset_name(policy.PresetName_.GetRef()); - } - if (policy.AutoPartitioning_) { - proto->mutable_partitioning_policy()->set_auto_partitioning(static_cast<Ydb::Table::PartitioningPolicy_AutoPartitioningPolicy>(policy.AutoPartitioning_.GetRef())); - } - if (policy.UniformPartitions_) { - proto->mutable_partitioning_policy()->set_uniform_partitions(policy.UniformPartitions_.GetRef()); - } +static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settings, Ydb::Table::TableProfile* proto) { + if (settings.PresetName_) { + proto->set_preset_name(settings.PresetName_.GetRef()); + } + if (settings.ExecutionPolicy_) { + proto->mutable_execution_policy()->set_preset_name(settings.ExecutionPolicy_.GetRef()); + } + if (settings.CompactionPolicy_) { + proto->mutable_compaction_policy()->set_preset_name(settings.CompactionPolicy_.GetRef()); + } + if (settings.PartitioningPolicy_) { + const auto& policy = settings.PartitioningPolicy_.GetRef(); + if (policy.PresetName_) { + proto->mutable_partitioning_policy()->set_preset_name(policy.PresetName_.GetRef()); + } + if (policy.AutoPartitioning_) { + proto->mutable_partitioning_policy()->set_auto_partitioning(static_cast<Ydb::Table::PartitioningPolicy_AutoPartitioningPolicy>(policy.AutoPartitioning_.GetRef())); + } + if (policy.UniformPartitions_) { + proto->mutable_partitioning_policy()->set_uniform_partitions(policy.UniformPartitions_.GetRef()); + } if (policy.ExplicitPartitions_) { auto* borders = proto->mutable_partitioning_policy()->mutable_explicit_partitions(); for (const auto& splitPoint : policy.ExplicitPartitions_->SplitPoints_) { @@ -3589,24 +3589,24 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); } } - } - if (settings.StoragePolicy_) { - const auto& policy = settings.StoragePolicy_.GetRef(); - if (policy.PresetName_) { - proto->mutable_storage_policy()->set_preset_name(policy.PresetName_.GetRef()); - } - if (policy.SysLog_) { + } + if (settings.StoragePolicy_) { + const auto& policy = settings.StoragePolicy_.GetRef(); + if (policy.PresetName_) { + proto->mutable_storage_policy()->set_preset_name(policy.PresetName_.GetRef()); + } + if (policy.SysLog_) { proto->mutable_storage_policy()->mutable_syslog()->set_media(policy.SysLog_.GetRef()); - } - if (policy.Log_) { + } + if (policy.Log_) { proto->mutable_storage_policy()->mutable_log()->set_media(policy.Log_.GetRef()); - } - if (policy.Data_) { + } + if (policy.Data_) { proto->mutable_storage_policy()->mutable_data()->set_media(policy.Data_.GetRef()); - } - if (policy.External_) { + } + if (policy.External_) { proto->mutable_storage_policy()->mutable_external()->set_media(policy.External_.GetRef()); - } + } for (const auto& familyPolicy : policy.ColumnFamilies_) { auto* familyProto = proto->mutable_storage_policy()->add_column_families(); if (familyPolicy.Name_) { @@ -3631,7 +3631,7 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin : Ydb::Table::ColumnFamilyPolicy::UNCOMPRESSED); } } - } + } if (settings.ReplicationPolicy_) { const auto& policy = settings.ReplicationPolicy_.GetRef(); if (policy.PresetName_) { @@ -3655,30 +3655,30 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin ); } } -} - -//////////////////////////////////////////////////////////////////////////////// - -TSession::TSession(std::shared_ptr<TTableClient::TImpl> client, const TString& sessionId, const TString& endpointId) +} + +//////////////////////////////////////////////////////////////////////////////// + +TSession::TSession(std::shared_ptr<TTableClient::TImpl> client, const TString& sessionId, const TString& endpointId) : Client_(client) - , SessionImpl_(new TSession::TImpl( - sessionId, - endpointId, + , SessionImpl_(new TSession::TImpl( + sessionId, + endpointId, client->Settings_.UseQueryCache_, - client->Settings_.QueryCacheSize_), - TSession::TImpl::GetSmartDeleter(client)) -{ - if (endpointId) { - Client_->LinkObjToEndpoint(endpointId, SessionImpl_.get(), Client_.get()); - } -} - + client->Settings_.QueryCacheSize_), + TSession::TImpl::GetSmartDeleter(client)) +{ + if (endpointId) { + Client_->LinkObjToEndpoint(endpointId, SessionImpl_.get(), Client_.get()); + } +} + TSession::TSession(std::shared_ptr<TTableClient::TImpl> client, std::shared_ptr<TImpl> sessionid) : Client_(client) , SessionImpl_(sessionid) {} TFuture<TStatus> TSession::CreateTable(const TString& path, TTableDescription&& tableDesc, - const TCreateTableSettings& settings) + const TCreateTableSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::CreateTableRequest>(settings); request.set_session_id(SessionImpl_->GetId()); @@ -3688,26 +3688,26 @@ TFuture<TStatus> TSession::CreateTable(const TString& path, TTableDescription&& ConvertCreateTableSettingsToProto(settings, request.mutable_profile()); - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, Client_->CreateTable(std::move(request), settings), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } TFuture<TStatus> TSession::DropTable(const TString& path, const TDropTableSettings& settings) { - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, Client_->DropTable(SessionImpl_->GetId(), path, settings), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } -static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( - const TString& path, const TAlterTableSettings& settings, const TString& sessionId) -{ +static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( + const TString& path, const TAlterTableSettings& settings, const TString& sessionId) +{ auto request = MakeOperationRequest<Ydb::Table::AlterTableRequest>(settings); - request.set_session_id(sessionId); + request.set_session_id(sessionId); request.set_path(path); for (const auto& column : settings.AddColumns_) { @@ -3727,14 +3727,14 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( protoAlter.set_family(alter.Family); } - for (const auto& addIndex : settings.AddIndexes_) { + for (const auto& addIndex : settings.AddIndexes_) { addIndex.SerializeTo(*request.add_add_indexes()); - } - - for (const auto& dropIndex : settings.DropIndexes_) { - request.add_drop_indexes(dropIndex); - } - + } + + for (const auto& dropIndex : settings.DropIndexes_) { + request.add_drop_indexes(dropIndex); + } + if (settings.AlterStorageSettings_) { request.mutable_alter_storage_settings()->CopyFrom(settings.AlterStorageSettings_->GetProto()); } @@ -3791,29 +3791,29 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( } } - return request; -} - -TAsyncStatus TSession::AlterTable(const TString& path, const TAlterTableSettings& settings) { - auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); - - return InjectSessionStatusInterception( + return request; +} + +TAsyncStatus TSession::AlterTable(const TString& path, const TAlterTableSettings& settings) { + auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); + + return InjectSessionStatusInterception( SessionImpl_, Client_->AlterTable(std::move(request), settings), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); -} - -TAsyncOperation TSession::AlterTableLong(const TString& path, const TAlterTableSettings& settings) { - auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); - - return InjectSessionStatusInterception( - SessionImpl_, - Client_->AlterTableLong(std::move(request), settings), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); -} - + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncOperation TSession::AlterTableLong(const TString& path, const TAlterTableSettings& settings) { + auto request = MakeAlterTableProtoRequest(path, settings, SessionImpl_->GetId()); + + return InjectSessionStatusInterception( + SessionImpl_, + Client_->AlterTableLong(std::move(request), settings), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + TAsyncStatus TSession::RenameTables(const TVector<TRenameItem>& renameItems, const TRenameTablesSettings& settings) { auto request = MakeOperationRequest<Ydb::Table::RenameTablesRequest>(settings); request.set_session_id(SessionImpl_->GetId()); @@ -3851,15 +3851,15 @@ TAsyncStatus TSession::CopyTables(const TVector<TCopyItem>& copyItems, const TCo } TFuture<TStatus> TSession::CopyTable(const TString& src, const TString& dst, const TCopyTableSettings& settings) { - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, Client_->CopyTable(SessionImpl_->GetId(), src, dst, settings), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } TAsyncDescribeTableResult TSession::DescribeTable(const TString& path, const TDescribeTableSettings& settings) { - return Client_->DescribeTable(SessionImpl_->GetId(), path, settings); + return Client_->DescribeTable(SessionImpl_->GetId(), path, settings); } TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTxControl& txControl, @@ -3897,9 +3897,9 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTx } TAsyncPrepareQueryResult TSession::PrepareDataQuery(const TString& query, const TPrepareDataQuerySettings& settings) { - auto maybeQuery = SessionImpl_->GetQueryFromCache(query, Client_->Settings_.AllowRequestMigration_); + auto maybeQuery = SessionImpl_->GetQueryFromCache(query, Client_->Settings_.AllowRequestMigration_); if (maybeQuery) { - TStatus status(EStatus::SUCCESS, NYql::TIssues()); + TStatus status(EStatus::SUCCESS, NYql::TIssues()); TDataQuery dataQuery(*this, query, maybeQuery->QueryId, maybeQuery->ParameterTypes); TPrepareQueryResult result(std::move(status), dataQuery, true); return MakeFuture(result); @@ -3907,93 +3907,93 @@ TAsyncPrepareQueryResult TSession::PrepareDataQuery(const TString& query, const Client_->CacheMissCounter.Inc(); - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, - Client_->PrepareDataQuery(*this, query, settings), - true, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + Client_->PrepareDataQuery(*this, query, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } TAsyncStatus TSession::ExecuteSchemeQuery(const TString& query, const TExecSchemeQuerySettings& settings) { - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, Client_->ExecuteSchemeQuery(SessionImpl_->GetId(), query, settings), - true, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } TAsyncBeginTransactionResult TSession::BeginTransaction(const TTxSettings& txSettings, const TBeginTxSettings& settings) { - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, - Client_->BeginTransaction(*this, txSettings, settings), - true, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); + Client_->BeginTransaction(*this, txSettings, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); } TAsyncExplainDataQueryResult TSession::ExplainDataQuery(const TString& query, const TExplainDataQuerySettings& settings) { - return InjectSessionStatusInterception( + return InjectSessionStatusInterception( SessionImpl_, - Client_->ExplainDataQuery(*this, query, settings), - true, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); -} - -TAsyncTablePartIterator TSession::ReadTable(const TString& path, - const TReadTableSettings& settings) -{ - auto promise = NThreading::NewPromise<TTablePartIterator>(); - auto readTableIteratorBuilder = [promise](NThreading::TFuture<std::pair<TPlainStatus, TTableClient::TImpl::TReadTableStreamProcessorPtr>> future) mutable { - Y_ASSERT(future.HasValue()); - auto pair = future.ExtractValue(); - promise.SetValue(TTablePartIterator( - pair.second ? std::make_shared<TTablePartIterator::TReaderImpl>( - pair.second, pair.first.Endpoint) : nullptr, std::move(pair.first)) - ); - }; + Client_->ExplainDataQuery(*this, query, settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + +TAsyncTablePartIterator TSession::ReadTable(const TString& path, + const TReadTableSettings& settings) +{ + auto promise = NThreading::NewPromise<TTablePartIterator>(); + auto readTableIteratorBuilder = [promise](NThreading::TFuture<std::pair<TPlainStatus, TTableClient::TImpl::TReadTableStreamProcessorPtr>> future) mutable { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TTablePartIterator( + pair.second ? std::make_shared<TTablePartIterator::TReaderImpl>( + pair.second, pair.first.Endpoint) : nullptr, std::move(pair.first)) + ); + }; Client_->ReadTable(SessionImpl_->GetId(), path, settings).Subscribe(readTableIteratorBuilder); - return InjectSessionStatusInterception( - SessionImpl_, - promise.GetFuture(), - false, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); -} - + return InjectSessionStatusInterception( + SessionImpl_, + promise.GetFuture(), + false, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + void TSession::InvalidateQueryCache() { SessionImpl_->InvalidateQueryCache(); } TAsyncStatus TSession::Close(const TCloseSessionSettings& settings) { - return Client_->Close(SessionImpl_.get(), settings); + return Client_->Close(SessionImpl_.get(), settings); } -TAsyncKeepAliveResult TSession::KeepAlive(const TKeepAliveSettings &settings) { - return InjectSessionStatusInterception( +TAsyncKeepAliveResult TSession::KeepAlive(const TKeepAliveSettings &settings) { + return InjectSessionStatusInterception( SessionImpl_, - Client_->KeepAlive(SessionImpl_.get(), settings), - true, - GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); -} - + Client_->KeepAlive(SessionImpl_.get(), settings), + true, + GetMinTimeToTouch(Client_->Settings_.SessionPoolSettings_)); +} + TTableBuilder TSession::GetTableBuilder() { return TTableBuilder(); } TParamsBuilder TSession::GetParamsBuilder() { - return TParamsBuilder(); + return TParamsBuilder(); } TTypeBuilder TSession::GetTypeBuilder() { - return TTypeBuilder(); + return TTypeBuilder(); } const TString& TSession::GetId() const { return SessionImpl_->GetId(); -} - +} + //////////////////////////////////////////////////////////////////////////////// TTxControl::TTxControl(const TTransaction& tx) @@ -4009,23 +4009,23 @@ TTransaction::TTransaction(const TSession& session, const TString& txId) , TxId_(txId) {} TAsyncCommitTransactionResult TTransaction::Commit(const TCommitTxSettings& settings) { - return Session_.Client_->CommitTransaction(Session_, *this, settings); + return Session_.Client_->CommitTransaction(Session_, *this, settings); } TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) { - return Session_.Client_->RollbackTransaction(Session_, *this, settings); + return Session_.Client_->RollbackTransaction(Session_, *this, settings); } //////////////////////////////////////////////////////////////////////////////// TDataQuery::TDataQuery(const TSession& session, const TString& text, const TString& id) - : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, - session.Client_->Settings_.AllowRequestMigration_)) {} + : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, + session.Client_->Settings_.AllowRequestMigration_)) {} TDataQuery::TDataQuery(const TSession& session, const TString& text, const TString& id, const ::google::protobuf::Map<TString, Ydb::Type>& types) - : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, - session.Client_->Settings_.AllowRequestMigration_, types)) {} + : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, + session.Client_->Settings_.AllowRequestMigration_, types)) {} const TString& TDataQuery::GetId() const { return Impl_->GetId(); @@ -4036,7 +4036,7 @@ const TMaybe<TString>& TDataQuery::GetText() const { } TParamsBuilder TDataQuery::GetParamsBuilder() const { - return TParamsBuilder(Impl_->ParameterTypes_); + return TParamsBuilder(Impl_->ParameterTypes_); } TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, @@ -4049,38 +4049,38 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, TParams&& const TExecDataQuerySettings& settings) { auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); - return Impl_->Session_.Client_->ExecuteDataQuery( - Impl_->Session_, - *this, - txControl, - paramsPtr, - settings, - false); -} - -TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TParams& params, - const TExecDataQuerySettings& settings) -{ - if (params.Empty()) { - return Impl_->Session_.Client_->ExecuteDataQuery( - Impl_->Session_, - *this, - txControl, - nullptr, - settings, - false); - } else { - using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; - return Impl_->Session_.Client_->ExecuteDataQuery<TProtoParamsType&>( - Impl_->Session_, - *this, - txControl, + return Impl_->Session_.Client_->ExecuteDataQuery( + Impl_->Session_, + *this, + txControl, + paramsPtr, + settings, + false); +} + +TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TParams& params, + const TExecDataQuerySettings& settings) +{ + if (params.Empty()) { + return Impl_->Session_.Client_->ExecuteDataQuery( + Impl_->Session_, + *this, + txControl, + nullptr, + settings, + false); + } else { + using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; + return Impl_->Session_.Client_->ExecuteDataQuery<TProtoParamsType&>( + Impl_->Session_, + *this, + txControl, params.GetProtoMap(), - settings, - false); - } -} - + settings, + false); + } +} + //////////////////////////////////////////////////////////////////////////////// TCreateSessionResult::TCreateSessionResult(TStatus&& status, TSession&& session) @@ -4094,16 +4094,16 @@ TSession TCreateSessionResult::GetSession() const { //////////////////////////////////////////////////////////////////////////////// -TKeepAliveResult::TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus) - : TStatus(std::move(status)) - , SessionStatus(sessionStatus) {} - -ESessionStatus TKeepAliveResult::GetSessionStatus() const { - return SessionStatus; -} - -//////////////////////////////////////////////////////////////////////////////// - +TKeepAliveResult::TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus) + : TStatus(std::move(status)) + , SessionStatus(sessionStatus) {} + +ESessionStatus TKeepAliveResult::GetSessionStatus() const { + return SessionStatus; +} + +//////////////////////////////////////////////////////////////////////////////// + TPrepareQueryResult::TPrepareQueryResult(TStatus&& status, const TDataQuery& query, bool fromCache) : TStatus(std::move(status)) , PreparedQuery_(query) @@ -4176,7 +4176,7 @@ TResultSetParser TDataQueryResult::GetResultSetParser(size_t resultIndex) const return TResultSetParser(GetResultSet(resultIndex)); } -TMaybe<TTransaction> TDataQueryResult::GetTransaction() const { +TMaybe<TTransaction> TDataQueryResult::GetTransaction() const { return Transaction_; } @@ -4224,27 +4224,27 @@ const TMaybe<TQueryStats>& TCommitTransactionResult::GetStats() const { //////////////////////////////////////////////////////////////////////////////// std::function<void(TSession::TImpl*)> TSession::TImpl::GetSmartDeleter(std::shared_ptr<TTableClient::TImpl> client) { - return [client](TSession::TImpl* sessionImpl) { - switch (sessionImpl->GetState()) { - case TSession::TImpl::S_STANDALONE: - case TSession::TImpl::S_BROKEN: + return [client](TSession::TImpl* sessionImpl) { + switch (sessionImpl->GetState()) { + case TSession::TImpl::S_STANDALONE: + case TSession::TImpl::S_BROKEN: case TSession::TImpl::S_CLOSING: - client->DeleteSession(sessionImpl); - break; - case TSession::TImpl::S_IDLE: - case TSession::TImpl::S_ACTIVE: - case TSession::TImpl::S_DISCONNECTED: { - if (!client->ReturnSession(sessionImpl)) { - client->DeleteSession(sessionImpl); - } + client->DeleteSession(sessionImpl); + break; + case TSession::TImpl::S_IDLE: + case TSession::TImpl::S_ACTIVE: + case TSession::TImpl::S_DISCONNECTED: { + if (!client->ReturnSession(sessionImpl)) { + client->DeleteSession(sessionImpl); + } break; - } - default: - break; - } - }; -} - + } + default: + break; + } + }; +} + //////////////////////////////////////////////////////////////////////////////// TCopyItem::TCopyItem(const TString& source, const TString& destination) @@ -4270,8 +4270,8 @@ bool TCopyItem::OmitIndexes() const { return OmitIndexes_; } -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + TRenameItem::TRenameItem(const TString& source, const TString& destination) : Source_(source) , Destination_(destination) @@ -4299,12 +4299,12 @@ bool TRenameItem::ReplaceDestination() const { TIndexDescription::TIndexDescription(const TString& name, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) - : IndexName_(name) + : IndexName_(name) , IndexType_(type) - , IndexColumns_(indexColumns) + , IndexColumns_(indexColumns) , DataColumns_(dataColumns) -{} - +{} + TIndexDescription::TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns) {} @@ -4317,22 +4317,22 @@ TIndexDescription::TIndexDescription(const Ydb::Table::TableIndexDescription& ta : TIndexDescription(FromProto(tableIndexDesc)) {} -const TString& TIndexDescription::GetIndexName() const { - return IndexName_; -} - +const TString& TIndexDescription::GetIndexName() const { + return IndexName_; +} + EIndexType TIndexDescription::GetIndexType() const { return IndexType_; } -const TVector<TString>& TIndexDescription::GetIndexColumns() const { - return IndexColumns_; -} - -const TVector<TString>& TIndexDescription::GetDataColumns() const { - return DataColumns_; -} - +const TVector<TString>& TIndexDescription::GetIndexColumns() const { + return IndexColumns_; +} + +const TVector<TString>& TIndexDescription::GetDataColumns() const { + return DataColumns_; +} + ui64 TIndexDescription::GetSizeBytes() const { return SizeBytes; } @@ -4344,8 +4344,8 @@ TIndexDescription TIndexDescription::FromProto(const TProto& proto) { TVector<TString> dataColumns; indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end()); - dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end()); - + dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end()); + switch (proto.type_case()) { case TProto::kGlobalIndex: type = EIndexType::GlobalSync; @@ -4372,14 +4372,14 @@ void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { proto.add_index_columns(indexCol); } - *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; - + *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; + switch (IndexType_) { case EIndexType::GlobalSync: - *proto.mutable_global_index() = Ydb::Table::GlobalIndex(); + *proto.mutable_global_index() = Ydb::Table::GlobalIndex(); break; case EIndexType::GlobalAsync: - *proto.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + *proto.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); break; case EIndexType::Unknown: break; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index e348f40704..42d21d2fec 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -22,9 +22,9 @@ class DescribeTableResult; class PartitioningSettings; class DateTypeColumnModeSettings; class TtlSettings; -class TableIndex; -class TableIndexDescription; -class ValueSinceUnixEpochModeSettings; +class TableIndex; +class TableIndexDescription; +class ValueSinceUnixEpochModeSettings; } } @@ -37,52 +37,52 @@ struct TPermissions; namespace NTable { -//////////////////////////////////////////////////////////////////////////////// - -class TKeyBound { -public: - static TKeyBound Inclusive(const TValue& value) { - return TKeyBound(value, true); - } - - static TKeyBound Exclusive(const TValue& value) { - return TKeyBound(value, false); - } - - bool IsInclusive() const { - return Equal_; - } - - const TValue& GetValue() const { - return Value_; - } -private: - TKeyBound(const TValue& value, bool equal = false) - : Value_(value) - , Equal_(equal) - {} - TValue Value_; - bool Equal_; -}; - -class TKeyRange { -public: +//////////////////////////////////////////////////////////////////////////////// + +class TKeyBound { +public: + static TKeyBound Inclusive(const TValue& value) { + return TKeyBound(value, true); + } + + static TKeyBound Exclusive(const TValue& value) { + return TKeyBound(value, false); + } + + bool IsInclusive() const { + return Equal_; + } + + const TValue& GetValue() const { + return Value_; + } +private: + TKeyBound(const TValue& value, bool equal = false) + : Value_(value) + , Equal_(equal) + {} + TValue Value_; + bool Equal_; +}; + +class TKeyRange { +public: TKeyRange(const TMaybe<TKeyBound>& from, const TMaybe<TKeyBound>& to) : From_(from) , To_(to) {} - - const TMaybe<TKeyBound>& From() const { - return From_; - } - - const TMaybe<TKeyBound>& To() const { - return To_; - } -private: - TMaybe<TKeyBound> From_; - TMaybe<TKeyBound> To_; -}; - + + const TMaybe<TKeyBound>& From() const { + return From_; + } + + const TMaybe<TKeyBound>& To() const { + return To_; + } +private: + TMaybe<TKeyBound> From_; + TMaybe<TKeyBound> To_; +}; + struct TTableColumn { TString Name; TType Type; @@ -130,31 +130,31 @@ struct TAlterTableColumn { { } }; -//////////////////////////////////////////////////////////////////////////////// - -//! Represents index description -class TIndexDescription { +//////////////////////////////////////////////////////////////////////////////// + +//! Represents index description +class TIndexDescription { friend class NYdb::TProtoAccessor; -public: +public: TIndexDescription( const TString& name, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns = TVector<TString>()); - TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns = TVector<TString>()); - - const TString& GetIndexName() const; + TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns = TVector<TString>()); + + const TString& GetIndexName() const; EIndexType GetIndexType() const; - const TVector<TString>& GetIndexColumns() const; - const TVector<TString>& GetDataColumns() const; + const TVector<TString>& GetIndexColumns() const; + const TVector<TString>& GetDataColumns() const; ui64 GetSizeBytes() const; void SerializeTo(Ydb::Table::TableIndex& proto) const; TString ToString() const; void Out(IOutputStream& o) const; -private: +private: explicit TIndexDescription(const Ydb::Table::TableIndex& tableIndex); explicit TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc); @@ -162,40 +162,40 @@ private: static TIndexDescription FromProto(const TProto& proto); private: - TString IndexName_; + TString IndexName_; EIndexType IndexType_; - TVector<TString> IndexColumns_; - TVector<TString> DataColumns_; + TVector<TString> IndexColumns_; + TVector<TString> DataColumns_; ui64 SizeBytes = 0; -}; - +}; + bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs); bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs); -class TBuildIndexOperation : public TOperation { -public: - using TOperation::TOperation; - TBuildIndexOperation(TStatus&& status, Ydb::Operations::Operation&& operation); - - struct TMetadata { - EBuildIndexState State; - float Progress; - TString Path; - TMaybe<TIndexDescription> Desctiption; - }; - - const TMetadata& Metadata() const; -private: - TMetadata Metadata_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct TPartitionStats { - ui64 Rows = 0; - ui64 Size = 0; -}; - +class TBuildIndexOperation : public TOperation { +public: + using TOperation::TOperation; + TBuildIndexOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + + struct TMetadata { + EBuildIndexState State; + float Progress; + TString Path; + TMaybe<TIndexDescription> Desctiption; + }; + + const TMetadata& Metadata() const; +private: + TMetadata Metadata_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TPartitionStats { + ui64 Rows = 0; + ui64 Size = 0; +}; + class TDateTypeColumnModeSettings { public: explicit TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter); @@ -396,32 +396,32 @@ public: // DEPRECATED: use GetTableColumns() TVector<TColumn> GetColumns() const; TVector<TTableColumn> GetTableColumns() const; - TVector<TIndexDescription> GetIndexDescriptions() const; + TVector<TIndexDescription> GetIndexDescriptions() const; TMaybe<TTtlSettings> GetTtlSettings() const; const TString& GetOwner() const; const TVector<NScheme::TPermissions>& GetPermissions() const; const TVector<NScheme::TPermissions>& GetEffectivePermissions() const; - const TVector<TKeyRange>& GetKeyRanges() const; - - // Folow options related to table statistics - // flag WithTableStatistics must be set - - // Number of partition - ui64 GetPartitionsCount() const; - // Approximate number of rows - ui64 GetTableRows() const; - // Approximate size of table (bytes) - ui64 GetTableSize() const; - // Timestamp of last modification - TInstant GetModificationTime() const; - // Timestamp of table creation - TInstant GetCreationTime() const; - - // Returns partition statistics for table - // flag WithTableStatistics and WithPartitionStatistics must be set - const TVector<TPartitionStats>& GetPartitionStats() const; - + const TVector<TKeyRange>& GetKeyRanges() const; + + // Folow options related to table statistics + // flag WithTableStatistics must be set + + // Number of partition + ui64 GetPartitionsCount() const; + // Approximate number of rows + ui64 GetTableRows() const; + // Approximate size of table (bytes) + ui64 GetTableSize() const; + // Timestamp of last modification + TInstant GetModificationTime() const; + // Timestamp of table creation + TInstant GetCreationTime() const; + + // Returns partition statistics for table + // flag WithTableStatistics and WithPartitionStatistics must be set + const TVector<TPartitionStats>& GetPartitionStats() const; + // Returns storage settings of the table const TStorageSettings& GetStorageSettings() const; @@ -460,8 +460,8 @@ private: void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); // default - void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); void SetTtlSettings(TTtlSettings&& settings); void SetTtlSettings(const TTtlSettings& settings); @@ -662,7 +662,7 @@ public: TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TString& indexColumn); - + // async TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); @@ -766,14 +766,14 @@ private: class TCreateSessionResult; class TDataQueryResult; -class TTablePartIterator; +class TTablePartIterator; class TPrepareQueryResult; class TExplainQueryResult; class TDescribeTableResult; class TBeginTransactionResult; class TCommitTransactionResult; -class TKeepAliveResult; -class TSessionPoolImpl; +class TKeepAliveResult; +class TSessionPoolImpl; class TBulkUpsertResult; class TScanQueryPartIterator; @@ -784,8 +784,8 @@ using TAsyncExplainDataQueryResult = NThreading::TFuture<TExplainQueryResult>; using TAsyncDescribeTableResult = NThreading::TFuture<TDescribeTableResult>; using TAsyncBeginTransactionResult = NThreading::TFuture<TBeginTransactionResult>; using TAsyncCommitTransactionResult = NThreading::TFuture<TCommitTransactionResult>; -using TAsyncTablePartIterator = NThreading::TFuture<TTablePartIterator>; -using TAsyncKeepAliveResult = NThreading::TFuture<TKeepAliveResult>; +using TAsyncTablePartIterator = NThreading::TFuture<TTablePartIterator>; +using TAsyncKeepAliveResult = NThreading::TFuture<TKeepAliveResult>; using TAsyncBulkUpsertResult = NThreading::TFuture<TBulkUpsertResult>; using TAsyncScanQueryPartIterator = NThreading::TFuture<TScanQueryPartIterator>; @@ -806,7 +806,7 @@ struct TRetryOperationSettings { FLUENT_SETTING_DEFAULT(ui32, MaxRetries, 10); FLUENT_SETTING_DEFAULT(bool, RetryNotFound, true); - FLUENT_SETTING_DEFAULT(TDuration, GetSessionClientTimeout, TDuration::Seconds(5)); + FLUENT_SETTING_DEFAULT(TDuration, GetSessionClientTimeout, TDuration::Seconds(5)); FLUENT_SETTING_DEFAULT(TBackoffSettings, FastBackoffSettings, DefaultFastBackoffSettings()); FLUENT_SETTING_DEFAULT(TBackoffSettings, SlowBackoffSettings, DefaultSlowBackoffSettings()); FLUENT_SETTING_FLAG(Idempotent); @@ -826,28 +826,28 @@ struct TRetryOperationSettings { } }; -struct TSessionPoolSettings { - using TSelf = TSessionPoolSettings; - - // Max number of sessions client can get from session pool - FLUENT_SETTING_DEFAULT(ui32, MaxActiveSessions, 50); - - // Max number of attempt to create session inside session pool - // to handle OVERLOADED error - FLUENT_SETTING_DEFAULT(ui32, RetryLimit, 5); - - // Max time session to be in idle state in session pool before - // keep alive start to touch it - FLUENT_SETTING_DEFAULT(TDuration, KeepAliveIdleThreshold, TDuration::Minutes(5)); - - // Max time session to be in idle state before closing - FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); - - // Min number of session in session pool. - // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. - FLUENT_SETTING_DEFAULT(ui32, MinPoolSize, 10); -}; - +struct TSessionPoolSettings { + using TSelf = TSessionPoolSettings; + + // Max number of sessions client can get from session pool + FLUENT_SETTING_DEFAULT(ui32, MaxActiveSessions, 50); + + // Max number of attempt to create session inside session pool + // to handle OVERLOADED error + FLUENT_SETTING_DEFAULT(ui32, RetryLimit, 5); + + // Max time session to be in idle state in session pool before + // keep alive start to touch it + FLUENT_SETTING_DEFAULT(TDuration, KeepAliveIdleThreshold, TDuration::Minutes(5)); + + // Max time session to be in idle state before closing + FLUENT_SETTING_DEFAULT(TDuration, CloseIdleThreshold, TDuration::Minutes(1)); + + // Min number of session in session pool. + // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. + FLUENT_SETTING_DEFAULT(ui32, MinPoolSize, 10); +}; + struct TClientSettings : public TCommonClientSettingsBase<TClientSettings> { using TSelf = TClientSettings; @@ -861,28 +861,28 @@ struct TClientSettings : public TCommonClientSettingsBase<TClientSettings> { FLUENT_SETTING_DEFAULT(bool, UseQueryCache, true); FLUENT_SETTING_DEFAULT(ui32, QueryCacheSize, 1000); FLUENT_SETTING_DEFAULT(bool, KeepDataQueryText, true); - - // Min allowed session variation coefficient (%) to start session balancing. - // Variation coefficient is a ratio of the standard deviation sigma to the mean - // Example: - // - 3 hosts with [90, 100, 110] sessions per host. Cv will be 10% - // - add new host ([90, 100, 110, 0] sessions per host). Cv will be 77% - // Balancing is will be performed if calculated cv greater than MinSessionCV - // Zero - disable this feature - FLUENT_SETTING_DEFAULT(ui32, MinSessionCV, 20); - - // Allow migrate requests between session during session balancing - FLUENT_SETTING_DEFAULT(bool, AllowRequestMigration, true); - - // Max number of seconds to keep session in SettlerSessionPool - // Settler pool is a session pool is used to keep - // sessions in case of errors when we dont known actual server - // status (e.g. CLIENT_RESOURCE_EXHAUSTED). In this case - // sessions from this pool we be keep alived with progressive timeout - // to try to return it to main session pool - FLUENT_SETTING_DEFAULT(ui32, SettlerSessionPoolTTL, 100); - // Settings of session pool - FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); + + // Min allowed session variation coefficient (%) to start session balancing. + // Variation coefficient is a ratio of the standard deviation sigma to the mean + // Example: + // - 3 hosts with [90, 100, 110] sessions per host. Cv will be 10% + // - add new host ([90, 100, 110, 0] sessions per host). Cv will be 77% + // Balancing is will be performed if calculated cv greater than MinSessionCV + // Zero - disable this feature + FLUENT_SETTING_DEFAULT(ui32, MinSessionCV, 20); + + // Allow migrate requests between session during session balancing + FLUENT_SETTING_DEFAULT(bool, AllowRequestMigration, true); + + // Max number of seconds to keep session in SettlerSessionPool + // Settler pool is a session pool is used to keep + // sessions in case of errors when we dont known actual server + // status (e.g. CLIENT_RESOURCE_EXHAUSTED). In this case + // sessions from this pool we be keep alived with progressive timeout + // to try to return it to main session pool + FLUENT_SETTING_DEFAULT(ui32, SettlerSessionPoolTTL, 100); + // Settings of session pool + FLUENT_SETTING(TSessionPoolSettings, SessionPoolSettings); }; struct TBulkUpsertSettings : public TOperationRequestSettings<TBulkUpsertSettings> { @@ -910,7 +910,7 @@ enum class EDataFormat { class TTableClient { friend class TSession; friend class TTransaction; - friend class TSessionPoolImpl; + friend class TSessionPoolImpl; friend class TRetryOperationContext; public: @@ -922,16 +922,16 @@ public: public: TTableClient(const TDriver& driver, const TClientSettings& settings = TClientSettings()); - //! Creates new session + //! Creates new session TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); - //! Returns session from session pool, - //! if all sessions are occupied will be generated session with CLIENT_RESOURCE_EXHAUSTED status. - TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); - - //! Returns number of active sessions given via session pool - i64 GetActiveSessionCount() const; - + //! Returns session from session pool, + //! if all sessions are occupied will be generated session with CLIENT_RESOURCE_EXHAUSTED status. + TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + + //! Returns number of active sessions given via session pool + i64 GetActiveSessionCount() const; + //! Returns the maximum number of sessions in session pool i64 GetActiveSessionsLimit() const; @@ -973,11 +973,11 @@ public: TStatus RetryOperationSync(const TOperationWithoutSessionSyncFunc& operation, const TRetryOperationSettings& settings = TRetryOperationSettings()); - //! Stop all client internal routines, drain session pools - //! Sessions returned to the session pool after this call will be closed - //! Using the client after call this method causes UB - NThreading::TFuture<void> Stop(); - + //! Stop all client internal routines, drain session pools + //! Sessions returned to the session pool after this call will be closed + //! Using the client after call this method causes UB + NThreading::TFuture<void> Stop(); + //! Non-transactional fast bulk write. //! Interanlly it uses an implicit session and thus doesn't need a session to be passed. //! "rows" parameter must be a list of structs where each stuct represents one row. @@ -1094,12 +1094,12 @@ private: TTxSettings BeginTx_; }; -enum class EAutoPartitioningPolicy { - Disabled = 1, - AutoSplit = 2, - AutoSplitMerge = 3 -}; - +enum class EAutoPartitioningPolicy { + Disabled = 1, + AutoSplit = 2, + AutoSplitMerge = 3 +}; + //////////////////////////////////////////////////////////////////////////////// struct TColumnFamilyPolicy { @@ -1116,40 +1116,40 @@ struct TColumnFamilyPolicy { FLUENT_SETTING_OPTIONAL(bool, Compressed); }; -struct TStoragePolicy { - using TSelf = TStoragePolicy; +struct TStoragePolicy { + using TSelf = TStoragePolicy; - FLUENT_SETTING_OPTIONAL(TString, PresetName); - - FLUENT_SETTING_OPTIONAL(TString, SysLog); - - FLUENT_SETTING_OPTIONAL(TString, Log); - - FLUENT_SETTING_OPTIONAL(TString, Data); - - FLUENT_SETTING_OPTIONAL(TString, External); + FLUENT_SETTING_OPTIONAL(TString, PresetName); + + FLUENT_SETTING_OPTIONAL(TString, SysLog); + + FLUENT_SETTING_OPTIONAL(TString, Log); + + FLUENT_SETTING_OPTIONAL(TString, Data); + + FLUENT_SETTING_OPTIONAL(TString, External); FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies); -}; - +}; + struct TExplicitPartitions { using TSelf = TExplicitPartitions; FLUENT_SETTING_VECTOR(TValue, SplitPoints); }; -struct TPartitioningPolicy { - using TSelf = TPartitioningPolicy; - - FLUENT_SETTING_OPTIONAL(TString, PresetName); - - FLUENT_SETTING_OPTIONAL(EAutoPartitioningPolicy, AutoPartitioning); - - FLUENT_SETTING_OPTIONAL(ui64, UniformPartitions); +struct TPartitioningPolicy { + using TSelf = TPartitioningPolicy; + + FLUENT_SETTING_OPTIONAL(TString, PresetName); + + FLUENT_SETTING_OPTIONAL(EAutoPartitioningPolicy, AutoPartitioning); + + FLUENT_SETTING_OPTIONAL(ui64, UniformPartitions); FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions); -}; - +}; + struct TReplicationPolicy { using TSelf = TReplicationPolicy; @@ -1162,27 +1162,27 @@ struct TReplicationPolicy { FLUENT_SETTING_OPTIONAL(bool, AllowPromotion); }; -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + struct TCreateTableSettings : public TOperationRequestSettings<TCreateTableSettings> { - using TSelf = TCreateTableSettings; - - FLUENT_SETTING_OPTIONAL(TString, PresetName); - - FLUENT_SETTING_OPTIONAL(TString, ExecutionPolicy); - - FLUENT_SETTING_OPTIONAL(TString, CompactionPolicy); - - FLUENT_SETTING_OPTIONAL(TPartitioningPolicy, PartitioningPolicy); - - FLUENT_SETTING_OPTIONAL(TStoragePolicy, StoragePolicy); + using TSelf = TCreateTableSettings; + + FLUENT_SETTING_OPTIONAL(TString, PresetName); + + FLUENT_SETTING_OPTIONAL(TString, ExecutionPolicy); + + FLUENT_SETTING_OPTIONAL(TString, CompactionPolicy); + + FLUENT_SETTING_OPTIONAL(TPartitioningPolicy, PartitioningPolicy); + + FLUENT_SETTING_OPTIONAL(TStoragePolicy, StoragePolicy); FLUENT_SETTING_OPTIONAL(TReplicationPolicy, ReplicationPolicy); -}; - +}; + //////////////////////////////////////////////////////////////////////////////// -struct TDropTableSettings : public TOperationRequestSettings<TDropTableSettings> {}; +struct TDropTableSettings : public TOperationRequestSettings<TDropTableSettings> {}; //////////////////////////////////////////////////////////////////////////////// @@ -1343,10 +1343,10 @@ struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSetting FLUENT_SETTING_VECTOR(TAlterTableColumn, AlterColumns); - FLUENT_SETTING_VECTOR(TString, DropIndexes); - - FLUENT_SETTING_VECTOR(TIndexDescription, AddIndexes); - + FLUENT_SETTING_VECTOR(TString, DropIndexes); + + FLUENT_SETTING_VECTOR(TIndexDescription, AddIndexes); + TSelf& AlterColumnFamily(TString name, TString family) { AlterColumns_.emplace_back(std::move(name), std::move(family)); return *this; @@ -1433,14 +1433,14 @@ struct TCopyTablesSettings : public TOperationRequestSettings<TCopyTablesSetting struct TRenameTablesSettings : public TOperationRequestSettings<TRenameTablesSettings> {}; struct TDescribeTableSettings : public TOperationRequestSettings<TDescribeTableSettings> { - FLUENT_SETTING_DEFAULT(bool, WithKeyShardBoundary, false); - FLUENT_SETTING_DEFAULT(bool, WithTableStatistics, false); - FLUENT_SETTING_DEFAULT(bool, WithPartitionStatistics, false); -}; + FLUENT_SETTING_DEFAULT(bool, WithKeyShardBoundary, false); + FLUENT_SETTING_DEFAULT(bool, WithTableStatistics, false); + FLUENT_SETTING_DEFAULT(bool, WithPartitionStatistics, false); +}; struct TExplainDataQuerySettings : public TOperationRequestSettings<TExplainDataQuerySettings> {}; -struct TPrepareDataQuerySettings : public TOperationRequestSettings<TPrepareDataQuerySettings> {}; +struct TPrepareDataQuerySettings : public TOperationRequestSettings<TPrepareDataQuerySettings> {}; struct TExecDataQuerySettings : public TOperationRequestSettings<TExecDataQuerySettings> { FLUENT_SETTING_OPTIONAL(bool, KeepInQueryCache); @@ -1461,45 +1461,45 @@ struct TRollbackTxSettings : public TOperationRequestSettings<TRollbackTxSetting struct TCloseSessionSettings : public TOperationRequestSettings<TCloseSessionSettings> {}; struct TKeepAliveSettings : public TOperationRequestSettings<TKeepAliveSettings> {}; - -struct TReadTableSettings : public TRequestSettings<TReadTableSettings> { - - using TSelf = TReadTableSettings; - - FLUENT_SETTING_OPTIONAL(TKeyBound, From); - - FLUENT_SETTING_OPTIONAL(TKeyBound, To); - - FLUENT_SETTING_VECTOR(TString, Columns); - - FLUENT_SETTING_FLAG(Ordered); - - FLUENT_SETTING_OPTIONAL(ui64, RowLimit); + +struct TReadTableSettings : public TRequestSettings<TReadTableSettings> { + + using TSelf = TReadTableSettings; + + FLUENT_SETTING_OPTIONAL(TKeyBound, From); + + FLUENT_SETTING_OPTIONAL(TKeyBound, To); + + FLUENT_SETTING_VECTOR(TString, Columns); + + FLUENT_SETTING_FLAG(Ordered); + + FLUENT_SETTING_OPTIONAL(ui64, RowLimit); FLUENT_SETTING_OPTIONAL(bool, UseSnapshot); -}; - +}; + //! Represents all session operations //! Session is transparent logic representation of connection class TSession { friend class TTableClient; friend class TDataQuery; friend class TTransaction; - friend class TSessionPoolImpl; + friend class TSessionPoolImpl; public: //! The following methods perform corresponding calls. //! Results are NThreading::TFuture<T> where T is corresponding result. TAsyncStatus CreateTable(const TString& path, TTableDescription&& tableDesc, - const TCreateTableSettings& settings = TCreateTableSettings()); + const TCreateTableSettings& settings = TCreateTableSettings()); TAsyncStatus DropTable(const TString& path, const TDropTableSettings& settings = TDropTableSettings()); TAsyncStatus AlterTable(const TString& path, const TAlterTableSettings& settings = TAlterTableSettings()); - // Same as AlterTable but may return operation in case of long running - TAsyncOperation AlterTableLong(const TString& path, const TAlterTableSettings& settings = TAlterTableSettings()); - + // Same as AlterTable but may return operation in case of long running + TAsyncOperation AlterTableLong(const TString& path, const TAlterTableSettings& settings = TAlterTableSettings()); + TAsyncStatus CopyTable(const TString& src, const TString& dst, const TCopyTableSettings& settings = TCopyTableSettings()); @@ -1533,13 +1533,13 @@ public: TAsyncStatus ExecuteSchemeQuery(const TString& query, const TExecSchemeQuerySettings& settings = TExecSchemeQuerySettings()); - TAsyncTablePartIterator ReadTable(const TString& path, - const TReadTableSettings& settings = TReadTableSettings()); - + TAsyncTablePartIterator ReadTable(const TString& path, + const TReadTableSettings& settings = TReadTableSettings()); + TAsyncStatus Close(const TCloseSessionSettings& settings = TCloseSessionSettings()); - TAsyncKeepAliveResult KeepAlive(const TKeepAliveSettings& settings = TKeepAliveSettings()); - + TAsyncKeepAliveResult KeepAlive(const TKeepAliveSettings& settings = TKeepAliveSettings()); + void InvalidateQueryCache(); //! Returns new table builder @@ -1548,12 +1548,12 @@ public: TParamsBuilder GetParamsBuilder(); //! Returns new type builder TTypeBuilder GetTypeBuilder(); - //! Returns session id + //! Returns session id const TString& GetId() const; - class TImpl; + class TImpl; private: - TSession(std::shared_ptr<TTableClient::TImpl> client, const TString& sessionId, const TString& endpointId); + TSession(std::shared_ptr<TTableClient::TImpl> client, const TString& sessionId, const TString& endpointId); TSession(std::shared_ptr<TTableClient::TImpl> client, std::shared_ptr<TSession::TImpl> SessionImpl_); std::shared_ptr<TTableClient::TImpl> Client_; @@ -1657,8 +1657,8 @@ public: const TExecDataQuerySettings& settings = TExecDataQuerySettings()); TAsyncDataQueryResult Execute(const TTxControl& txControl, TParams&& params, - const TExecDataQuerySettings& settings = TExecDataQuerySettings()); - + const TExecDataQuerySettings& settings = TExecDataQuerySettings()); + private: TDataQuery(const TSession& session, const TString& text, const TString& id); TDataQuery(const TSession& session, const TString& text, const TString& id, @@ -1708,7 +1708,7 @@ public: private: TTableDescription TableDescription_; - + }; class TDataQueryResult : public TStatus { @@ -1721,7 +1721,7 @@ public: TResultSetParser GetResultSetParser(size_t resultIndex) const; - TMaybe<TTransaction> GetTransaction() const; + TMaybe<TTransaction> GetTransaction() const; TMaybe<TDataQuery> GetQuery() const; bool IsQueryFromCache() const; @@ -1738,38 +1738,38 @@ private: TMaybe<TQueryStats> QueryStats_; }; -template<typename TPart> +template<typename TPart> class TSimpleStreamPart : public TStreamPartStatus { -public: - const TPart& GetPart() const { return Part_; } - - TPart ExtractPart() { return std::move(Part_); } - +public: + const TPart& GetPart() const { return Part_; } + + TPart ExtractPart() { return std::move(Part_); } + TSimpleStreamPart(TPart&& part, TStatus&& status) - : TStreamPartStatus(std::move(status)) - , Part_(std::move(part)) - {} - -private: - TPart Part_; -}; - -template<typename TPart> + : TStreamPartStatus(std::move(status)) + , Part_(std::move(part)) + {} + +private: + TPart Part_; +}; + +template<typename TPart> using TAsyncSimpleStreamPart = NThreading::TFuture<TSimpleStreamPart<TPart>>; - -class TTablePartIterator : public TStatus { - friend class TSession; -public: + +class TTablePartIterator : public TStatus { + friend class TSession; +public: TAsyncSimpleStreamPart<TResultSet> ReadNext(); - class TReaderImpl; -private: - TTablePartIterator( - std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status - ); - std::shared_ptr<TReaderImpl> ReaderImpl_; -}; - + class TReaderImpl; +private: + TTablePartIterator( + std::shared_ptr<TReaderImpl> impl, + TPlainStatus&& status + ); + std::shared_ptr<TReaderImpl> ReaderImpl_; +}; + using TReadTableResultPart = TSimpleStreamPart<TResultSet>; class TScanQueryPart : public TStreamPartStatus { @@ -1812,7 +1812,7 @@ public: private: TScanQueryPartIterator( std::shared_ptr<TReaderImpl> impl, - TPlainStatus&& status + TPlainStatus&& status ); std::shared_ptr<TReaderImpl> ReaderImpl_; }; @@ -1838,7 +1838,7 @@ private: }; class TCreateSessionResult: public TStatus { - friend class TSession::TImpl; + friend class TSession::TImpl; public: TCreateSessionResult(TStatus&& status, TSession&& session); TSession GetSession() const; @@ -1847,20 +1847,20 @@ private: TSession Session_; }; -enum class ESessionStatus { - Unspecified = 0, - Ready = 1, - Busy = 2 -}; - -class TKeepAliveResult : public TStatus { -public: - TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus); - ESessionStatus GetSessionStatus() const; -private: - ESessionStatus SessionStatus; -}; - +enum class ESessionStatus { + Unspecified = 0, + Ready = 1, + Busy = 2 +}; + +class TKeepAliveResult : public TStatus { +public: + TKeepAliveResult(TStatus&& status, ESessionStatus sessionStatus); + ESessionStatus GetSessionStatus() const; +private: + ESessionStatus SessionStatus; +}; + class TBulkUpsertResult : public TStatus { public: explicit TBulkUpsertResult(TStatus&& status); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h index 70d56cca8f..4a04b6ba5c 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h @@ -11,17 +11,17 @@ enum class EColumnFamilyCompression { LZ4, }; -//! State of build index operation -enum class EBuildIndexState { - Unspecified = 0, - Preparing = 1, - TransferData = 2, - Applying = 3, - Done = 4, - Cancellation = 5, - Cancelled = 6, -}; - +//! State of build index operation +enum class EBuildIndexState { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + Applying = 3, + Done = 4, + Cancellation = 5, + Cancelled = 6, +}; + enum class EIndexType { GlobalSync, GlobalAsync, diff --git a/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h b/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h index b75b897221..91b65f7181 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h +++ b/ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h @@ -1,19 +1,19 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - -namespace NYdb { -using TPeriodicCb = std::function<bool(NYql::TIssues&&, EStatus)>; - -// !!!Experimental!!! -// Allows to communicate with sdk core -class ICoreFacility { -public: - virtual ~ICoreFacility() = default; - // Add task to execute periodicaly - // Task should return false to stop execution - virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; -}; - -} // namespace NYdb + +namespace NYdb { +using TPeriodicCb = std::function<bool(NYql::TIssues&&, EStatus)>; + +// !!!Experimental!!! +// Allows to communicate with sdk core +class ICoreFacility { +public: + virtual ~ICoreFacility() = default; + // Add task to execute periodicaly + // Task should return false to stop execution + virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h index ddf23022cd..75b86933ce 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h @@ -15,16 +15,16 @@ public: using TCredentialsProviderPtr = std::shared_ptr<ICredentialsProvider>; -class ICoreFacility; +class ICoreFacility; class ICredentialsProviderFactory { public: virtual ~ICredentialsProviderFactory() = default; virtual TCredentialsProviderPtr CreateProvider() const = 0; - // !!!Experimental!!! - virtual TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const { - Y_UNUSED(facility); - return CreateProvider(); - } + // !!!Experimental!!! + virtual TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const { + Y_UNUSED(facility); + return CreateProvider(); + } virtual TStringType GetClientIdentity() const = 0; }; diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make index 1196ae317c..c58b2112f8 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make @@ -7,11 +7,11 @@ SRCS( login.cpp ) -PEERDIR( +PEERDIR( ydb/library/login ydb/public/api/grpc ydb/public/sdk/cpp/client/ydb_types/status ydb/library/yql/public/issue -) - +) + END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp index 75781c3e2e..f103a88ecc 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp @@ -1,12 +1,12 @@ #include "exceptions.h" - + namespace NYdb { - + TYdbException::TYdbException(const TString& reason) { - Append(reason); -} - -TContractViolation::TContractViolation(const TString& reason) + Append(reason); +} + +TContractViolation::TContractViolation(const TString& reason) : TYdbException(reason) {} - + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h index 6e899ac4ec..38c94b168a 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h @@ -1,18 +1,18 @@ -#pragma once - -#include <util/generic/yexception.h> - +#pragma once + +#include <util/generic/yexception.h> + namespace NYdb { - + class TYdbException : public yexception { -public: - using yexception::yexception; +public: + using yexception::yexception; TYdbException(const TString& reason); -}; - +}; + class TContractViolation : public TYdbException { -public: - TContractViolation(const TString& reason); -}; - +public: + TContractViolation(const TString& reason); +}; + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp index 0070f6ad37..69b94f9b40 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp @@ -3,8 +3,8 @@ namespace NYdb { -void ThrowFatalError(const TString& str) { - throw TContractViolation(str); +void ThrowFatalError(const TString& str) { + throw TContractViolation(str); } } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h index e1a5d8f3cd..6af54a52f2 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h @@ -6,6 +6,6 @@ namespace NYdb { -void ThrowFatalError(const TString& str); +void ThrowFatalError(const TString& str); } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h index 4412887993..bc5dce05eb 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h @@ -7,14 +7,14 @@ #include <google/protobuf/stubs/status.h> #include <google/protobuf/util/json_util.h> -namespace Ydb { -namespace Operations { - -class Operation; - -} // namespace Operations -} // namespace Ydb - +namespace Ydb { +namespace Operations { + +class Operation; + +} // namespace Operations +} // namespace Ydb + namespace NYdb { class TStatus; diff --git a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h index 92427fbcdc..fb8aa8ac68 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h @@ -51,7 +51,7 @@ struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { FLUENT_SETTING(TDuration, OperationTimeout); FLUENT_SETTING(TDuration, CancelAfter); FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); - FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); + FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); TOperationRequestSettings() = default; @@ -61,7 +61,7 @@ struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { , OperationTimeout_(other.OperationTimeout_) , CancelAfter_(other.CancelAfter_) , UseClientTimeoutForOperation_(other.UseClientTimeoutForOperation_) - , ReportCostInfo_(other.ReportCostInfo_) + , ReportCostInfo_(other.ReportCostInfo_) {} TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) { diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index 799031c23f..d3f2ca05af 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -5,14 +5,14 @@ #undef INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> - + namespace NYdb { class TStatus::TImpl { public: const TPlainStatus Status; - TImpl(TPlainStatus&& status) + TImpl(TPlainStatus&& status) : Status(std::move(status)) { } @@ -23,16 +23,16 @@ public: } void RaiseError(const TStringType& str) const { - ythrow TContractViolation(str); + ythrow TContractViolation(str); } }; -TStatus::TStatus(EStatus statusCode, NYql::TIssues&& issues) - : Impl_(std::make_shared<TImpl>(TPlainStatus{statusCode, std::move(issues)})) +TStatus::TStatus(EStatus statusCode, NYql::TIssues&& issues) + : Impl_(std::make_shared<TImpl>(TPlainStatus{statusCode, std::move(issues)})) { } -TStatus::TStatus(TPlainStatus&& plain) - : Impl_(std::make_shared<TImpl>(std::move(plain))) +TStatus::TStatus(TPlainStatus&& plain) + : Impl_(std::make_shared<TImpl>(std::move(plain))) { } const NYql::TIssues& TStatus::GetIssues() const { @@ -68,10 +68,10 @@ const std::multimap<TStringType, TStringType>& TStatus::GetResponseMetadata() co return Impl_->Status.Metadata; } -float TStatus::GetConsumedRu() const { - return Impl_->Status.ConstInfo.consumed_units(); -} - +float TStatus::GetConsumedRu() const { + return Impl_->Status.ConstInfo.consumed_units(); +} + //////////////////////////////////////////////////////////////////////////////// TStreamPartStatus::TStreamPartStatus(TStatus&& status) diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index eed43e78e2..7185a03964 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -3,7 +3,7 @@ #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> #include <library/cpp/threading/future/future.h> @@ -16,8 +16,8 @@ struct TPlainStatus; //! Represents status of call class TStatus { public: - TStatus(EStatus statusCode, NYql::TIssues&& issues); - TStatus(TPlainStatus&& plain); + TStatus(EStatus statusCode, NYql::TIssues&& issues); + TStatus(TPlainStatus&& plain); EStatus GetStatus() const; const NYql::TIssues& GetIssues() const; @@ -25,7 +25,7 @@ public: bool IsTransportError() const; const TStringType& GetEndpoint() const; const std::multimap<TStringType, TStringType>& GetResponseMetadata() const; - float GetConsumedRu() const; + float GetConsumedRu() const; protected: void CheckStatusOk(const TStringType& str) const; diff --git a/ydb/public/sdk/cpp/client/ydb_types/status_codes.h b/ydb/public/sdk/cpp/client/ydb_types/status_codes.h index 6eb80fb633..9bc239c057 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status_codes.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status_codes.h @@ -1,53 +1,53 @@ -#pragma once - -#include <util/system/types.h> - +#pragma once + +#include <util/system/types.h> + namespace NYdb { - -constexpr size_t TRANSPORT_STATUSES_FIRST = 401000; -constexpr size_t TRANSPORT_STATUSES_LAST = 401999; -constexpr size_t INTERNAL_CLIENT_FIRST = 402000; - -enum class EStatus : size_t { - // Server statuses + +constexpr size_t TRANSPORT_STATUSES_FIRST = 401000; +constexpr size_t TRANSPORT_STATUSES_LAST = 401999; +constexpr size_t INTERNAL_CLIENT_FIRST = 402000; + +enum class EStatus : size_t { + // Server statuses STATUS_UNDEFINED = 0, - SUCCESS = 400000, - BAD_REQUEST = 400010, - UNAUTHORIZED = 400020, - INTERNAL_ERROR = 400030, - ABORTED = 400040, - UNAVAILABLE = 400050, - OVERLOADED = 400060, - SCHEME_ERROR = 400070, - GENERIC_ERROR = 400080, - TIMEOUT = 400090, - BAD_SESSION = 400100, - PRECONDITION_FAILED = 400120, - ALREADY_EXISTS = 400130, + SUCCESS = 400000, + BAD_REQUEST = 400010, + UNAUTHORIZED = 400020, + INTERNAL_ERROR = 400030, + ABORTED = 400040, + UNAVAILABLE = 400050, + OVERLOADED = 400060, + SCHEME_ERROR = 400070, + GENERIC_ERROR = 400080, + TIMEOUT = 400090, + BAD_SESSION = 400100, + PRECONDITION_FAILED = 400120, + ALREADY_EXISTS = 400130, NOT_FOUND = 400140, SESSION_EXPIRED = 400150, CANCELLED = 400160, UNDETERMINED = 400170, UNSUPPORTED = 400180, SESSION_BUSY = 400190, - - // Client statuses + + // Client statuses // Cannot connect or unrecoverable network error. (map from gRPC UNAVAILABLE) - TRANSPORT_UNAVAILABLE = TRANSPORT_STATUSES_FIRST + 10, - // No more resources to accept RPC call - CLIENT_RESOURCE_EXHAUSTED = TRANSPORT_STATUSES_FIRST + 20, + TRANSPORT_UNAVAILABLE = TRANSPORT_STATUSES_FIRST + 10, + // No more resources to accept RPC call + CLIENT_RESOURCE_EXHAUSTED = TRANSPORT_STATUSES_FIRST + 20, // Network layer does not receive response in given time - CLIENT_DEADLINE_EXCEEDED = TRANSPORT_STATUSES_FIRST + 30, - // Unknown client error - CLIENT_INTERNAL_ERROR = TRANSPORT_STATUSES_FIRST + 50, - CLIENT_CANCELLED = TRANSPORT_STATUSES_FIRST + 60, - CLIENT_UNAUTHENTICATED = TRANSPORT_STATUSES_FIRST + 70, + CLIENT_DEADLINE_EXCEEDED = TRANSPORT_STATUSES_FIRST + 30, + // Unknown client error + CLIENT_INTERNAL_ERROR = TRANSPORT_STATUSES_FIRST + 50, + CLIENT_CANCELLED = TRANSPORT_STATUSES_FIRST + 60, + CLIENT_UNAUTHENTICATED = TRANSPORT_STATUSES_FIRST + 70, // Unknown gRPC call - CLIENT_CALL_UNIMPLEMENTED = TRANSPORT_STATUSES_FIRST + 80, - // Attempt to read out of stream - CLIENT_OUT_OF_RANGE = TRANSPORT_STATUSES_FIRST + 90, - CLIENT_DISCOVERY_FAILED = INTERNAL_CLIENT_FIRST + 10, // Not used - CLIENT_LIMITS_REACHED = INTERNAL_CLIENT_FIRST + 20 -}; - + CLIENT_CALL_UNIMPLEMENTED = TRANSPORT_STATUSES_FIRST + 80, + // Attempt to read out of stream + CLIENT_OUT_OF_RANGE = TRANSPORT_STATUSES_FIRST + 90, + CLIENT_DISCOVERY_FAILED = INTERNAL_CLIENT_FIRST + 10, // Not used + CLIENT_LIMITS_REACHED = INTERNAL_CLIENT_FIRST + 20 +}; + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index b148133f86..8c18d4d50f 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -1,8 +1,8 @@ #include "value.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h> -#undef INCLUDE_YDB_INTERNAL_H +#undef INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> @@ -13,22 +13,22 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> - + #include <util/generic/bitmap.h> #include <util/generic/map.h> #include <util/string/builder.h> namespace NYdb { -static void CheckKind(TTypeParser::ETypeKind actual, TTypeParser::ETypeKind expected, const TString& method) +static void CheckKind(TTypeParser::ETypeKind actual, TTypeParser::ETypeKind expected, const TString& method) { if (expected != actual) { - ThrowFatalError(TStringBuilder() << method << "(): invalid state, expected type: " + ThrowFatalError(TStringBuilder() << method << "(): invalid state, expected type: " << expected << ", actual: " << actual); } } -static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) { +static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) { using ETypeKind = TTypeParser::ETypeKind; switch (type.type_case()) { @@ -62,7 +62,7 @@ static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) { break; } - ThrowFatalError(TStringBuilder() << "Unexpected proto type kind: " << (ui32) type.type_case()); + ThrowFatalError(TStringBuilder() << "Unexpected proto type kind: " << (ui32) type.type_case()); return ETypeKind::Void; } @@ -107,7 +107,7 @@ const Ydb::Type& TType::GetProto() const { class TTypeParser::TImpl { public: - TImpl(const TType& type) + TImpl(const TType& type) : Type_(type) { Reset(); @@ -119,7 +119,7 @@ public: } ETypeKind GetKind(ui32 offset = 0) const { - return NYdb::GetKind(GetProto(offset)); + return NYdb::GetKind(GetProto(offset)); } EPrimitiveType GetPrimitive() const { @@ -140,34 +140,34 @@ public: ForwardStep(); } - void OpenVariant(int index) { - CheckKind(ETypeKind::Variant, "Open"); - const Ydb::VariantType& variantType = GetProto().variant_type(); - const google::protobuf::Message* nextPtr = nullptr; - switch (variantType.type_case()) { - case Ydb::VariantType::kTupleItems: { - auto& tupleType = variantType.tuple_items(); - if (index >= tupleType.elements_size()) { - return FatalError("variant index is out of range"); - } - nextPtr = &tupleType.elements(index); - break; - } - case Ydb::VariantType::kStructItems: { - auto& structType = variantType.struct_items(); - if (index >= structType.members_size()) { - return FatalError("variant index is out of range"); - } - nextPtr = &structType.members(index).type(); - break; - } - default: { - return FatalError("unknown variant type case"); - } - } - Path_.emplace_back(TProtoPosition{nextPtr, -1}); - } - + void OpenVariant(int index) { + CheckKind(ETypeKind::Variant, "Open"); + const Ydb::VariantType& variantType = GetProto().variant_type(); + const google::protobuf::Message* nextPtr = nullptr; + switch (variantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + auto& tupleType = variantType.tuple_items(); + if (index >= tupleType.elements_size()) { + return FatalError("variant index is out of range"); + } + nextPtr = &tupleType.elements(index); + break; + } + case Ydb::VariantType::kStructItems: { + auto& structType = variantType.struct_items(); + if (index >= structType.members_size()) { + return FatalError("variant index is out of range"); + } + nextPtr = &structType.members(index).type(); + break; + } + default: { + return FatalError("unknown variant type case"); + } + } + Path_.emplace_back(TProtoPosition{nextPtr, -1}); + } + template<ETypeKind kind> void Close() { CheckPreviousKind(kind, "Close"); @@ -229,9 +229,9 @@ public: idx = structType.members_size() - 1; hasIdx = false; } - if (idx >= 0) { - nextPtr = &structType.members(idx).type(); - } + if (idx >= 0) { + nextPtr = &structType.members(idx).type(); + } } else { nextPtr = &GetProto(); } @@ -245,9 +245,9 @@ public: idx = tupleType.elements_size() - 1; hasIdx = false; } - if (idx >= 0) { - nextPtr = &tupleType.elements(idx); - } + if (idx >= 0) { + nextPtr = &tupleType.elements(idx); + } } else { nextPtr = &GetProto(); } @@ -303,7 +303,7 @@ public: private: void CheckKind(ETypeKind kind, const TString& method) const { - NYdb::CheckKind(GetKind(), kind, method); + NYdb::CheckKind(GetKind(), kind, method); } void CheckPreviousKind(ETypeKind kind, const TString method) const { @@ -312,7 +312,7 @@ private: return; } - NYdb::CheckKind(GetKind(1), kind, method); + NYdb::CheckKind(GetKind(1), kind, method); } const Ydb::Type& GetProto(ui32 offset = 0) const { @@ -324,7 +324,7 @@ private: } void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TTypeParser: " << msg); + ThrowFatalError(TStringBuilder() << "TTypeParser: " << msg); } private: @@ -345,7 +345,7 @@ TTypeParser::TTypeParser(TTypeParser&&) = default; TTypeParser::~TTypeParser() = default; TTypeParser::TTypeParser(const TType& type) - : Impl_(new TImpl(type)) {} + : Impl_(new TImpl(type)) {} TTypeParser::ETypeKind TTypeParser::GetKind() const { return Impl_->GetKind(); @@ -417,20 +417,20 @@ void TTypeParser::DictKey() { void TTypeParser::DictPayload() { Impl_->DictPayload(); -} +} -void TTypeParser::OpenVariant(size_t index) { - Impl_->OpenVariant(index); +void TTypeParser::OpenVariant(size_t index) { + Impl_->OpenVariant(index); } void TTypeParser::OpenVariant() { Impl_->Open<ETypeKind::Variant>(); } -void TTypeParser::CloseVariant() { - Impl_->Close<ETypeKind::Variant>(); -} - +void TTypeParser::CloseVariant() { + Impl_->Close<ETypeKind::Variant>(); +} + void TTypeParser::OpenTagged() { Impl_->Open<ETypeKind::Tagged>(); } @@ -453,7 +453,7 @@ void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { case TTypeParser::ETypeKind::Decimal: { auto decimal = parser.GetDecimal(); - out << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")"; + out << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")"; //out << "Decimal"; break; } @@ -527,7 +527,7 @@ void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { break; default: - ThrowFatalError(TStringBuilder() + ThrowFatalError(TStringBuilder() << "Unexpected type kind: " << parser.GetKind()); } } @@ -545,12 +545,12 @@ class TTypeBuilder::TImpl { using ETypeKind = TTypeParser::ETypeKind; public: - TImpl() + TImpl() { Path_.emplace_back(TProtoPosition{&ProtoType_}); } - TImpl(Ydb::Type& type) + TImpl(Ydb::Type& type) { Path_.emplace_back(TProtoPosition{&type}); } @@ -703,19 +703,19 @@ private: } void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TTypeBuilder: " << msg); + ThrowFatalError(TStringBuilder() << "TTypeBuilder: " << msg); } void CheckKind(ETypeKind kind, const TString& method) { - NYdb::CheckKind(GetKind(), kind, method); + NYdb::CheckKind(GetKind(), kind, method); } void CheckPreviousKind(ETypeKind kind, const TString& method) { - NYdb::CheckKind(GetKind(1), kind, method); + NYdb::CheckKind(GetKind(1), kind, method); } ETypeKind GetKind(ui32 offset = 0) { - return NYdb::GetKind(GetProto(offset)); + return NYdb::GetKind(GetProto(offset)); } template<ETypeKind kind> @@ -745,7 +745,7 @@ TTypeBuilder::TTypeBuilder(TTypeBuilder&&) = default; TTypeBuilder::~TTypeBuilder() = default; TTypeBuilder::TTypeBuilder() - : Impl_(new TImpl()) {} + : Impl_(new TImpl()) {} TType TTypeBuilder::Build() { return Impl_->Build(); @@ -863,29 +863,29 @@ TTypeBuilder& TTypeBuilder::EndDict() { //////////////////////////////////////////////////////////////////////////////// -TDecimalValue::TDecimalValue(const Ydb::Value& valueProto, const TDecimalType& decimalType) - : DecimalType_(decimalType) - , Low_(valueProto.low_128()) - , Hi_(valueProto.high_128()) -{} - -TDecimalValue::TDecimalValue(const TString& decimalString, ui8 precision, ui8 scale) - : DecimalType_(precision, scale) -{ +TDecimalValue::TDecimalValue(const Ydb::Value& valueProto, const TDecimalType& decimalType) + : DecimalType_(decimalType) + , Low_(valueProto.low_128()) + , Hi_(valueProto.high_128()) +{} + +TDecimalValue::TDecimalValue(const TString& decimalString, ui8 precision, ui8 scale) + : DecimalType_(precision, scale) +{ NYql::NDecimal::TInt128 val = NYql::NDecimal::FromString(decimalString, precision, scale); - static_assert(sizeof(val) == 16, "wrong TInt128 size"); - char* buf = reinterpret_cast<char*>(&val); - Low_ = *(ui64*)buf; - Hi_ = *(i64*)(buf + 8); -} - -TString TDecimalValue::ToString() const { + static_assert(sizeof(val) == 16, "wrong TInt128 size"); + char* buf = reinterpret_cast<char*>(&val); + Low_ = *(ui64*)buf; + Hi_ = *(i64*)(buf + 8); +} + +TString TDecimalValue::ToString() const { NYql::NDecimal::TInt128 val = NYql::NDecimal::FromHalfs(Low_, Hi_); return NYql::NDecimal::ToString(val, DecimalType_.Precision, DecimalType_.Scale); -} - -//////////////////////////////////////////////////////////////////////////////// - +} + +//////////////////////////////////////////////////////////////////////////////// + class TValue::TImpl { public: TImpl(const TType& type, const Ydb::Value& valueProto) @@ -936,15 +936,15 @@ class TValueParser::TImpl { }; public: - TImpl(const TValue& value) + TImpl(const TValue& value) : Value_(value.Impl_) , TypeParser_(value.GetType()) { Reset(Value_->ProtoValue_); } - TImpl(const TType& type) - : TypeParser_(type) {} + TImpl(const TType& type) + : TypeParser_(type) {} void Reset(const Ydb::Value& value) { TypeParser_.Impl_->Reset(); @@ -1081,11 +1081,11 @@ public: return GetProto().text_value(); } - TDecimalValue GetDecimal() const { - CheckDecimal(); - return TDecimalValue(GetProto(), TypeParser_.GetDecimal()); - } - + TDecimalValue GetDecimal() const { + CheckDecimal(); + return TDecimalValue(GetProto(), TypeParser_.GetDecimal()); + } + void OpenOptional() { TypeParser_.OpenOptional(); @@ -1203,21 +1203,21 @@ public: TypeParser_.CloseDict(); } - void OpenVariant() { - auto variantIndex = GetProto().variant_index(); - TypeParser_.OpenVariant(variantIndex); - if (GetProto().value_case() == Ydb::Value::kNestedValue) { - AddPath(EParseKind::Value, &GetProto().nested_value()); - } else { - FatalError(TStringBuilder() << "No nested value for variant type."); - } - } - - void CloseVariant() { - PopPath(); - TypeParser_.CloseVariant(); - } - + void OpenVariant() { + auto variantIndex = GetProto().variant_index(); + TypeParser_.OpenVariant(variantIndex); + if (GetProto().value_case() == Ydb::Value::kNestedValue) { + AddPath(EParseKind::Value, &GetProto().nested_value()); + } else { + FatalError(TStringBuilder() << "No nested value for variant type."); + } + } + + void CloseVariant() { + PopPath(); + TypeParser_.CloseVariant(); + } + void OpenTagged() { TypeParser_.OpenTagged(); } @@ -1335,7 +1335,7 @@ private: } void CheckKind(ETypeKind kind, const TString& method) const { - NYdb::CheckKind(TypeParser_.GetKind(), kind, method); + NYdb::CheckKind(TypeParser_.GetKind(), kind, method); } void CheckTransportKind(Ydb::Value::ValueCase expectedCase) const { @@ -1356,11 +1356,11 @@ private: CheckTransportKind(GetPrimitiveValueCase(primitiveType)); } - void CheckDecimal() const { - CheckKind(ETypeKind::Decimal, "Get"); - CheckTransportKind(Ydb::Value::kLow128); - } - + void CheckDecimal() const { + CheckKind(ETypeKind::Decimal, "Get"); + CheckTransportKind(Ydb::Value::kLow128); + } + const Ydb::Value& GetProto() const { return *static_cast<const Ydb::Value*>(GetPathBack().Ptr); } @@ -1425,7 +1425,7 @@ private: } void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TValueParser: " << msg); + ThrowFatalError(TStringBuilder() << "TValueParser: " << msg); } private: @@ -1440,10 +1440,10 @@ TValueParser::TValueParser(TValueParser&&) = default; TValueParser::~TValueParser() = default; TValueParser::TValueParser(const TValue& value) - : Impl_(new TImpl(value)) {} + : Impl_(new TImpl(value)) {} TValueParser::TValueParser(const TType& type) - : Impl_(new TImpl(type)) {} + : Impl_(new TImpl(type)) {} void TValueParser::Reset(const Ydb::Value& value) { Impl_->Reset(value); @@ -1555,10 +1555,10 @@ const TString& TValueParser::GetDyNumber() const { return Impl_->GetDyNumber(); } -TDecimalValue TValueParser::GetDecimal() const { - return Impl_->GetDecimal(); -} - +TDecimalValue TValueParser::GetDecimal() const { + return Impl_->GetDecimal(); +} + //////////////////////////////////////////////////////////////////////////////// #define RET_OPT_VALUE(Type, Name) \ @@ -1663,10 +1663,10 @@ TMaybe<TString> TValueParser::GetOptionalDyNumber() const { RET_OPT_VALUE(TString, DyNumber); } -TMaybe<TDecimalValue> TValueParser::GetOptionalDecimal() const { - RET_OPT_VALUE(TDecimalValue, Decimal); -} - +TMaybe<TDecimalValue> TValueParser::GetOptionalDecimal() const { + RET_OPT_VALUE(TDecimalValue, Decimal); +} + //////////////////////////////////////////////////////////////////////////////// void TValueParser::OpenOptional() { @@ -1741,14 +1741,14 @@ void TValueParser::CloseDict() { Impl_->CloseDict(); } -void TValueParser::OpenVariant() { - Impl_->OpenVariant(); -} - -void TValueParser::CloseVariant() { - Impl_->CloseVariant(); -} - +void TValueParser::OpenVariant() { + Impl_->OpenVariant(); +} + +void TValueParser::CloseVariant() { + Impl_->CloseVariant(); +} + void TValueParser::OpenTagged() { Impl_->OpenTagged(); } @@ -1786,21 +1786,21 @@ class TValueBuilderImpl { }; public: - TValueBuilderImpl() - : TypeBuilder_() + TValueBuilderImpl() + : TypeBuilder_() { PushPath(ProtoValue_); } - TValueBuilderImpl(const TType& type) - : TypeBuilder_() + TValueBuilderImpl(const TType& type) + : TypeBuilder_() { PushPath(ProtoValue_); GetType().CopyFrom(TProtoAccessor::GetProto(type)); } - TValueBuilderImpl(Ydb::Type& type, Ydb::Value& value) - : TypeBuilder_(type) + TValueBuilderImpl(Ydb::Type& type, Ydb::Value& value) + : TypeBuilder_(type) { PushPath(value); } @@ -1941,12 +1941,12 @@ public: GetValue().set_text_value(value); } - void Decimal(const TDecimalValue& value) { - FillDecimalType(value.DecimalType_); - GetValue().set_low_128(value.Low_); - GetValue().set_high_128(value.Hi_); - } - + void Decimal(const TDecimalValue& value) { + FillDecimalType(value.DecimalType_); + GetValue().set_low_128(value.Low_); + GetValue().set_high_128(value.Hi_); + } + void BeginOptional() { SetBuildType(!CheckType(ETypeKind::Optional)); @@ -2313,12 +2313,12 @@ private: } } - void FillDecimalType(const TDecimalType& type) { - if (!CheckDecimalType()) { - TypeBuilder_.Decimal(type); - } - } - + void FillDecimalType(const TDecimalType& type) { + if (!CheckDecimalType()) { + TypeBuilder_.Decimal(type); + } + } + bool CheckType() { if (!GetType().type_case()) { return false; @@ -2332,7 +2332,7 @@ private: return false; } - auto expectedKind = GetKind(GetType()); + auto expectedKind = GetKind(GetType()); if (expectedKind != kind) { FatalError(TStringBuilder() << "Type mismatch, expected: " << expectedKind << ", actual: " << kind); @@ -2371,20 +2371,20 @@ private: return true; } - bool CheckDecimalType() { - if (!CheckType(ETypeKind::Decimal)) { - return false; - } - - return true; - } - + bool CheckDecimalType() { + if (!CheckType(ETypeKind::Decimal)) { + return false; + } + + return true; + } + void CheckContainerKind(ETypeKind kind) { if (Path_.size() < 2) { FatalError(TStringBuilder() << "No opened container"); } - auto actualKind = GetKind(GetType(1)); + auto actualKind = GetKind(GetType(1)); if (actualKind != kind) { FatalError(TStringBuilder() << "Container type mismatch, expected: " << kind << ", actual: " << actualKind); @@ -2443,7 +2443,7 @@ private: } void FatalError(const TString& msg) const { - ThrowFatalError(TStringBuilder() << "TValueBuilder: " << msg); + ThrowFatalError(TStringBuilder() << "TValueBuilder: " << msg); } private: @@ -2467,15 +2467,15 @@ TValueBuilderBase<TDerived>::~TValueBuilderBase() = default; template<typename TDerived> TValueBuilderBase<TDerived>::TValueBuilderBase() - : Impl_(new TValueBuilderImpl()) {} + : Impl_(new TValueBuilderImpl()) {} template<typename TDerived> TValueBuilderBase<TDerived>::TValueBuilderBase(const TType& type) - : Impl_(new TValueBuilderImpl(type)) {} + : Impl_(new TValueBuilderImpl(type)) {} template<typename TDerived> -TValueBuilderBase<TDerived>::TValueBuilderBase(Ydb::Type& type, Ydb::Value& value) - : Impl_(new TValueBuilderImpl(type, value)) {} +TValueBuilderBase<TDerived>::TValueBuilderBase(Ydb::Type& type, Ydb::Value& value) + : Impl_(new TValueBuilderImpl(type, value)) {} template<typename TDerived> void TValueBuilderBase<TDerived>::CheckValue() { @@ -2614,7 +2614,7 @@ TDerived& TValueBuilderBase<TDerived>::Json(const TString& value) { return static_cast<TDerived&>(*this); } -template<typename TDerived> +template<typename TDerived> TDerived& TValueBuilderBase<TDerived>::JsonDocument(const TString& value) { Impl_->JsonDocument(value); return static_cast<TDerived&>(*this); @@ -2627,11 +2627,11 @@ TDerived& TValueBuilderBase<TDerived>::DyNumber(const TString& value) { } template<typename TDerived> -TDerived& TValueBuilderBase<TDerived>::Decimal(const TDecimalValue& value) { - Impl_->Decimal(value); - return static_cast<TDerived&>(*this); -} - +TDerived& TValueBuilderBase<TDerived>::Decimal(const TDecimalValue& value) { + Impl_->Decimal(value); + return static_cast<TDerived&>(*this); +} + #define SET_OPT_VALUE_MAYBE(Name) \ if (value) { \ Impl_->BeginOptional(); \ diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index b9315a2e95..4a163b9220 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -125,11 +125,11 @@ public: void DictPayload(); void CloseDict(); - // Variant - void OpenVariant(size_t index); + // Variant + void OpenVariant(size_t index); void OpenVariant(); - void CloseVariant(); - + void CloseVariant(); + // Tagged void OpenTagged(); const TString& GetTag(); @@ -195,16 +195,16 @@ private: std::unique_ptr<TImpl> Impl_; }; -struct TDecimalValue { - TString ToString() const; - TDecimalValue(const Ydb::Value& decimalValueProto, const TDecimalType& decimalType); +struct TDecimalValue { + TString ToString() const; + TDecimalValue(const Ydb::Value& decimalValueProto, const TDecimalType& decimalType); TDecimalValue(const TString& decimalString, ui8 precision = 22, ui8 scale = 9); - - TDecimalType DecimalType_; - ui64 Low_; - i64 Hi_; -}; - + + TDecimalType DecimalType_; + ui64 Low_; + i64 Hi_; +}; + //! Representation of YDB value. class TValue { friend class TValueParser; @@ -256,7 +256,7 @@ public: const TString& GetUtf8() const; const TString& GetYson() const; const TString& GetJson() const; - TDecimalValue GetDecimal() const; + TDecimalValue GetDecimal() const; const TString& GetJsonDocument() const; const TString& GetDyNumber() const; @@ -282,7 +282,7 @@ public: TMaybe<TString> GetOptionalUtf8() const; TMaybe<TString> GetOptionalYson() const; TMaybe<TString> GetOptionalJson() const; - TMaybe<TDecimalValue> GetOptionalDecimal() const; + TMaybe<TDecimalValue> GetOptionalDecimal() const; TMaybe<TString> GetOptionalJsonDocument() const; TMaybe<TString> GetOptionalDyNumber() const; @@ -314,10 +314,10 @@ public: void DictPayload(); void CloseDict(); - // Variant - void OpenVariant(); - void CloseVariant(); - + // Variant + void OpenVariant(); + void CloseVariant(); + // Tagged void OpenTagged(); const TString& GetTag() const; @@ -359,7 +359,7 @@ public: TDerived& Utf8(const TString& value); TDerived& Yson(const TString& value); TDerived& Json(const TString& value); - TDerived& Decimal(const TDecimalValue& value); + TDerived& Decimal(const TDecimalValue& value); TDerived& JsonDocument(const TString& value); TDerived& DyNumber(const TString& value); @@ -433,7 +433,7 @@ protected: TValueBuilderBase(const TType& type); - TValueBuilderBase(Ydb::Type& type, Ydb::Value& value); + TValueBuilderBase(Ydb::Type& type, Ydb::Value& value); ~TValueBuilderBase(); diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp index 2af642775c..d7508a5f9b 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp @@ -11,7 +11,7 @@ namespace NYdb { -using TExpectedErrorException = yexception; +using TExpectedErrorException = yexception; Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(ParseType1) { @@ -20,7 +20,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { members { name: "Member1" type { - type_id: UINT32 + type_id: UINT32 } } members { @@ -28,7 +28,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { type { list_type { item { - type_id: STRING + type_id: STRING } } } @@ -40,7 +40,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { elements { optional_type { item { - type_id: UTF8 + type_id: UTF8 } } } @@ -63,21 +63,21 @@ Y_UNIT_TEST_SUITE(YdbValue) { NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType); UNIT_ASSERT_NO_DIFF(FormatType(protoType), - R"(Struct<'Member1':Uint32,'Member2':List<String>,'Member3':Tuple<Utf8?,Decimal(8,13),Void>>)"); + R"(Struct<'Member1':Uint32,'Member2':List<String>,'Member3':Tuple<Utf8?,Decimal(8,13),Void>>)"); } Y_UNIT_TEST(ParseType2) { auto protoTypeStr = R"( dict_type { key { - type_id: UINT32 + type_id: UINT32 } payload { struct_type { members { name: "Member1" type { - type_id: DATE + type_id: DATE } } } @@ -118,7 +118,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { .Build(); UNIT_ASSERT_NO_DIFF(FormatType(type), - R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Utf8?>>>)"); + R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Utf8?>>>)"); } Y_UNIT_TEST(BuildTypeReuse) { @@ -171,7 +171,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildTypeIncomplete) { try { - auto value = TTypeBuilder() + auto value = TTypeBuilder() .BeginTuple() .AddElement() .Primitive(EPrimitiveType::Uint32) @@ -191,7 +191,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { members { name: "Member1" type { - type_id: UINT32 + type_id: UINT32 } } members { @@ -199,7 +199,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { type { list_type { item { - type_id: STRING + type_id: STRING } } } @@ -211,12 +211,12 @@ Y_UNIT_TEST_SUITE(YdbValue) { elements { optional_type { item { - type_id: UTF8 + type_id: UTF8 } } } elements { - type_id: UTF8 + type_id: UTF8 } elements { void_type: NULL_VALUE @@ -267,14 +267,14 @@ Y_UNIT_TEST_SUITE(YdbValue) { auto protoTypeStr = R"( dict_type { key { - type_id: UINT32 + type_id: UINT32 } payload { struct_type { members { name: "Member1" type { - type_id: DATE + type_id: DATE } } } @@ -325,21 +325,21 @@ Y_UNIT_TEST_SUITE(YdbValue) { elements { optional_type { item { - type_id: UTF8 + type_id: UTF8 } } } elements { optional_type { item { - type_id: INT8 + type_id: INT8 } } } elements { optional_type { item { - type_id: DOUBLE + type_id: DOUBLE } } } @@ -350,13 +350,13 @@ Y_UNIT_TEST_SUITE(YdbValue) { } } } - elements { - optional_type { - item { - type_id: DYNUMBER - } - } - } + elements { + optional_type { + item { + type_id: DYNUMBER + } + } + } } )"; @@ -375,11 +375,11 @@ Y_UNIT_TEST_SUITE(YdbValue) { uint64_value: 7 } } - items { - nested_value { - text_value: "12.345" - } - } + items { + nested_value { + text_value: "12.345" + } + } )"; Ydb::Type protoType; @@ -400,14 +400,14 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalDouble(), TMaybe<double>()); UNIT_ASSERT(parser.TryNextElement()); UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalUint64(), (ui64)7); - UNIT_ASSERT(parser.TryNextElement()); - UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalDyNumber(), "12.345"); + UNIT_ASSERT(parser.TryNextElement()); + UNIT_ASSERT_VALUES_EQUAL(parser.GetOptionalDyNumber(), "12.345"); parser.CloseTuple(); } Y_UNIT_TEST(BuildValueIncomplete) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginTuple() .AddElement() .Uint32(10) @@ -423,7 +423,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueListItemMismatch1) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .Int32(17) @@ -444,7 +444,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { .Build(); try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem(itemValue) .AddListItem() @@ -464,7 +464,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { .Build(); try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .Int32(17) @@ -480,7 +480,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueListItemMismatch4) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginList() @@ -501,7 +501,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueEmptyListUnknown) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .EmptyList() .Build(); } catch (const TExpectedErrorException& e) { @@ -511,17 +511,17 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT(false); } - Y_UNIT_TEST(BuildDyNumberValue) { - auto value = TValueBuilder() - .DyNumber("12.345") - .Build(); - - UNIT_ASSERT_NO_DIFF(FormatValueYson(value), - R"("12.345")"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"("12.345")"); - } - + Y_UNIT_TEST(BuildDyNumberValue) { + auto value = TValueBuilder() + .DyNumber("12.345") + .Build(); + + UNIT_ASSERT_NO_DIFF(FormatValueYson(value), + R"("12.345")"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"("12.345")"); + } + Y_UNIT_TEST(BuildValueList) { auto intValue = TValueBuilder() .Int32(21) @@ -596,7 +596,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueBadCall) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .AddListItem() .EndList() .Build(); @@ -713,7 +713,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueOptionalMismatch1) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginOptional() @@ -734,7 +734,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueOptionalMismatch2) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginOptional() @@ -791,7 +791,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueStructMissingMember) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginStruct() @@ -908,7 +908,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueTupleElementsMismatch1) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginTuple() @@ -933,7 +933,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueTupleElementsMismatch2) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginTuple() @@ -962,7 +962,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueTupleTypeMismatch) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .BeginTuple() @@ -1029,7 +1029,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueDictTypeMismatch1) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginDict() .AddDictItem() .DictKey(TValueBuilder().Int32(1).Build()) @@ -1050,7 +1050,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueDictTypeMismatch2) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginDict() .AddDictItem() .DictKey() @@ -1114,7 +1114,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueDictEmptyNoType) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .EmptyDict() @@ -1129,7 +1129,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { Y_UNIT_TEST(BuildValueDictEmptyTypeMismatch) { try { - auto value = TValueBuilder() + auto value = TValueBuilder() .BeginList() .AddListItem() .EmptyDict( @@ -1175,10 +1175,10 @@ Y_UNIT_TEST_SUITE(YdbValue) { .Primitive(EPrimitiveType::Uint8) .EndOptional() .EndDict() - .AddElement() - .BeginOptional() - .Primitive(EPrimitiveType::DyNumber) - .EndOptional() + .AddElement() + .BeginOptional() + .Primitive(EPrimitiveType::DyNumber) + .EndOptional() .EndTuple() .Build(); @@ -1206,19 +1206,19 @@ Y_UNIT_TEST_SUITE(YdbValue) { .DictPayload() .EmptyOptional() .EndDict() - .AddElement() - .BeginOptional() - .DyNumber("12.345") - .EndOptional() + .AddElement() + .BeginOptional() + .DyNumber("12.345") + .EndOptional() .EndTuple() .Build(); UNIT_ASSERT_NO_DIFF(FormatType(value.GetType()), - R"(Tuple<Struct<'Name':String,'Value':Uint64>,Utf8?,List<Bool>,Dict<Int32,Uint8?>,DyNumber?>)"); + R"(Tuple<Struct<'Name':String,'Value':Uint64>,Utf8?,List<Bool>,Dict<Int32,Uint8?>,DyNumber?>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), - R"([["Sergey";1u];#;[%true];[[10;#]];["12.345"]])"); + R"([["Sergey";1u];#;[%true];[[10;#]];["12.345"]])"); UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([{"Name":"Sergey","Value":1},null,[true],[[10,null]],"12.345"])"); + R"([{"Name":"Sergey","Value":1},null,[true],[[10,null]],"12.345"])"); } } diff --git a/ydb/public/sdk/python/ydb/connection.py b/ydb/public/sdk/python/ydb/connection.py index 7baa4ff84e..573b276e20 100644 --- a/ydb/public/sdk/python/ydb/connection.py +++ b/ydb/public/sdk/python/ydb/connection.py @@ -21,7 +21,7 @@ _stubs_list = ( logger = logging.getLogger(__name__) DEFAULT_TIMEOUT = 600 YDB_DATABASE_HEADER = "x-ydb-database" -YDB_TRACE_ID_HEADER = "x-ydb-trace-id" +YDB_TRACE_ID_HEADER = "x-ydb-trace-id" YDB_REQUEST_TYPE_HEADER = "x-ydb-request-type" diff --git a/ydb/public/tools/lib/cmds/__init__.py b/ydb/public/tools/lib/cmds/__init__.py index a7c3c46ea1..977f0a4aae 100644 --- a/ydb/public/tools/lib/cmds/__init__.py +++ b/ydb/public/tools/lib/cmds/__init__.py @@ -37,9 +37,9 @@ def ensure_path_exists(path): def parse_erasure(args): - erasure = os.getenv("YDB_ERASURE") - if erasure is not None: - return Erasure.from_string(erasure) + erasure = os.getenv("YDB_ERASURE") + if erasure is not None: + return Erasure.from_string(erasure) if args.erasure is None: return None return Erasure.from_string(args.erasure) diff --git a/ydb/services/cms/cms_ut.cpp b/ydb/services/cms/cms_ut.cpp index b35e8b012c..7d5234715e 100644 --- a/ydb/services/cms/cms_ut.cpp +++ b/ydb/services/cms/cms_ut.cpp @@ -42,25 +42,25 @@ struct TCmsTestSettingsWithAuth : TCmsTestSettings { using TKikimrWithGrpcAndRootSchema = NYdb::TBasicKikimrWithGrpcAndRootSchema<TCmsTestSettings>; static Ydb::StatusIds::StatusCode WaitForOperationStatus(std::shared_ptr<grpc::Channel> channel, const TString& opId, const TString &token = "") { - std::unique_ptr<Ydb::Operation::V1::OperationService::Stub> stub; - stub = Ydb::Operation::V1::OperationService::NewStub(channel); - Ydb::Operations::GetOperationRequest request; + std::unique_ptr<Ydb::Operation::V1::OperationService::Stub> stub; + stub = Ydb::Operation::V1::OperationService::NewStub(channel); + Ydb::Operations::GetOperationRequest request; request.set_id(opId); - Ydb::Operations::GetOperationResponse response; + Ydb::Operations::GetOperationResponse response; bool run = true; while (run) { grpc::ClientContext context; if (token) - context.AddMetadata(YDB_AUTH_TICKET_HEADER, token); + context.AddMetadata(YDB_AUTH_TICKET_HEADER, token); auto status = stub->GetOperation(&context, request, &response); UNIT_ASSERT(status.ok()); //GRpc layer - OK - if (response.operation().ready() == false) { + if (response.operation().ready() == false) { Sleep(ITERATION_DURATION); } else { run = false; } } - return response.operation().status(); + return response.operation().status(); } static Ydb::Cms::GetDatabaseStatusResult WaitForTenantState(std::shared_ptr<grpc::Channel> channel, const TString& path, Ydb::Cms::GetDatabaseStatusResult::State state = Ydb::Cms::GetDatabaseStatusResult::RUNNING, const TString &token = "") { @@ -72,7 +72,7 @@ static Ydb::Cms::GetDatabaseStatusResult WaitForTenantState(std::shared_ptr<grpc while (true) { grpc::ClientContext context; if (token) - context.AddMetadata(YDB_AUTH_TICKET_HEADER, token); + context.AddMetadata(YDB_AUTH_TICKET_HEADER, token); auto status = stub->GetDatabaseStatus(&context, request, &response); UNIT_ASSERT(status.ok()); @@ -123,166 +123,166 @@ void InitConsoleConfig(TKikimrWithGrpcAndRootSchema &server) UNIT_ASSERT_VALUES_EQUAL(resp.GetStatus().GetCode(), Ydb::StatusIds::SUCCESS); } -template<typename TRequest> -static void SetSyncOperation(TRequest& req) { +template<typename TRequest> +static void SetSyncOperation(TRequest& req) { req.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); -} - -static void doSimpleTenantsTest(bool sync) { - TKikimrWithGrpcAndRootSchema server; - - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::CMS_TENANTS, NLog::PRI_TRACE); - - ui16 grpc = server.GetPort(); - TString id; - - std::shared_ptr<grpc::Channel> channel; - std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; - channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - - const TString tenant = "/Root/users/user-1"; - // create tenant - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::CreateDatabaseRequest request; - request.set_path(tenant); - if (sync) - SetSyncOperation(request); - auto unit = request.mutable_resources()->add_storage_units(); - unit->set_unit_kind("hdd"); - unit->set_count(1); - - Ydb::Cms::CreateDatabaseResponse response; - auto status = stub->CreateDatabase(&context, request, &response); - UNIT_ASSERT(status.ok()); - // ready flag true in sync mode - UNIT_ASSERT(response.operation().ready() == sync); - if (sync) { - UNIT_ASSERT_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } else { - id = response.operation().id(); +} + +static void doSimpleTenantsTest(bool sync) { + TKikimrWithGrpcAndRootSchema server; + + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::CMS_TENANTS, NLog::PRI_TRACE); + + ui16 grpc = server.GetPort(); + TString id; + + std::shared_ptr<grpc::Channel> channel; + std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; + channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + + const TString tenant = "/Root/users/user-1"; + // create tenant + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::CreateDatabaseRequest request; + request.set_path(tenant); + if (sync) + SetSyncOperation(request); + auto unit = request.mutable_resources()->add_storage_units(); + unit->set_unit_kind("hdd"); + unit->set_count(1); + + Ydb::Cms::CreateDatabaseResponse response; + auto status = stub->CreateDatabase(&context, request, &response); + UNIT_ASSERT(status.ok()); + // ready flag true in sync mode + UNIT_ASSERT(response.operation().ready() == sync); + if (sync) { + UNIT_ASSERT_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } else { + id = response.operation().id(); } - } + } if (!sync) { auto status = WaitForOperationStatus(channel, id); UNIT_ASSERT_EQUAL(status, Ydb::StatusIds::SUCCESS); } - { - server.Tenants_->Run(tenant); - WaitForTenantState(channel, tenant, Ydb::Cms::GetDatabaseStatusResult::RUNNING); - } - - // alter tenant - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::AlterDatabaseRequest request; - request.set_path(tenant); - - auto unit = request.add_storage_units_to_add(); - unit->set_unit_kind("hdd"); - unit->set_count(1); - - Ydb::Cms::AlterDatabaseResponse response; - auto status = stub->AlterDatabase(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready()); - UNIT_ASSERT_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - - // get tenant status - { - Ydb::Cms::GetDatabaseStatusResult result = WaitForTenantState(channel, tenant, Ydb::Cms::GetDatabaseStatusResult::RUNNING); - UNIT_ASSERT_VALUES_EQUAL(result.path(), tenant); - UNIT_ASSERT_VALUES_EQUAL(result.state(), Ydb::Cms::GetDatabaseStatusResult::RUNNING); - UNIT_ASSERT_VALUES_EQUAL(result.required_resources().storage_units_size(), 1); - auto unit = result.required_resources().storage_units(0); - UNIT_ASSERT_VALUES_EQUAL(unit.unit_kind(), "hdd"); - UNIT_ASSERT_VALUES_EQUAL(unit.count(), 2); - } - - // list tenants - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::ListDatabasesRequest request; - - Ydb::Cms::ListDatabasesResponse response; - auto status = stub->ListDatabases(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready()); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Cms::ListDatabasesResult result; - auto res = response.operation().result().UnpackTo(&result); - UNIT_ASSERT(res); - UNIT_ASSERT_VALUES_EQUAL(result.paths_size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result.paths(0), tenant); - } - - // remove tenants - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::RemoveDatabaseRequest request; - request.set_path("/Root/users/user-1"); - if (sync) - SetSyncOperation(request); - - Ydb::Cms::RemoveDatabaseResponse response; - auto status = stub->RemoveDatabase(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == sync); - if (sync) { - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - } else { - id = response.operation().id(); + { + server.Tenants_->Run(tenant); + WaitForTenantState(channel, tenant, Ydb::Cms::GetDatabaseStatusResult::RUNNING); + } + + // alter tenant + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::AlterDatabaseRequest request; + request.set_path(tenant); + + auto unit = request.add_storage_units_to_add(); + unit->set_unit_kind("hdd"); + unit->set_count(1); + + Ydb::Cms::AlterDatabaseResponse response; + auto status = stub->AlterDatabase(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready()); + UNIT_ASSERT_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + + // get tenant status + { + Ydb::Cms::GetDatabaseStatusResult result = WaitForTenantState(channel, tenant, Ydb::Cms::GetDatabaseStatusResult::RUNNING); + UNIT_ASSERT_VALUES_EQUAL(result.path(), tenant); + UNIT_ASSERT_VALUES_EQUAL(result.state(), Ydb::Cms::GetDatabaseStatusResult::RUNNING); + UNIT_ASSERT_VALUES_EQUAL(result.required_resources().storage_units_size(), 1); + auto unit = result.required_resources().storage_units(0); + UNIT_ASSERT_VALUES_EQUAL(unit.unit_kind(), "hdd"); + UNIT_ASSERT_VALUES_EQUAL(unit.count(), 2); + } + + // list tenants + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::ListDatabasesRequest request; + + Ydb::Cms::ListDatabasesResponse response; + auto status = stub->ListDatabases(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready()); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Cms::ListDatabasesResult result; + auto res = response.operation().result().UnpackTo(&result); + UNIT_ASSERT(res); + UNIT_ASSERT_VALUES_EQUAL(result.paths_size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.paths(0), tenant); + } + + // remove tenants + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::RemoveDatabaseRequest request; + request.set_path("/Root/users/user-1"); + if (sync) + SetSyncOperation(request); + + Ydb::Cms::RemoveDatabaseResponse response; + auto status = stub->RemoveDatabase(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == sync); + if (sync) { + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + } else { + id = response.operation().id(); } - } - if (!sync) { - auto status = WaitForOperationStatus(channel, id); - UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - - // get tenant status - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::GetDatabaseStatusRequest request; - request.set_path("/Root/users/user-1"); - - Ydb::Cms::GetDatabaseStatusResponse response; - auto status = stub->GetDatabaseStatus(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready()); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::NOT_FOUND); - } - - // list tenants - { - stub = Ydb::Cms::V1::CmsService::NewStub(channel); - grpc::ClientContext context; - - Ydb::Cms::ListDatabasesRequest request; - - Ydb::Cms::ListDatabasesResponse response; - auto status = stub->ListDatabases(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready()); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Cms::ListDatabasesResult result; - auto res = response.operation().result().UnpackTo(&result); - UNIT_ASSERT(res); - UNIT_ASSERT_VALUES_EQUAL(result.paths_size(), 0); } -} + if (!sync) { + auto status = WaitForOperationStatus(channel, id); + UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); + } + + // get tenant status + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::GetDatabaseStatusRequest request; + request.set_path("/Root/users/user-1"); + + Ydb::Cms::GetDatabaseStatusResponse response; + auto status = stub->GetDatabaseStatus(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready()); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::NOT_FOUND); + } + + // list tenants + { + stub = Ydb::Cms::V1::CmsService::NewStub(channel); + grpc::ClientContext context; + + Ydb::Cms::ListDatabasesRequest request; + + Ydb::Cms::ListDatabasesResponse response; + auto status = stub->ListDatabases(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready()); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Cms::ListDatabasesResult result; + auto res = response.operation().result().UnpackTo(&result); + UNIT_ASSERT(res); + UNIT_ASSERT_VALUES_EQUAL(result.paths_size(), 0); + } +} template <typename TTestSettings> void CheckCreateDatabase(NYdb::TBasicKikimrWithGrpcAndRootSchema<TTestSettings> &server, @@ -294,7 +294,7 @@ void CheckCreateDatabase(NYdb::TBasicKikimrWithGrpcAndRootSchema<TTestSettings> { std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; TString id; - + stub = Ydb::Cms::V1::CmsService::NewStub(channel); grpc::ClientContext context; if (token) @@ -357,15 +357,15 @@ void CheckRemoveDatabase(std::shared_ptr<grpc::Channel> channel, } } -Y_UNIT_TEST_SUITE(TGRpcCmsTest) { - Y_UNIT_TEST(SimpleTenantsTest) { - doSimpleTenantsTest(false); - } - - Y_UNIT_TEST(SimpleTenantsTestSyncOperation) { - doSimpleTenantsTest(true); - } - +Y_UNIT_TEST_SUITE(TGRpcCmsTest) { + Y_UNIT_TEST(SimpleTenantsTest) { + doSimpleTenantsTest(false); + } + + Y_UNIT_TEST(SimpleTenantsTestSyncOperation) { + doSimpleTenantsTest(true); + } + Y_UNIT_TEST(AuthTokenTest) { NYdb::TBasicKikimrWithGrpcAndRootSchema<TCmsTestSettingsWithAuth> server; server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::CMS_TENANTS, NLog::PRI_TRACE); @@ -373,7 +373,7 @@ Y_UNIT_TEST_SUITE(TGRpcCmsTest) { TString id; std::shared_ptr<grpc::Channel> channel; - std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; + std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); // create tenant @@ -397,10 +397,10 @@ Y_UNIT_TEST_SUITE(TGRpcCmsTest) { InitConsoleConfig(server); std::shared_ptr<grpc::Channel> channel; - std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; + std::unique_ptr<Ydb::Cms::V1::CmsService::Stub> stub; channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - stub = Ydb::Cms::V1::CmsService::NewStub(channel); + stub = Ydb::Cms::V1::CmsService::NewStub(channel); grpc::ClientContext context; Ydb::Cms::DescribeDatabaseOptionsRequest request; diff --git a/ydb/services/cms/grpc_service.cpp b/ydb/services/cms/grpc_service.cpp index e16a2adda5..33a7569982 100644 --- a/ydb/services/cms/grpc_service.cpp +++ b/ydb/services/cms/grpc_service.cpp @@ -41,26 +41,26 @@ void TGRpcCmsService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { #error ADD_REQUEST macro already defined #endif #define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::Cms::IN, Ydb::Cms::OUT, TGRpcCmsService>>(this, &Service_, CQ_, \ + MakeIntrusive<TGRpcRequest<Ydb::Cms::IN, Ydb::Cms::OUT, TGRpcCmsService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Ydb::Cms::V1::CmsService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("cms", #NAME))->Run(); - ADD_REQUEST(CreateDatabase, CreateDatabaseRequest, CreateDatabaseResponse, { + ADD_REQUEST(CreateDatabase, CreateDatabaseRequest, CreateDatabaseResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvCreateTenantRequest(ctx)); }) - ADD_REQUEST(AlterDatabase, AlterDatabaseRequest, AlterDatabaseResponse, { + ADD_REQUEST(AlterDatabase, AlterDatabaseRequest, AlterDatabaseResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvAlterTenantRequest(ctx)); }) - ADD_REQUEST(GetDatabaseStatus, GetDatabaseStatusRequest, GetDatabaseStatusResponse, { + ADD_REQUEST(GetDatabaseStatus, GetDatabaseStatusRequest, GetDatabaseStatusResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvGetTenantStatusRequest(ctx)); }) - ADD_REQUEST(ListDatabases, ListDatabasesRequest, ListDatabasesResponse, { + ADD_REQUEST(ListDatabases, ListDatabasesRequest, ListDatabasesResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvListTenantsRequest(ctx)); }) - ADD_REQUEST(RemoveDatabase, RemoveDatabaseRequest, RemoveDatabaseResponse, { + ADD_REQUEST(RemoveDatabase, RemoveDatabaseRequest, RemoveDatabaseResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvRemoveTenantRequest(ctx)); }) ADD_REQUEST(DescribeDatabaseOptions, DescribeDatabaseOptionsRequest, DescribeDatabaseOptionsResponse, { diff --git a/ydb/services/discovery/grpc_service.cpp b/ydb/services/discovery/grpc_service.cpp index 3354468a51..010322b5d9 100644 --- a/ydb/services/discovery/grpc_service.cpp +++ b/ydb/services/discovery/grpc_service.cpp @@ -8,13 +8,13 @@ namespace NKikimr { namespace NGRpcService { static TString GetSdkBuildInfo(NGrpc::IRequestContextBase* reqCtx) { - const auto& res = reqCtx->GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); - if (res.empty()) { - return {}; - } + const auto& res = reqCtx->GetPeerMetaValues(NYdb::YDB_SDK_BUILD_INFO_HEADER); + if (res.empty()) { + return {}; + } return TString{res[0]}; -} - +} + TGRpcDiscoveryService::TGRpcDiscoveryService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) @@ -48,9 +48,9 @@ TGRpcDiscoveryService::TGRpcDiscoveryService(NActors::TActorSystem *system, #error ADD_REQUEST macro already defined #endif #define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::Discovery::IN, Ydb::Discovery::OUT, TGRpcDiscoveryService>>(this, &Service_, CQ_, \ + MakeIntrusive<TGRpcRequest<Ydb::Discovery::IN, Ydb::Discovery::OUT, TGRpcDiscoveryService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *reqCtx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, reqCtx->GetPeer(), GetSdkBuildInfo(reqCtx)); \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, reqCtx->GetPeer(), GetSdkBuildInfo(reqCtx)); \ ACTION; \ }, &Ydb::Discovery::V1::DiscoveryService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("discovery", #NAME))->Run(); diff --git a/ydb/services/kesus/grpc_service.cpp b/ydb/services/kesus/grpc_service.cpp index 361e87d24d..c5826b6627 100644 --- a/ydb/services/kesus/grpc_service.cpp +++ b/ydb/services/kesus/grpc_service.cpp @@ -22,7 +22,7 @@ namespace NKikimr { namespace NKesus { //////////////////////////////////////////////////////////////////////////////// - + class TGRpcSessionActor : public TActorBootstrapped<TGRpcSessionActor> { @@ -645,7 +645,7 @@ void TKesusGRpcService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { #endif #define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<NGRpcService::TGRpcRequest<Ydb::Coordination::IN, Ydb::Coordination::OUT, TKesusGRpcService>>( \ + MakeIntrusive<NGRpcService::TGRpcRequest<Ydb::Coordination::IN, Ydb::Coordination::OUT, TKesusGRpcService>>( \ this, \ &Service_, \ CQ, \ diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp index ce48d1768f..aabd11d4ca 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp @@ -203,7 +203,7 @@ private: void RespondServiceUnavailable(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { Counters->DroppedRequestsCount->Inc(); - ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); + ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); } void HandleDiscoverPQClustersRequestWhileIniting(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp index 2abb04675e..0c061b9bea 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp @@ -160,7 +160,7 @@ public: } void Process() { - auto* result = TEvDiscoverPQClustersRequest::AllocateResult<DiscoverClustersResult>(Request); + auto* result = TEvDiscoverPQClustersRequest::AllocateResult<DiscoverClustersResult>(Request); auto statusCode = Ydb::StatusIds::INTERNAL_ERROR; diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index 495341eebf..654edfcfcd 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -948,7 +948,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct clientConfig.CheckAliveness = false; clientConfig.RetryPolicy = RetryPolicyForPipes; - t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig)); + t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig)); auto it = TopicGroups.find(t.second.TopicNameConverter->GetModernName()); if (it != TopicGroups.end()) { @@ -1257,7 +1257,7 @@ void TReadSessionActor::CloseSession(const TString& errorReason, const PersQueue LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc write failed"); Die(ctx); return; - } + } } else { LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed"); if (!Request->GetStreamCtx()->Finish(std::move(grpc::Status::OK))) { @@ -1355,7 +1355,7 @@ bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorConte NTabletPipe::TClientConfig clientConfig; clientConfig.CheckAliveness = false; clientConfig.RetryPolicy = RetryPolicyForPipes; - t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig)); + t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig)); if (InitDone) { ++(*PipeReconnects); ++(*Errors); @@ -1870,7 +1870,7 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct .BackoffMultiplier = 2, .DoFirstRetryInstantly = true }; - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); Y_VERIFY(TabletID); LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition @@ -2314,7 +2314,7 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) { .BackoffMultiplier = 2, .DoFirstRetryInstantly = true }; - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); NKikimrClient::TPersQueueRequest request; @@ -2801,20 +2801,20 @@ void TReadInfoActor::Bootstrap(const TActorContext& ctx) { TBase::Bootstrap(ctx); Become(&TThis::StateFunc); - auto request = dynamic_cast<const ReadInfoRequest*>(GetProtoRequest()); + auto request = dynamic_cast<const ReadInfoRequest*>(GetProtoRequest()); Y_VERIFY(request); ClientId = NPersQueue::ConvertNewConsumerName(request->consumer().path(), ctx); bool readOnlyLocal = request->get_only_original(); TIntrusivePtr<NACLib::TUserToken> token; - if (Request_->GetInternalToken().empty()) { + if (Request_->GetInternalToken().empty()) { if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { AnswerError("Unauthenticated access is forbidden, please provide credentials", PersQueue::ErrorCode::ACCESS_DENIED, ctx); return; } } else { - token = new NACLib::TUserToken(Request_->GetInternalToken()); + token = new NACLib::TUserToken(Request_->GetInternalToken()); } THashSet<TString> topicsToResolve; diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp index 07c75c1ce7..5f17a2f3ab 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp @@ -52,8 +52,8 @@ void TPQWriteService::Bootstrap(const TActorContext& ctx) { ui64 TPQWriteService::NextCookie() { return ++LastCookie; -} - +} + void TPQWriteService::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev, const TActorContext& ctx) { if (!DatacenterClassifier) { diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index 2105ad5076..3038cc82a6 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -565,7 +565,7 @@ void TWriteSessionActor::RequestNextPartition(const TActorContext& ctx) { .BackoffMultiplier = 2, .DoFirstRetryInstantly = true }; - PipeToBalancer = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, BalancerTabletId, clientConfig)); + PipeToBalancer = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, BalancerTabletId, clientConfig)); NTabletPipe::SendData(ctx, PipeToBalancer, x.Release()); } @@ -577,7 +577,7 @@ void TWriteSessionActor::Handle(TEvPersQueue::TEvGetPartitionIdForWriteResponse: } void TWriteSessionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TActorContext &ctx) { - auto& record = ev->Get()->Record.GetRef(); + auto& record = ev->Get()->Record.GetRef(); if (record.GetYdbStatus() == Ydb::StatusIds::ABORTED) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " messageGroupId " diff --git a/ydb/services/rate_limiter/rate_limiter_ut.cpp b/ydb/services/rate_limiter/rate_limiter_ut.cpp index 435904279a..4ac05a571c 100644 --- a/ydb/services/rate_limiter/rate_limiter_ut.cpp +++ b/ydb/services/rate_limiter/rate_limiter_ut.cpp @@ -4,29 +4,29 @@ #include <ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - + #include <ydb/core/grpc_services/local_rate_limiter.h> - + #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <util/string/builder.h> #include <ydb/public/api/protos/ydb_rate_limiter.pb.h> - + namespace NKikimr { using NYdb::NRateLimiter::TCreateResourceSettings; namespace { -void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { - protoValue.set_seconds(duration.Seconds()); - protoValue.set_nanos(duration.NanoSecondsOfSecond()); -} - +void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { + protoValue.set_seconds(duration.Seconds()); + protoValue.set_nanos(duration.NanoSecondsOfSecond()); +} + TString PrintStatus(const NYdb::TStatus& status) { TStringBuilder builder; builder << status.GetStatus(); @@ -52,8 +52,8 @@ TString PrintStatus(const NYdb::TStatus& status) { #define ASSERT_STATUS_SUCCESS(expr) ASSERT_STATUS_SUCCESS_C(expr, "") -class TTestSetup { -public: +class TTestSetup { +public: TTestSetup() : Driver(MakeDriverConfig()) , CoordinationClient(Driver) @@ -62,8 +62,8 @@ public: CreateCoordinationNode(); } - virtual ~TTestSetup() = default; - + virtual ~TTestSetup() = default; + NYdb::TDriverConfig MakeDriverConfig() { return NYdb::TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << Server.GetPort()); @@ -73,7 +73,7 @@ public: ASSERT_STATUS_SUCCESS_C(CoordinationClient.CreateNode(path), "\nPath: " << path); } - void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) { + void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) { const auto acquireResultFuture = RateLimiterClient.AcquireResource(coordinationNodePath, resourcePath, settings); ASSERT_STATUS(acquireResultFuture, expected); } @@ -86,67 +86,67 @@ public: NYdb::NRateLimiter::TRateLimiterClient RateLimiterClient; }; -class TTestSetupAcquireActor : public TTestSetup { -private: - class TAcquireActor : public TActorBootstrapped<TAcquireActor> { - public: - TAcquireActor( - const TString& coordinationNodePath, - const TString& resourcePath, - const NYdb::NRateLimiter::TAcquireResourceSettings& settings, - NThreading::TPromise<NYdb::TStatus> promise) - : CoordinationNodePath(coordinationNodePath) - , ResourcePath(resourcePath) - , Settings(settings) - , Promise(promise) - {} - - void Bootstrap(const TActorContext& ctx) { - Become(&TThis::StateWork); - Ydb::RateLimiter::AcquireResourceRequest request; - request.set_coordination_node_path(CoordinationNodePath); - request.set_resource_path(ResourcePath); - - SetDuration(Settings.OperationTimeout_, *request.mutable_operation_params()->mutable_operation_timeout()); - - if (Settings.IsUsedAmount_) { - request.set_used(Settings.Amount_.GetRef()); - } else { - request.set_required(Settings.Amount_.GetRef()); - } - - auto id = SelfId(); - - auto cb = [this, id](Ydb::RateLimiter::AcquireResourceResponse resp) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(resp.operation().issues(), opIssues); - NYdb::TStatus status(static_cast<NYdb::EStatus>(resp.operation().status()), std::move(opIssues)); - Promise.SetValue(status); - Send(id, new TEvents::TEvPoisonPill); - }; - - NKikimr::NRpcService::RateLimiterAcquireUseSameMailbox(std::move(request), "", "", std::move(cb), ctx); - } - - STATEFN(StateWork) { - Y_UNUSED(ev); - } - - private: - const TString CoordinationNodePath; - const TString ResourcePath; - const NYdb::NRateLimiter::TAcquireResourceSettings Settings; - NThreading::TPromise<NYdb::TStatus> Promise; - }; -public: - void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) { - auto promise = NThreading::NewPromise<NYdb::TStatus>(); - auto actor = new TAcquireActor(coordinationNodePath, resourcePath, settings, promise); - Server.GetRuntime()->GetAnyNodeActorSystem()->Register(actor); - ASSERT_STATUS(promise.GetFuture(), expected); - } -}; - +class TTestSetupAcquireActor : public TTestSetup { +private: + class TAcquireActor : public TActorBootstrapped<TAcquireActor> { + public: + TAcquireActor( + const TString& coordinationNodePath, + const TString& resourcePath, + const NYdb::NRateLimiter::TAcquireResourceSettings& settings, + NThreading::TPromise<NYdb::TStatus> promise) + : CoordinationNodePath(coordinationNodePath) + , ResourcePath(resourcePath) + , Settings(settings) + , Promise(promise) + {} + + void Bootstrap(const TActorContext& ctx) { + Become(&TThis::StateWork); + Ydb::RateLimiter::AcquireResourceRequest request; + request.set_coordination_node_path(CoordinationNodePath); + request.set_resource_path(ResourcePath); + + SetDuration(Settings.OperationTimeout_, *request.mutable_operation_params()->mutable_operation_timeout()); + + if (Settings.IsUsedAmount_) { + request.set_used(Settings.Amount_.GetRef()); + } else { + request.set_required(Settings.Amount_.GetRef()); + } + + auto id = SelfId(); + + auto cb = [this, id](Ydb::RateLimiter::AcquireResourceResponse resp) { + NYql::TIssues opIssues; + NYql::IssuesFromMessage(resp.operation().issues(), opIssues); + NYdb::TStatus status(static_cast<NYdb::EStatus>(resp.operation().status()), std::move(opIssues)); + Promise.SetValue(status); + Send(id, new TEvents::TEvPoisonPill); + }; + + NKikimr::NRpcService::RateLimiterAcquireUseSameMailbox(std::move(request), "", "", std::move(cb), ctx); + } + + STATEFN(StateWork) { + Y_UNUSED(ev); + } + + private: + const TString CoordinationNodePath; + const TString ResourcePath; + const NYdb::NRateLimiter::TAcquireResourceSettings Settings; + NThreading::TPromise<NYdb::TStatus> Promise; + }; +public: + void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) { + auto promise = NThreading::NewPromise<NYdb::TStatus>(); + auto actor = new TAcquireActor(coordinationNodePath, resourcePath, settings, promise); + Server.GetRuntime()->GetAnyNodeActorSystem()->Register(actor); + ASSERT_STATUS(promise.GetFuture(), expected); + } +}; + TString TTestSetup::CoordinationNodePath = "/Root/CoordinationNode"; } // namespace @@ -310,58 +310,58 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) { } } - std::unique_ptr<TTestSetup> MakeTestSetup(bool useActorApi) { - if (useActorApi) { - return std::make_unique<TTestSetupAcquireActor>(); - } - return std::make_unique<TTestSetup>(); - } - - void AcquireResourceManyRequired(bool useActorApi) { + std::unique_ptr<TTestSetup> MakeTestSetup(bool useActorApi) { + if (useActorApi) { + return std::make_unique<TTestSetupAcquireActor>(); + } + return std::make_unique<TTestSetup>(); + } + + void AcquireResourceManyRequired(bool useActorApi) { using NYdb::NRateLimiter::TAcquireResourceSettings; - auto setup = MakeTestSetup(useActorApi); - - ASSERT_STATUS_SUCCESS(setup->RateLimiterClient.CreateResource(TTestSetup::CoordinationNodePath, "res", + auto setup = MakeTestSetup(useActorApi); + + ASSERT_STATUS_SUCCESS(setup->RateLimiterClient.CreateResource(TTestSetup::CoordinationNodePath, "res", TCreateResourceSettings().MaxUnitsPerSecond(1).MaxBurstSizeCoefficient(42))); - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(10000).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(10000).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); for (int i = 0; i < 3; ++i) { - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT); - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); } } - void AcquireResourceManyUsed(bool useActorApi) { + void AcquireResourceManyUsed(bool useActorApi) { using NYdb::NRateLimiter::TAcquireResourceSettings; - auto setup = MakeTestSetup(useActorApi); - ASSERT_STATUS_SUCCESS(setup->RateLimiterClient.CreateResource(TTestSetup::CoordinationNodePath, "res", + auto setup = MakeTestSetup(useActorApi); + ASSERT_STATUS_SUCCESS(setup->RateLimiterClient.CreateResource(TTestSetup::CoordinationNodePath, "res", TCreateResourceSettings().MaxUnitsPerSecond(1).MaxBurstSizeCoefficient(42))); - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(10000).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(10000).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); for (int i = 0; i < 3; ++i) { - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT); - setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT); + setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); } } - - Y_UNIT_TEST(AcquireResourceManyRequiredGrpcApi) { - AcquireResourceManyRequired(false); - } - - Y_UNIT_TEST(AcquireResourceManyRequiredActorApi) { - AcquireResourceManyRequired(true); - } - - Y_UNIT_TEST(AcquireResourceManyUsedGrpcApi) { - AcquireResourceManyUsed(false); - } - - Y_UNIT_TEST(AcquireResourceManyUsedActorApi) { - AcquireResourceManyUsed(true); - } + + Y_UNIT_TEST(AcquireResourceManyRequiredGrpcApi) { + AcquireResourceManyRequired(false); + } + + Y_UNIT_TEST(AcquireResourceManyRequiredActorApi) { + AcquireResourceManyRequired(true); + } + + Y_UNIT_TEST(AcquireResourceManyUsedGrpcApi) { + AcquireResourceManyUsed(false); + } + + Y_UNIT_TEST(AcquireResourceManyUsedActorApi) { + AcquireResourceManyUsed(true); + } } } // namespace NKikimr diff --git a/ydb/services/ydb/index_ut/ya.make b/ydb/services/ydb/index_ut/ya.make index 07c0bd456a..7f61bca4bb 100644 --- a/ydb/services/ydb/index_ut/ya.make +++ b/ydb/services/ydb/index_ut/ya.make @@ -1,5 +1,5 @@ UNITTEST_FOR(ydb/services/ydb) - + OWNER( dcherednik g:kikimr @@ -9,31 +9,31 @@ FORK_SUBTESTS() SPLIT_FACTOR(5) -IF (SANITIZER_TYPE OR WITH_VALGRIND) +IF (SANITIZER_TYPE OR WITH_VALGRIND) TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) + SIZE(LARGE) + TAG(ya:fat) REQUIREMENTS( cpu:4 ram:32 ) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -SRCS( - ydb_index_ut.cpp -) - -PEERDIR( +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + ydb_index_ut.cpp +) + +PEERDIR( ydb/core/testlib ydb/public/lib/idx_test ydb/public/lib/yson_value ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_table -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/services/ydb/sdk_credprovider_ut/dummy_provider_ut.cpp b/ydb/services/ydb/sdk_credprovider_ut/dummy_provider_ut.cpp index 5107f6b4ba..17cc284573 100644 --- a/ydb/services/ydb/sdk_credprovider_ut/dummy_provider_ut.cpp +++ b/ydb/services/ydb/sdk_credprovider_ut/dummy_provider_ut.cpp @@ -1,128 +1,128 @@ #include <ydb/services/ydb/ydb_common_ut.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h> - -#define INCLUDE_YDB_INTERNAL_H - -/// !!!! JUST FOR UT, DO NOT COPY-PASTE !!! /// + +#define INCLUDE_YDB_INTERNAL_H + +/// !!!! JUST FOR UT, DO NOT COPY-PASTE !!! /// #include <ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h> -#undef INCLUDE_YDB_INTERNAL_H - -using namespace NYdb; - -namespace { - -class TExampleDummyProviderFactory : public ICredentialsProviderFactory { - class TExampleDummyProvider : public ICredentialsProvider { - private: - TPeriodicCb CreatePingPongTask(std::weak_ptr<ICoreFacility> facility) { - auto periodicCb = [this, facility](NYql::TIssues&&, EStatus status) { - if (status != EStatus::SUCCESS) { - return false; - } - - auto strong = facility.lock(); - if (!strong) - return false; - - auto responseCb = [this](Draft::Dummy::PingResponse* resp, TPlainStatus status) -> void { - UNIT_ASSERT(status.Ok()); - UNIT_ASSERT_VALUES_EQUAL(resp->payload(), "abc"); - (*RunCnt)++; - }; - - Draft::Dummy::PingRequest request; - request.set_copy(true); - request.set_payload("abc"); - - TGRpcConnectionsImpl::RunOnDiscoveryEndpoint<Draft::Dummy::DummyService, Draft::Dummy::PingRequest, Draft::Dummy::PingResponse>( - strong, std::move(request), std::move(responseCb), &Draft::Dummy::DummyService::Stub::AsyncPing, - TRpcRequestSettings(), TDuration::Seconds(1)); - return true; - }; - return periodicCb; - } - public: - TExampleDummyProvider(std::weak_ptr<ICoreFacility> facility, int* runCnt) - : RunCnt(runCnt) - { - auto strong = facility.lock(); - // must be able to promote in ctor - Y_VERIFY(strong); - strong->AddPeriodicTask(CreatePingPongTask(facility), TDuration::Seconds(1)); - } - - TString GetAuthInfo() const override { - return ""; - } - - bool IsValid() const override { - return true; - } - private: - int* RunCnt; - }; - -public: - TExampleDummyProviderFactory(int* runCnt) - : RunCnt(runCnt) - {} - - std::shared_ptr<ICredentialsProvider> CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { - return std::make_shared<TExampleDummyProvider>(std::move(facility), RunCnt); - } - - TStringType GetClientIdentity() const override { - return ToString((ui64)this); - } - - // Just for compatibility with old interface - std::shared_ptr<ICredentialsProvider> CreateProvider() const override { - Y_FAIL(); - return {}; - } - -private: - int* RunCnt; - -}; - -void RunTest(bool sync) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - int runCnt = 0; - - { - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location) - .SetDiscoveryMode(sync ? EDiscoveryMode::Sync : EDiscoveryMode::Async) - .SetCredentialsProviderFactory(std::make_shared<TExampleDummyProviderFactory>(&runCnt))); - // Creates DbDriverState in case of empty database - auto client = NYdb::NTable::TTableClient(driver); - - Y_UNUSED(client); - Sleep(TDuration::Seconds(5)); - } - - Cerr << runCnt << Endl; - UNIT_ASSERT_C(runCnt > 1, runCnt); -} - -} - -Y_UNIT_TEST_SUITE(SdkCredProvider) { - Y_UNIT_TEST(PingFromProviderSyncDiscovery) { - RunTest(true); - } - - Y_UNIT_TEST(PingFromProviderAsyncDiscovery) { - RunTest(false); - } -} - +#undef INCLUDE_YDB_INTERNAL_H + +using namespace NYdb; + +namespace { + +class TExampleDummyProviderFactory : public ICredentialsProviderFactory { + class TExampleDummyProvider : public ICredentialsProvider { + private: + TPeriodicCb CreatePingPongTask(std::weak_ptr<ICoreFacility> facility) { + auto periodicCb = [this, facility](NYql::TIssues&&, EStatus status) { + if (status != EStatus::SUCCESS) { + return false; + } + + auto strong = facility.lock(); + if (!strong) + return false; + + auto responseCb = [this](Draft::Dummy::PingResponse* resp, TPlainStatus status) -> void { + UNIT_ASSERT(status.Ok()); + UNIT_ASSERT_VALUES_EQUAL(resp->payload(), "abc"); + (*RunCnt)++; + }; + + Draft::Dummy::PingRequest request; + request.set_copy(true); + request.set_payload("abc"); + + TGRpcConnectionsImpl::RunOnDiscoveryEndpoint<Draft::Dummy::DummyService, Draft::Dummy::PingRequest, Draft::Dummy::PingResponse>( + strong, std::move(request), std::move(responseCb), &Draft::Dummy::DummyService::Stub::AsyncPing, + TRpcRequestSettings(), TDuration::Seconds(1)); + return true; + }; + return periodicCb; + } + public: + TExampleDummyProvider(std::weak_ptr<ICoreFacility> facility, int* runCnt) + : RunCnt(runCnt) + { + auto strong = facility.lock(); + // must be able to promote in ctor + Y_VERIFY(strong); + strong->AddPeriodicTask(CreatePingPongTask(facility), TDuration::Seconds(1)); + } + + TString GetAuthInfo() const override { + return ""; + } + + bool IsValid() const override { + return true; + } + private: + int* RunCnt; + }; + +public: + TExampleDummyProviderFactory(int* runCnt) + : RunCnt(runCnt) + {} + + std::shared_ptr<ICredentialsProvider> CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { + return std::make_shared<TExampleDummyProvider>(std::move(facility), RunCnt); + } + + TStringType GetClientIdentity() const override { + return ToString((ui64)this); + } + + // Just for compatibility with old interface + std::shared_ptr<ICredentialsProvider> CreateProvider() const override { + Y_FAIL(); + return {}; + } + +private: + int* RunCnt; + +}; + +void RunTest(bool sync) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + int runCnt = 0; + + { + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location) + .SetDiscoveryMode(sync ? EDiscoveryMode::Sync : EDiscoveryMode::Async) + .SetCredentialsProviderFactory(std::make_shared<TExampleDummyProviderFactory>(&runCnt))); + // Creates DbDriverState in case of empty database + auto client = NYdb::NTable::TTableClient(driver); + + Y_UNUSED(client); + Sleep(TDuration::Seconds(5)); + } + + Cerr << runCnt << Endl; + UNIT_ASSERT_C(runCnt > 1, runCnt); +} + +} + +Y_UNIT_TEST_SUITE(SdkCredProvider) { + Y_UNIT_TEST(PingFromProviderSyncDiscovery) { + RunTest(true); + } + + Y_UNIT_TEST(PingFromProviderAsyncDiscovery) { + RunTest(false); + } +} + diff --git a/ydb/services/ydb/sdk_credprovider_ut/ya.make b/ydb/services/ydb/sdk_credprovider_ut/ya.make index 3c47f97e88..22052fcae8 100644 --- a/ydb/services/ydb/sdk_credprovider_ut/ya.make +++ b/ydb/services/ydb/sdk_credprovider_ut/ya.make @@ -1,38 +1,38 @@ UNITTEST_FOR(ydb/services/ydb) - + OWNER( dcherednik g:kikimr ) -FORK_SUBTESTS() - -SPLIT_FACTOR(5) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) - REQUIREMENTS( - cpu:4 - ram:32 - ) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -SRCS( - dummy_provider_ut.cpp -) - -PEERDIR( +FORK_SUBTESTS() + +SPLIT_FACTOR(5) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) + REQUIREMENTS( + cpu:4 + ram:32 + ) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + dummy_provider_ut.cpp +) + +PEERDIR( ydb/core/testlib ydb/public/lib/yson_value ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_table -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/services/ydb/ut/ya.make b/ydb/services/ydb/ut/ya.make index 2dcb7078ee..b7209faa5f 100644 --- a/ydb/services/ydb/ut/ya.make +++ b/ydb/services/ydb/ut/ya.make @@ -1,11 +1,11 @@ UNITTEST_FOR(ydb/services/ydb) - + OWNER( dcherednik g:kikimr ) -FORK_SUBTESTS() +FORK_SUBTESTS() IF (SANITIZER_TYPE OR WITH_VALGRIND) SPLIT_FACTOR(60) @@ -16,14 +16,14 @@ ELSE() TIMEOUT(300) SIZE(MEDIUM) ENDIF() - -SRCS( + +SRCS( ydb_bulk_upsert_ut.cpp ydb_bulk_upsert_olap_ut.cpp ydb_coordination_ut.cpp ydb_index_table_ut.cpp ydb_import_ut.cpp - ydb_ut.cpp + ydb_ut.cpp ydb_s3_internal_ut.cpp ydb_scripting_ut.cpp ydb_table_ut.cpp @@ -34,9 +34,9 @@ SRCS( ydb_olapstore_ut.cpp json_udf.cpp re2_udf.cpp -) - -PEERDIR( +) + +PEERDIR( contrib/libs/apache/arrow library/cpp/getopt library/cpp/grpc/client @@ -55,10 +55,10 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_operation ydb/public/sdk/cpp/client/ydb_scheme ydb/services/ydb -) - +) + YQL_LAST_ABI_VERSION() REQUIREMENTS(ram:14) -END() +END() diff --git a/ydb/services/ydb/ya.make b/ydb/services/ydb/ya.make index 42ec00d6c5..a2f41a3e75 100644 --- a/ydb/services/ydb/ya.make +++ b/ydb/services/ydb/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( ydb_clickhouse_internal.cpp - ydb_dummy.cpp + ydb_dummy.cpp ydb_experimental.cpp ydb_export.cpp ydb_import.cpp @@ -18,9 +18,9 @@ SRCS( ydb_scripting.cpp ydb_table.cpp ydb_long_tx.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/monlib/encode library/cpp/uri ydb/core/base @@ -36,9 +36,9 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/api/protos ydb/library/yql/public/types -) - -END() +) + +END() RECURSE_FOR_TESTS( index_ut diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index 375e0d48f3..c067928c78 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <ydb/core/testlib/test_client.h> #include <ydb/core/formats/arrow_helpers.h> #include <ydb/services/ydb/ydb_dummy.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> - -#include "ydb_keys_ut.h" - + +#include "ydb_keys_ut.h" + #include <contrib/libs/apache/arrow/cpp/src/arrow/csv/api.h> // for WriteCSV() #include <contrib/libs/apache/arrow/cpp/src/arrow/io/api.h> #include <functional> -using namespace NKikimr; +using namespace NKikimr; namespace NYdb { - -using namespace Tests; + +using namespace Tests; using namespace NYdb; - + struct TKikimrTestSettings { static constexpr bool SSL = false; static constexpr bool AUTH = false; @@ -41,7 +41,7 @@ struct TKikimrTestNoSystemViews : TKikimrTestSettings { template <typename TestSettings = TKikimrTestSettings> class TBasicKikimrWithGrpcAndRootSchema { -public: +public: TBasicKikimrWithGrpcAndRootSchema( NKikimrConfig::TAppConfig appConfig = {}, const TVector<NKikimrKqp::TKqpSetting>& kqpSettings = {}, @@ -49,7 +49,7 @@ public: bool enableYq = false, TAppPrepare::TFnReg udfFrFactory = nullptr, std::function<void(TServerSettings& settings)> builder = nullptr) - { + { ui16 port = PortManager.GetPort(2134); ui16 grpc = PortManager.GetPort(2135); ServerSettings = new TServerSettings(port); @@ -73,7 +73,7 @@ public: ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); ServerSettings->SetKqpSettings(kqpSettings); ServerSettings->SetEnableAsyncIndexes(true); - ServerSettings->SetEnableDataColumnForIndexTable(true); + ServerSettings->SetEnableDataColumnForIndexTable(true); ServerSettings->SetEnableNotNullColumns(true); ServerSettings->SetEnableSystemViews(TestSettings::EnableSystemViews); ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); @@ -83,7 +83,7 @@ public: if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) { ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath()); } - ServerSettings->RegisterGrpcService<NKikimr::NGRpcService::TGRpcYdbDummyService>("dummy"); + ServerSettings->RegisterGrpcService<NKikimr::NGRpcService::TGRpcYdbDummyService>("dummy"); if (udfFrFactory) { ServerSettings->SetFrFactory(udfFrFactory); } @@ -94,10 +94,10 @@ public: Server_.Reset(new TServer(*ServerSettings)); Tenants_.Reset(new Tests::TTenants(Server_)); - //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NActors::NLog::PRI_DEBUG); - //Server_->GetRuntime()->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NActors::NLog::PRI_DEBUG); + //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NActors::NLog::PRI_DEBUG); + //Server_->GetRuntime()->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NActors::NLog::PRI_DEBUG); Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_INFO); - //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_DEBUG); + //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_DEBUG); //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_OLAPSHARD, NActors::NLog::PRI_DEBUG); //Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); if (enableYq) { @@ -107,29 +107,29 @@ public: NGrpc::TServerOptions grpcOption; if (TestSettings::AUTH) { - grpcOption.SetUseAuth(true); - } - grpcOption.SetPort(grpc); + grpcOption.SetUseAuth(true); + } + grpcOption.SetPort(grpc); if (TestSettings::SSL) { NGrpc::TSslData sslData{NYdbSslTestData::ServerCrt, - NYdbSslTestData::ServerKey, - NYdbSslTestData::CaCrt}; - grpcOption.SetSslData(sslData); - } - Server_->EnableGRpc(grpcOption); + NYdbSslTestData::ServerKey, + NYdbSslTestData::CaCrt}; + grpcOption.SetSslData(sslData); + } + Server_->EnableGRpc(grpcOption); TClient annoyingClient(*ServerSettings); - if (ServerSettings->AppConfig.GetDomainsConfig().GetSecurityConfig().GetEnforceUserTokenRequirement()) { - annoyingClient.SetSecurityToken("root@builtin"); - } - annoyingClient.InitRootScheme("Root"); - GRpcPort_ = grpc; - } - - ui16 GetPort() { - return GRpcPort_; - } - + if (ServerSettings->AppConfig.GetDomainsConfig().GetSecurityConfig().GetEnforceUserTokenRequirement()) { + annoyingClient.SetSecurityToken("root@builtin"); + } + annoyingClient.InitRootScheme("Root"); + GRpcPort_ = grpc; + } + + ui16 GetPort() { + return GRpcPort_; + } + TPortManager& GetPortManager() { return PortManager; } @@ -151,11 +151,11 @@ public: TServerSettings::TPtr ServerSettings; Tests::TServer::TPtr Server_; THolder<Tests::TTenants> Tenants_; -private: +private: TPortManager PortManager; - ui16 GRpcPort_; -}; - + ui16 GRpcPort_; +}; + struct TTestOlap { static constexpr const char * StoreName = "OlapStore"; static constexpr const char * TableName = "OlapTable"; @@ -310,4 +310,4 @@ using TKikimrWithGrpcAndRootSchemaWithAuth = TBasicKikimrWithGrpcAndRootSchema<T using TKikimrWithGrpcAndRootSchemaWithAuthAndSsl = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestWithAuthAndSsl>; using TKikimrWithGrpcAndRootSchemaNoSystemViews = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestNoSystemViews>; -} +} diff --git a/ydb/services/ydb/ydb_coordination_ut.cpp b/ydb/services/ydb/ydb_coordination_ut.cpp index 8cb0982d10..54c973a7e8 100644 --- a/ydb/services/ydb/ydb_coordination_ut.cpp +++ b/ydb/services/ydb/ydb_coordination_ut.cpp @@ -131,24 +131,24 @@ void ExpectError(TFuture<T>&& future, EStatus status) { } Y_UNIT_TEST_SUITE(TGRpcNewCoordinationClient) { - Y_UNIT_TEST(CheckUnauthorized) { - TKikimrWithGrpcAndRootSchema server; - TClientContext context(server, "bad@builtin"); - - auto result = context.Client.CreateNode("/Root/node1").ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::UNAUTHORIZED, TStatusDescription(result)); - bool issueFound = false; - for (auto& issue : result.GetIssues()) { - NYql::WalkThroughIssues(issue, false, [&issueFound](const NYql::TIssue& issue, ui16){ - if (issue.GetCode() == NKikimrIssues::TIssuesIds::ACCESS_DENIED) { - issueFound = true; - } - }); - } - UNIT_ASSERT_C(issueFound, TStatusDescription(result)); - } - + Y_UNIT_TEST(CheckUnauthorized) { + TKikimrWithGrpcAndRootSchema server; + TClientContext context(server, "bad@builtin"); + + auto result = context.Client.CreateNode("/Root/node1").ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::UNAUTHORIZED, TStatusDescription(result)); + bool issueFound = false; + for (auto& issue : result.GetIssues()) { + NYql::WalkThroughIssues(issue, false, [&issueFound](const NYql::TIssue& issue, ui16){ + if (issue.GetCode() == NKikimrIssues::TIssuesIds::ACCESS_DENIED) { + issueFound = true; + } + }); + } + UNIT_ASSERT_C(issueFound, TStatusDescription(result)); + } + Y_UNIT_TEST(CreateDropDescribe) { TKikimrWithGrpcAndRootSchema server; TClientContext context(server); diff --git a/ydb/services/ydb/ydb_dummy.cpp b/ydb/services/ydb/ydb_dummy.cpp index f7bf6a87af..c628b0b3e4 100644 --- a/ydb/services/ydb/ydb_dummy.cpp +++ b/ydb/services/ydb/ydb_dummy.cpp @@ -1,30 +1,30 @@ -#include "ydb_dummy.h" - +#include "ydb_dummy.h" + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> #include <ydb/core/grpc_services/rpc_deferrable.h> - + #include <ydb/core/grpc_streaming/grpc_streaming.h> - + #include <ydb/public/api/grpc/draft/dummy.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace Draft::Dummy; - -using TEvInfiniteRequest = TGRpcRequestWrapper<0, InfiniteRequest, InfiniteResponse, true>; + +namespace NKikimr { +namespace NGRpcService { + +using namespace Draft::Dummy; + +using TEvInfiniteRequest = TGRpcRequestWrapper<0, InfiniteRequest, InfiniteResponse, true>; static void HandlePing(NGrpc::IRequestContextBase* ctx) { - auto req = static_cast<const PingRequest*>(ctx->GetRequest()); - auto resp = google::protobuf::Arena::CreateMessage<PingResponse>(ctx->GetArena()); - if (req->copy()) { - resp->set_payload(req->payload()); - } + auto req = static_cast<const PingRequest*>(ctx->GetRequest()); + auto resp = google::protobuf::Arena::CreateMessage<PingResponse>(ctx->GetArena()); + if (req->copy()) { + resp->set_payload(req->payload()); + } ctx->Reply(resp, Ydb::StatusIds::SUCCESS); -} - +} + class TInfiniteRpc : public TRpcOperationRequestActor<TInfiniteRpc, TEvInfiniteRequest> { using TBase = TRpcOperationRequestActor<TInfiniteRpc, TEvInfiniteRequest>; @@ -57,136 +57,136 @@ private: } }; -class TBiStreamPingRequestRPC : public TActorBootstrapped<TBiStreamPingRequestRPC> { - using TBase = TActorBootstrapped<TBiStreamPingRequestRPC>; +class TBiStreamPingRequestRPC : public TActorBootstrapped<TBiStreamPingRequestRPC> { + using TBase = TActorBootstrapped<TBiStreamPingRequestRPC>; using IContext = NGRpcServer::IGRpcStreamingContext< - TEvBiStreamPingRequest::TRequest, - TEvBiStreamPingRequest::TResponse>; - -public: + TEvBiStreamPingRequest::TRequest, + TEvBiStreamPingRequest::TResponse>; + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::GRPC_REQ; } - - TBiStreamPingRequestRPC(TEvBiStreamPingRequest* msg) - : Request_(msg) {} - - void Bootstrap(const TActorContext& ctx) { - Y_UNUSED(ctx); - Request_->GetStreamCtx()->Attach(SelfId()); - Request_->GetStreamCtx()->Read(); - Become(&TBiStreamPingRequestRPC::StateWork); - } - - void Handle(IContext::TEvReadFinished::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::GRPC_SERVER, - "Received TEvReadFinished, success = " << ev->Get()->Success); - auto req = static_cast<const TEvBiStreamPingRequest::TRequest&>(ev->Get()->Record); - - if (req.copy()) { - Resp_.set_payload(req.payload()); - } - Request_->RefreshToken("someInvalidNewToken", ctx, SelfId()); - } - - void Handle(TGRpcRequestProxy::TEvRefreshTokenResponse::TPtr& ev, const TActorContext& ctx) { - LOG_ERROR_S(ctx, NKikimrServices::GRPC_SERVER, - "Received TEvRefreshTokenResponse, Authenticated = " << ev->Get()->Authenticated); - Request_->GetStreamCtx()->Write(std::move(Resp_)); - auto grpcStatus = grpc::Status(ev->Get()->Authenticated ? - grpc::StatusCode::OK : grpc::StatusCode::UNAUTHENTICATED, - ""); - Request_->GetStreamCtx()->Finish(grpcStatus); - PassAway(); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - HFunc(IContext::TEvReadFinished, Handle); - HFunc(TGRpcRequestProxy::TEvRefreshTokenResponse, Handle); - } - } -private: - std::unique_ptr<TEvBiStreamPingRequest> Request_; - TEvBiStreamPingRequest::TResponse Resp_; -}; - -TGRpcYdbDummyService::TGRpcYdbDummyService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId proxyActorId) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(proxyActorId) -{ } - -void TGRpcYdbDummyService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; - SetupIncomingRequests(std::move(logger)); -} - -void TGRpcYdbDummyService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - -bool TGRpcYdbDummyService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcYdbDummyService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - -void TGRpcYdbDummyService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Draft::Dummy::IN, Draft::Dummy::OUT, TGRpcYdbDummyService>>(this, &Service_, CQ_, \ + + TBiStreamPingRequestRPC(TEvBiStreamPingRequest* msg) + : Request_(msg) {} + + void Bootstrap(const TActorContext& ctx) { + Y_UNUSED(ctx); + Request_->GetStreamCtx()->Attach(SelfId()); + Request_->GetStreamCtx()->Read(); + Become(&TBiStreamPingRequestRPC::StateWork); + } + + void Handle(IContext::TEvReadFinished::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::GRPC_SERVER, + "Received TEvReadFinished, success = " << ev->Get()->Success); + auto req = static_cast<const TEvBiStreamPingRequest::TRequest&>(ev->Get()->Record); + + if (req.copy()) { + Resp_.set_payload(req.payload()); + } + Request_->RefreshToken("someInvalidNewToken", ctx, SelfId()); + } + + void Handle(TGRpcRequestProxy::TEvRefreshTokenResponse::TPtr& ev, const TActorContext& ctx) { + LOG_ERROR_S(ctx, NKikimrServices::GRPC_SERVER, + "Received TEvRefreshTokenResponse, Authenticated = " << ev->Get()->Authenticated); + Request_->GetStreamCtx()->Write(std::move(Resp_)); + auto grpcStatus = grpc::Status(ev->Get()->Authenticated ? + grpc::StatusCode::OK : grpc::StatusCode::UNAUTHENTICATED, + ""); + Request_->GetStreamCtx()->Finish(grpcStatus); + PassAway(); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + HFunc(IContext::TEvReadFinished, Handle); + HFunc(TGRpcRequestProxy::TEvRefreshTokenResponse, Handle); + } + } +private: + std::unique_ptr<TEvBiStreamPingRequest> Request_; + TEvBiStreamPingRequest::TResponse Resp_; +}; + +TGRpcYdbDummyService::TGRpcYdbDummyService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId proxyActorId) + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(proxyActorId) +{ } + +void TGRpcYdbDummyService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) { + CQ_ = cq; + SetupIncomingRequests(std::move(logger)); +} + +void TGRpcYdbDummyService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { + Limiter_ = limiter; +} + +bool TGRpcYdbDummyService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcYdbDummyService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + +void TGRpcYdbDummyService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + MakeIntrusive<TGRpcRequest<Draft::Dummy::IN, Draft::Dummy::OUT, TGRpcYdbDummyService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Draft::Dummy::DummyService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("dummy", #NAME))->Run(); - - ADD_REQUEST(Ping, PingRequest, PingResponse, { - HandlePing(ctx); - }) - - ADD_REQUEST(Infinite, InfiniteRequest, InfiniteResponse, { - ActorSystem_->Register(new TInfiniteRpc(new TEvInfiniteRequest(ctx))); - }) -#undef ADD_REQUEST - - // Example of using biStreamingRequest with GRpcRequestProxy - { - using TBiRequest = Draft::Dummy::PingRequest; - using TBiResponse = Draft::Dummy::PingResponse; - using TBiStreamGRpcRequest = NGRpcServer::TGRpcStreamingRequest< - TBiRequest, - TBiResponse, - TGRpcYdbDummyService, - 413>; - - TBiStreamGRpcRequest::Start( - this, - this->GetService(), - CQ_, - &Draft::Dummy::DummyService::AsyncService::RequestBiStreamPing, - [this](TIntrusivePtr<TBiStreamGRpcRequest::IContext> context) { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvBiStreamPingRequest(context)); - }, - *ActorSystem_, - "DummyService/BiStreamPing", - getCounterBlock("dummy", "biStreamPing", true, true), - nullptr); - } -} - -void TGRpcRequestProxy::Handle(TEvBiStreamPingRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TBiStreamPingRequestRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + + ADD_REQUEST(Ping, PingRequest, PingResponse, { + HandlePing(ctx); + }) + + ADD_REQUEST(Infinite, InfiniteRequest, InfiniteResponse, { + ActorSystem_->Register(new TInfiniteRpc(new TEvInfiniteRequest(ctx))); + }) +#undef ADD_REQUEST + + // Example of using biStreamingRequest with GRpcRequestProxy + { + using TBiRequest = Draft::Dummy::PingRequest; + using TBiResponse = Draft::Dummy::PingResponse; + using TBiStreamGRpcRequest = NGRpcServer::TGRpcStreamingRequest< + TBiRequest, + TBiResponse, + TGRpcYdbDummyService, + 413>; + + TBiStreamGRpcRequest::Start( + this, + this->GetService(), + CQ_, + &Draft::Dummy::DummyService::AsyncService::RequestBiStreamPing, + [this](TIntrusivePtr<TBiStreamGRpcRequest::IContext> context) { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvBiStreamPingRequest(context)); + }, + *ActorSystem_, + "DummyService/BiStreamPing", + getCounterBlock("dummy", "biStreamPing", true, true), + nullptr); + } +} + +void TGRpcRequestProxy::Handle(TEvBiStreamPingRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TBiStreamPingRequestRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_dummy.h b/ydb/services/ydb/ydb_dummy.h index 4a099ad622..7e3e74c506 100644 --- a/ydb/services/ydb/ydb_dummy.h +++ b/ydb/services/ydb/ydb_dummy.h @@ -1,35 +1,35 @@ -#pragma once - +#pragma once + #include <library/cpp/grpc/server/grpc_server.h> #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/monlib/counters/counters.h> #include <ydb/public/api/grpc/draft/dummy.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -class TGRpcYdbDummyService - : public NGrpc::TGrpcServiceBase<Draft::Dummy::DummyService> -{ -public: - TGRpcYdbDummyService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId proxyActorId); - - void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; - void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); - -private: - void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; - grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; - NActors::TActorId GRpcRequestProxyId_; - NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} -} + +namespace NKikimr { +namespace NGRpcService { + +class TGRpcYdbDummyService + : public NGrpc::TGrpcServiceBase<Draft::Dummy::DummyService> +{ +public: + TGRpcYdbDummyService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId proxyActorId); + + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; + void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; + + bool IncRequest(); + void DecRequest(); + +private: + void SetupIncomingRequests(NGrpc::TLoggerPtr logger); + + NActors::TActorSystem* ActorSystem_; + grpc::ServerCompletionQueue* CQ_ = nullptr; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + NActors::TActorId GRpcRequestProxyId_; + NGrpc::TGlobalLimiter* Limiter_ = nullptr; +}; + +} +} diff --git a/ydb/services/ydb/ydb_index_table_ut.cpp b/ydb/services/ydb/ydb_index_table_ut.cpp index 198c46a16e..3c31f43a5a 100644 --- a/ydb/services/ydb/ydb_index_table_ut.cpp +++ b/ydb/services/ydb/ydb_index_table_ut.cpp @@ -1,28 +1,28 @@ #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> - + #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/client/flat_ut_client.h> #include <ydb/library/yql/public/issue/yql_issue.h> - + #include "ydb_common_ut.h" - + #include <util/thread/factory.h> - + #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> - + using namespace NYdb; using namespace NYdb::NTable; - + void CreateTestTableWithIndex(NYdb::NTable::TTableClient& client) { auto sessionResult = client.CreateSession().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); auto session = sessionResult.GetSession(); - + const ui32 SHARD_COUNT = 4; - + { auto tableBuilder = client.GetTableBuilder() .AddNullableColumn("NameHash", EPrimitiveType::Uint32) @@ -32,16 +32,16 @@ void CreateTestTableWithIndex(NYdb::NTable::TTableClient& client) { .AddNullableColumn("Data", EPrimitiveType::String) .SetPrimaryKeyColumns({"NameHash", "Name"}) .AddSecondaryIndex("TimestampIndex",TVector<TString>({"Timestamp", "Name", "Version"})); - + auto tableSettings = NYdb::NTable::TCreateTableSettings().PartitioningPolicy( NYdb::NTable::TPartitioningPolicy().UniformPartitions(SHARD_COUNT)); - + auto result = session.CreateTable("/Root/Foo", tableBuilder.Build(), tableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } } - + Y_UNIT_TEST_SUITE(YdbIndexTable) { Y_UNIT_TEST(FastSplitIndex) { TKikimrWithGrpcAndRootSchema server; @@ -257,4 +257,4 @@ Y_UNIT_TEST_SUITE(YdbIndexTable) { UNIT_ASSERT_VALUES_EQUAL(result->Record.GetErrorReason(), "Adding or dropping columns in index table is not supported"); } } -} +} diff --git a/ydb/services/ydb/ydb_index_ut.cpp b/ydb/services/ydb/ydb_index_ut.cpp index 67ee89eb80..fa90b593e5 100644 --- a/ydb/services/ydb/ydb_index_ut.cpp +++ b/ydb/services/ydb/ydb_index_ut.cpp @@ -1,243 +1,243 @@ #include <ydb/public/lib/idx_test/idx_test.h> - -#include "ydb_common_ut.h" - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NIdxTest; - -struct TRunSettings { - const bool PkOverlap; - const bool IndexOverlap; - const bool WithDataColumn; - const bool UseNewEngine; -}; - -static void RunTest(ui32 shardsCount, ui32 rowsCount, ui32 indexCount, const TRunSettings& settings) { - bool pkOverlap = settings.PkOverlap; - bool indexOverlap = settings.IndexOverlap; - bool withDataColumn = settings.WithDataColumn; - - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - auto uploader = CreateUploader(driver, "Root/Test", TUploaderParams{shardsCount}); - - const TString& keyColumnName = "key"; - auto builder = TTableBuilder() - .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); - - TVector<TString> pks; - pks.push_back(keyColumnName); - - TVector<TString> dataColumn; - if (withDataColumn) { - dataColumn.push_back("value"); - } - - for (ui32 i = 0; i < indexCount; i++) { - TStringStream ss; - ss << "index_" << i; - builder.AddNullableColumn(ss.Str(), EPrimitiveType::Utf8); - - if (!pkOverlap) { - builder.AddSecondaryIndex(ss.Str() + "_name", TVector<TString>{ss.Str()}, dataColumn); - } else { - builder.AddSecondaryIndex(ss.Str() + "_name", TVector<TString>{ss.Str(), keyColumnName}, dataColumn); - } - if (indexOverlap) { - pks.push_back(ss.Str()); - } - } - builder.AddNullableColumn("value", EPrimitiveType::Uint32); - - builder.SetPrimaryKeyColumns(pks); - uploader->Run(CreateDataProvider(rowsCount, shardsCount, builder.Build())); - auto workLoader = CreateWorkLoader(driver); - ui32 stms = - IWorkLoader::LC_UPDATE | - IWorkLoader::LC_UPSERT | - IWorkLoader::LC_REPLACE | - IWorkLoader::LC_INSERT | - IWorkLoader::LC_UPDATE_ON | - IWorkLoader::LC_DELETE_ON | - IWorkLoader::LC_DELETE; - workLoader->Run("Root/Test", stms, IWorkLoader::TRunSettings{rowsCount, 5, settings.UseNewEngine}); - auto checker = CreateChecker(driver); - checker->Run("Root/Test"); - driver.Stop(true); -} - -Y_UNIT_TEST_SUITE(YdbIndexTable) { - Y_UNIT_TEST(MultiShardTableOneIndex) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings { - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableOneIndexNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = true - })); - }*/ - - Y_UNIT_TEST(MultiShardTableOneIndexDataColumn) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = true, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableOneIndexDataColumnNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = true, - .UseNewEngine = true - })); - }*/ - - Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlap) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = true, - .WithDataColumn = false, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = true, - .WithDataColumn = false, - .UseNewEngine = true - })); - }*/ - - Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapDataColumn) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = true, - .WithDataColumn = true, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapDataColumnNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = true, - .WithDataColumn = true, - .UseNewEngine = true - })); - }*/ - - Y_UNIT_TEST(MultiShardTableOneIndexPkOverlap) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableOneIndexPkOverlapNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ - .PkOverlap = true, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = true - })); - }*/ - - Y_UNIT_TEST(MultiShardTableTwoIndexes) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 2, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = false - })); - } - -/* Y_UNIT_TEST(MultiShardTableTwoIndexesNewEngine) { - UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 2, TRunSettings{ - .PkOverlap = false, - .IndexOverlap = false, - .WithDataColumn = false, - .UseNewEngine = true - })); - }*/ - - void RunOnlineBuildTest(bool withDataColumn, bool useNewEngine) { - - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - auto uploader = CreateUploader(driver, "Root/Test", TUploaderParams{1}); - - const TString& keyColumnName = "key"; - auto builder = TTableBuilder() - .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .AddSecondaryIndex("value", "value") - .SetPrimaryKeyColumns({keyColumnName}); - - if (withDataColumn) { - builder.AddNullableColumn("payload", EPrimitiveType::Double); - } - - try { - uploader->Run(CreateDataProvider(5000, 1, builder.Build())); - - auto workLoader = CreateWorkLoader(driver); - ui32 stms = - IWorkLoader::LC_UPSERT | - (withDataColumn ? IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN : IWorkLoader::LC_ALTER_ADD_INDEX); - workLoader->Run("Root/Test", stms, IWorkLoader::TRunSettings{2000, 1, useNewEngine}); - auto checker = CreateChecker(driver); - checker->Run("Root/Test"); - driver.Stop(true); - } catch (const std::exception& ex) { - Cerr << ex.what() << Endl; - Y_FAIL("test failed with exception"); - } - } - - Y_UNIT_TEST(OnlineBuild) { - RunOnlineBuildTest(false, false); - } - - Y_UNIT_TEST(OnlineBuildWithDataColumn) { - RunOnlineBuildTest(true, false); - } - - Y_UNIT_TEST(OnlineBuildNewEngine) { - RunOnlineBuildTest(false, true); - } - - Y_UNIT_TEST(OnlineBuildWithDataColumnNewEngine) { - RunOnlineBuildTest(true, true); - } - -} + +#include "ydb_common_ut.h" + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NIdxTest; + +struct TRunSettings { + const bool PkOverlap; + const bool IndexOverlap; + const bool WithDataColumn; + const bool UseNewEngine; +}; + +static void RunTest(ui32 shardsCount, ui32 rowsCount, ui32 indexCount, const TRunSettings& settings) { + bool pkOverlap = settings.PkOverlap; + bool indexOverlap = settings.IndexOverlap; + bool withDataColumn = settings.WithDataColumn; + + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + auto uploader = CreateUploader(driver, "Root/Test", TUploaderParams{shardsCount}); + + const TString& keyColumnName = "key"; + auto builder = TTableBuilder() + .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64); + + TVector<TString> pks; + pks.push_back(keyColumnName); + + TVector<TString> dataColumn; + if (withDataColumn) { + dataColumn.push_back("value"); + } + + for (ui32 i = 0; i < indexCount; i++) { + TStringStream ss; + ss << "index_" << i; + builder.AddNullableColumn(ss.Str(), EPrimitiveType::Utf8); + + if (!pkOverlap) { + builder.AddSecondaryIndex(ss.Str() + "_name", TVector<TString>{ss.Str()}, dataColumn); + } else { + builder.AddSecondaryIndex(ss.Str() + "_name", TVector<TString>{ss.Str(), keyColumnName}, dataColumn); + } + if (indexOverlap) { + pks.push_back(ss.Str()); + } + } + builder.AddNullableColumn("value", EPrimitiveType::Uint32); + + builder.SetPrimaryKeyColumns(pks); + uploader->Run(CreateDataProvider(rowsCount, shardsCount, builder.Build())); + auto workLoader = CreateWorkLoader(driver); + ui32 stms = + IWorkLoader::LC_UPDATE | + IWorkLoader::LC_UPSERT | + IWorkLoader::LC_REPLACE | + IWorkLoader::LC_INSERT | + IWorkLoader::LC_UPDATE_ON | + IWorkLoader::LC_DELETE_ON | + IWorkLoader::LC_DELETE; + workLoader->Run("Root/Test", stms, IWorkLoader::TRunSettings{rowsCount, 5, settings.UseNewEngine}); + auto checker = CreateChecker(driver); + checker->Run("Root/Test"); + driver.Stop(true); +} + +Y_UNIT_TEST_SUITE(YdbIndexTable) { + Y_UNIT_TEST(MultiShardTableOneIndex) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings { + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableOneIndexNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = true + })); + }*/ + + Y_UNIT_TEST(MultiShardTableOneIndexDataColumn) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = true, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableOneIndexDataColumnNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = true, + .UseNewEngine = true + })); + }*/ + + Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlap) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = true, + .WithDataColumn = false, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = true, + .WithDataColumn = false, + .UseNewEngine = true + })); + }*/ + + Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapDataColumn) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = true, + .WithDataColumn = true, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableOneIndexIndexOverlapDataColumnNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = true, + .WithDataColumn = true, + .UseNewEngine = true + })); + }*/ + + Y_UNIT_TEST(MultiShardTableOneIndexPkOverlap) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableOneIndexPkOverlapNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 1, TRunSettings{ + .PkOverlap = true, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = true + })); + }*/ + + Y_UNIT_TEST(MultiShardTableTwoIndexes) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 2, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = false + })); + } + +/* Y_UNIT_TEST(MultiShardTableTwoIndexesNewEngine) { + UNIT_ASSERT_NO_EXCEPTION(RunTest(10, 1000, 2, TRunSettings{ + .PkOverlap = false, + .IndexOverlap = false, + .WithDataColumn = false, + .UseNewEngine = true + })); + }*/ + + void RunOnlineBuildTest(bool withDataColumn, bool useNewEngine) { + + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + auto uploader = CreateUploader(driver, "Root/Test", TUploaderParams{1}); + + const TString& keyColumnName = "key"; + auto builder = TTableBuilder() + .AddNullableColumn(keyColumnName, EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .AddSecondaryIndex("value", "value") + .SetPrimaryKeyColumns({keyColumnName}); + + if (withDataColumn) { + builder.AddNullableColumn("payload", EPrimitiveType::Double); + } + + try { + uploader->Run(CreateDataProvider(5000, 1, builder.Build())); + + auto workLoader = CreateWorkLoader(driver); + ui32 stms = + IWorkLoader::LC_UPSERT | + (withDataColumn ? IWorkLoader::LC_ALTER_ADD_INDEX_WITH_DATA_COLUMN : IWorkLoader::LC_ALTER_ADD_INDEX); + workLoader->Run("Root/Test", stms, IWorkLoader::TRunSettings{2000, 1, useNewEngine}); + auto checker = CreateChecker(driver); + checker->Run("Root/Test"); + driver.Stop(true); + } catch (const std::exception& ex) { + Cerr << ex.what() << Endl; + Y_FAIL("test failed with exception"); + } + } + + Y_UNIT_TEST(OnlineBuild) { + RunOnlineBuildTest(false, false); + } + + Y_UNIT_TEST(OnlineBuildWithDataColumn) { + RunOnlineBuildTest(true, false); + } + + Y_UNIT_TEST(OnlineBuildNewEngine) { + RunOnlineBuildTest(false, true); + } + + Y_UNIT_TEST(OnlineBuildWithDataColumnNewEngine) { + RunOnlineBuildTest(true, true); + } + +} diff --git a/ydb/services/ydb/ydb_keys_ut.h b/ydb/services/ydb/ydb_keys_ut.h index 7e563ba60f..b3b04dc37b 100644 --- a/ydb/services/ydb/ydb_keys_ut.h +++ b/ydb/services/ydb/ydb_keys_ut.h @@ -1,130 +1,130 @@ -#pragma once - -// This file contains self signed ca and server cert for -// grpc ut -// Valid for 3560 days -namespace NYdbSslTestData { - -const TString CaCrt = R"___( ------BEGIN CERTIFICATE----- -MIIFgzCCA2ugAwIBAgIJAMrBPGTKCsO/MA0GCSqGSIb3DQEBCwUAMFgxCzAJBgNV -BAYTAlJVMQwwCgYDVQQIDANSdXMxCzAJBgNVBAcMAllhMQ0wCwYDVQQKDARUZXN0 -MQ0wCwYDVQQLDARUZXN0MRAwDgYDVQQDDAdSb290IENBMB4XDTE4MDYwNTE0NDMz -MFoXDTI4MDYwMjE0NDMzMFowWDELMAkGA1UEBhMCUlUxDDAKBgNVBAgMA1J1czEL -MAkGA1UEBwwCWWExDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxEDAOBgNV -BAMMB1Jvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDAM9Pi -KaewiIWqcgcWmpedv0hSWRGvMrrwoeab9a9SbvZcJoZaKla11kg4+SFb9uUoxKYk -sa4VCCHeeIwzdVuChjcliFxi+oIoqXehF+ezqqU5qWczyMM1+Gq6Dvoqc0duzBXD -nB6fYfe6mKchx8GRDb8XyGiNVTWXZBeLtDmAPFonr5qfsFNaDLWVtAcNdSYxhJSH -Mj4Ua+MdRK9QuNn2YXc37fp7VJ5VK8kHXyRnUXADMMZUPXj6MRbKp6bpMeKYvqUy -KciXB5WIjRCsD4q53/a08wMChwqxGuBNGGrLqmGRHsb0WrJeuzXPvYkOiEJdFDvH -TS3cLS/V/kSKFmfGJ9VAyFrN59zczoxTRvrNKMDa/2d5vYBgnkmGlRqyLMLld/vL -uA7YilOhDrjrf/8JjtFKxTMwrhUrRg036iWbrkwD76aywnY4cE9FEff6aXqVkVnE -EOCPi2GOh/oO5ZlJgK8D+FPdkf7chdoyGFyHHhjSE5j4FzvSzSIuNA5KqRjFnYhm -6fQYVNvoXO4Q53VBvCof1lzDy7jDAeugcIihd+ccJnHuAkfyI7p/3I1ScdL2mKVs -oifQcagf7V7nqpiOaPgpBl6Bm5pgP1oJgaELrBG5L1SiW9BzjBiUCryOspJb6t6R -S1AB3ni4BBm0QXSWM39GZyt5Iucb9l4cn5lpeQIDAQABo1AwTjAdBgNVHQ4EFgQU -xwtpEVC+iu4iPSnIaXFcVLBpQggwHwYDVR0jBBgwFoAUxwtpEVC+iu4iPSnIaXFc -VLBpQggwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAZDBp3To967TU -jg8wy9V18LeP32Fwfd1DR1Q4R0aujEfyYFNyQlDNPCRqN84n8P4Elt3Q6GFeyvRR -Kfp9Yl3/+Hm37K5YYi4YM/Av+jKknHqkmMDnB530o/Plvh2Rfpsc9WgNjbQT8VQ6 -QtReVS7anre9HY8/PMhN9ZZlozvJlnb0Hn9XfprJ3ozALAqVoUcCUnvq8/FyCRuK -NZW1jLWWxxhHiwluONoBnzEj34muzAVZ/3zB0ZIFl7qoIufq+pIDpuV+5j/GqnU/ -WSBHPOFBlbUOnp6MDmJxo1VXgbRn4Xvn+7Q5yAUId6h6cr8VHpFoFsLKoeRXebkO -BCcIOlLvhVMaDQHdLM9BThxe/s+Cr1QJ8I7WBcs1v/DiGCSBzMDlXxv3P8rN4NAJ -kUPT9s7jUPOzlnsEo0lZb8a5HVHJYx903U3ruJKOtNV0S+DvQLGAdIr9kgIB/nuH -mYqYZPYHgXak1TCtkdS1opmM7Y7LCC/mMLBdZ+CqZ6Xz1be4l55z0VKqh8Eia5/O -dk+s9cl/PCKOW8e77Aw9LlDwCoQV8ObidfCGaCp6kGFka3cChc7hu0/y0e3HVdF6 -q0shVparVk1Gu5b//FkkO21kDBK4soL2uayaOVPrpcryT/UInEgaq1i9cYFDsbVF -3aRN6nfiRZ49DH71I/CnhIy6d5/pwD4= ------END CERTIFICATE----- -)___"; - -const TString ServerCrt = R"___( ------BEGIN CERTIFICATE----- -MIIFKDCCAxACAQEwDQYJKoZIhvcNAQELBQAwWDELMAkGA1UEBhMCUlUxDDAKBgNV -BAgMA1J1czELMAkGA1UEBwwCWWExDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRl -c3QxEDAOBgNVBAMMB1Jvb3QgQ0EwHhcNMTgwNjA1MTQ0MzMyWhcNMjgwNjAyMTQ0 -MzMyWjBcMQswCQYDVQQGEwJSVTEMMAoGA1UECAwDUnVzMQswCQYDVQQHDAJZYTEN -MAsGA1UECgwEVGVzdDEPMA0GA1UECwwGU2VydmVyMRIwEAYDVQQDDAlsb2NhbGhv -c3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDiq7OYi9moBz3SQ8eW -YHK0L47IC+gmYKm87KZpph7CcDLmtqRhLUVGy1DhE6AdcMJqPD02+ddcqO/XnZCg -/8nTv7jbNyN7FkDLASfmRnzeG3XVWJgC2OKbu7zhwNLg48CHdXuwkxB7HzvQptB8 -3pXg7vccTYG7fm+lpUxy4OZRr43qmcPqXvh5f9G72xm1Q6O2/OSibxxab/xcaAIC -h3UGkXaLJykd8LKLF5kbKxM0rXT9TJ4FNQYhtCD99O5VVaRvtnNfDTG28WrhXD5S -tzDau99x7MWfj6DV6pG29j4udcmadhng8qyhIoYpYKkiKzI3Qz1AZTeX7KDQjTAV -yb3HMRZw6U2dxvX4yZLkKAAjWRTSzCJGf8Y8g4wj4s0NCVEOlJ4vx4PvMh39aQye -ha0pMs+Q0NkmKUYNmXqv71DsbWIkJDjvWqC/X+wREaGZykF1o+kGXFDQnObCEaX3 -Ctz92s7zWXsI2q7Bkw50ksS0LPb9ukD3XWSh6L0tTN+C6Lhxv+gwTqJu1XTv9toJ -Jm52RuS2UWZ25noYshUMas2sVM0Ak83R+XW2RSvOibGSoHXSpcW+bXnkwVxIdgNo -o7E4skSbaGUKYUpk2rvJI/D2HdLgYwnD9zGSe09mbqRfBPlb0KO16L4RsWtrH6EX -L60VoFDikVV6Rw1Ec2cP0dKD+QIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQAehG3o -Axyd7SXlq4QFB1y1062BU2+keI515pRfyAIpaMa8gDA17El6FLYkYhYro6QhfuSD -aNf6l681LiQZr9Xje6dXO77QR5fe1u72ocRzdE9d/XaYeHocHHwBwfAHRt7kSGgt -emYO0mwju9MxgJ3buvUvvJCjyWPEpaE2ot/cIeEU/4DzqVdWtGhvtUD/w1vGUV4d -F6K4k1Q1tl8ZN+mmZKYa9ppHTx92iDC3ZLH0tU8FA1sblmrBZWwDRimUSijuFM/W -+YS0C0+F+4DyDqJ/1oI3Y+BO3BY5A+GPhEbITH+sZLHUAWBIgRKXekmJg/1HuPNu -OsqKsnVMbgtFmitfg5HSUx99maEOlWEjbdFClH22Wpdxudlrq8IV0wCI5ZTvaP+y -pQfeAqYJOSBmFHiF30Rt0jYVRM4CEwqtqP7WCKZlQQQuVW+hnZFDcY0RkynQPJNM -VW9+pdUAXUHlIxn6HxSfZISLJeGihLlnKwnnjIj++oWT8tgQH8pBVmU8PSPLnLxH -zNllxqSFryAgXBmLeVymWu34nEL2MqASDsswDcMiJj0M9w1EHs/seis0PYB/c54g -/Y5Cgk2sRuiRWOr39H0Z49SfxqkMnmbjw0iLlopN4QFBQUqLmitJUoAZiHiwJPJc -4/YLQABjdGmF5lfq8xHflkOa83Wm3H5I5WgWGQ== ------END CERTIFICATE----- -)___"; - -const TString ServerKey = R"___( ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEA4quzmIvZqAc90kPHlmBytC+OyAvoJmCpvOymaaYewnAy5rak -YS1FRstQ4ROgHXDCajw9NvnXXKjv152QoP/J07+42zcjexZAywEn5kZ83ht11ViY -Atjim7u84cDS4OPAh3V7sJMQex870KbQfN6V4O73HE2Bu35vpaVMcuDmUa+N6pnD -6l74eX/Ru9sZtUOjtvzkom8cWm/8XGgCAod1BpF2iycpHfCyixeZGysTNK10/Uye -BTUGIbQg/fTuVVWkb7ZzXw0xtvFq4Vw+Urcw2rvfcezFn4+g1eqRtvY+LnXJmnYZ -4PKsoSKGKWCpIisyN0M9QGU3l+yg0I0wFcm9xzEWcOlNncb1+MmS5CgAI1kU0swi -Rn/GPIOMI+LNDQlRDpSeL8eD7zId/WkMnoWtKTLPkNDZJilGDZl6r+9Q7G1iJCQ4 -71qgv1/sERGhmcpBdaPpBlxQ0JzmwhGl9wrc/drO81l7CNquwZMOdJLEtCz2/bpA -911koei9LUzfgui4cb/oME6ibtV07/baCSZudkbktlFmduZ6GLIVDGrNrFTNAJPN -0fl1tkUrzomxkqB10qXFvm155MFcSHYDaKOxOLJEm2hlCmFKZNq7ySPw9h3S4GMJ -w/cxkntPZm6kXwT5W9Cjtei+EbFrax+hFy+tFaBQ4pFVekcNRHNnD9HSg/kCAwEA -AQKCAgBOWWU5vFVetCoVTOJnQy1CxRGIaj0zTsQ9DluzNv143glqIAWPpXNFti9d -rUfyBTDeQbYzE4bye15z2/3K+L9Nlv6Rn2x/NkDtKpgdC45Lw5gmR3o7ubYSeIEg -U2NQ4siAygYYEa1nsXMeexqjntiVqGP3/35xTZHP4uQa7UwPPixCxCWpFGy0qo7X -bTNkqV2keaOZ1egqBn1nf3f6YdH8lDkyfjXDKJi+ZUjB0FDSK4a9q0cPq7VT3wxs -W8Yp7vFov9r/JvNhNe9ouFa+hp23basdmObycVX5uxvk7xatPn+SCXKGg7tR8zoG -gWfU7LNt0KsSYCooNF2d2L+fOF2FHuXffsHM090bf0GE5J829CN/H6Wa44NI9LDn -wmtTXoHOWLqKL+iPj12HfMsx5zhcdMBY1qo7vIBZfR1mYyeWcuAAApBJWED9Z5tJ -pEpIVBRgKq6Q/Tr66gTmU5FLRCFKlbIjD2sZdMl/Rkp1rskNdjRLw5cDhn2X+MyM -cRdF0YkJjRph/DnPP/z2aQgx3/NfjXpEhWtd7o0Ipkp4zrPqZ6yyTThY2VYgT2Ce -PzBBy1Ib+avIXU1EExkFahaijFALq4h4c60bstoTfoGq12Y2B512FJKtwB8+80Ih -Jlrtr4vAsWzndTxpIIrBSgQxy/F5lDcxPg0J2xFOr9DbGC34pQKCAQEA/JV3hLCV -GhciZPzsZHb7gBfkC1l2dn1y4QdnaTdVO4n2p/l4k5ucvuHjWqrBzEiO/CxcfRAg -bqUqs/IKwJ4TLvQWAk5zwAe4diuy/jV5VY8ruOiFc56QhieYuog0asPkP2LRucVq -skunljIZNoHLUTfs80ggMmEHBuewv2gnrNuWPp1cNbjCstivpeRuUwKmL2bdQfFA -TyQQHWn9eItNhRqX7MNr4fz1QUUSUt7CWYYoDhJC/Oo45320AjQR/Lms2QYvz6iG -CzVmIJxSqC7Oykz6gVcKu+R+RdO8DLRb+fIaA7QgAX8/lxLlJuKVn5F8WfaAzRQ6 -yKmbI9+HFifGDwKCAQEA5byDrObyWTL4BTeRyFlHRYF6cTe2xBv/Kuk4Ec6zHaaV -E39PfTJn2t0BkDvK9FXtF9sjKTmhsjg3mmgMNOz8Ll2/z4df6/u4zjy1dZ309zyh -yDWcP4tW8Hj6kl8PFStdlRXxIBMt/Gpsf4w45q2cNmqdMU4VBgwhycp87fq7EWJX -0zPDJnzOHo/twG+tsl/ZzQsHyxPC54oG3uFKWnT8azJTBgGeAD+huoONGk44TpOj -uV6LQpIS+q+vwRLmSaMeAiEG2N/IbMEU6l1gbz9ohfcASqFIVmxht2AE1GpD0I1o -XlxeoP0uSP9TtSuhSHfXowjaD/IBxzMJRtsA+X5ddwKCAQEAy5JbpaH8Se7jValT -jRUoVnDq5wrPo2gwMpWZDv/9veLP3Un/mFgO2PmOGAEP+Olx9GR8ln9s5EBSTn2B -lQTSSUGIi4tXVynhzbwioyfOBttBTeJ5zFm7+aPoQE6OkI4ZY8ztY2BtQg4fn7n5 -AClUCL2eR+WVrYTt+O67UUlM0NCaIxUIwHOM2EA0MOwOzvCPqByrrv4V6rMSGeLW -21TKwcBROg224YjS0iwtPIU09ppdphmpy9Wqz0hM0InPBXVQjgmidydIAbij+xyC -sfIn0HyCWcQhbpYV/4lLQqIKj0RFGz8NnKdGRSiBb/mmxdin9Inr/V2Uky2/UAZU -BdNAmQKCAQAYZPcacf+D5zyc2TS6sHg38jK9OOxIUKy8sr0Ibwln+ZtU0azwH10V -yWf0F9VKMqGVaeiG5R69XOjSlX/OUufISJ4ofDh8R2NtStb94UL0ydRn/QFVYgde -S4pX5o4kclFilkzfWgoFBov00z/rhr3SrWl5pc/nr3wbAExZvMkGZIns3E85lAET -D7dwOquYCEOJWUV/k96bVXW7TvLlPgzbmSFlvuA3KIqU0ok2JN4nwdedxGNHM1me -ku83sjkP0qlKEpW2i7Stj6cX58hop7QCnaLDSfLzcljB7wk0QQBoccuGUYqez6ON -jsclsrdSiZ81Kah2Dv2PWGUAyBqHY5qrAoIBAQCtjsqk71UkhTi5LB2ism1aGAme -pKMY459g57wmSEWE/+rBWOBY5CZ7FMV1W+P1bXqydZOr6ptBEUjM4hW6YcsKmDtk -g/LriBtDnPkd/QvrrpOZnj8rrtFyOF/wMx2jFwp6Nes+QtRqQk0RXerWDSbsn9vL -xSN/nSB6IoYs5rlC2TRoxCHqEGPSGImi4hhpLhIrArpqTH8+OhuRRVwtMO8Qo2fS -LEbMU1ARtp9TVPVktwsdNOnCeyynJnQiB6fL6QnpQYFY3xyO1MuBSjruTKLqorPM -pqUJavKfAG3qyVCaGn4msSaPDNnG5QH7eCpbuemWgPDmx/br4fdx/U7VH8HE ------END RSA PRIVATE KEY----- -)___"; - -}; // namespace NYdbSslTestData +#pragma once + +// This file contains self signed ca and server cert for +// grpc ut +// Valid for 3560 days +namespace NYdbSslTestData { + +const TString CaCrt = R"___( +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIJAMrBPGTKCsO/MA0GCSqGSIb3DQEBCwUAMFgxCzAJBgNV +BAYTAlJVMQwwCgYDVQQIDANSdXMxCzAJBgNVBAcMAllhMQ0wCwYDVQQKDARUZXN0 +MQ0wCwYDVQQLDARUZXN0MRAwDgYDVQQDDAdSb290IENBMB4XDTE4MDYwNTE0NDMz +MFoXDTI4MDYwMjE0NDMzMFowWDELMAkGA1UEBhMCUlUxDDAKBgNVBAgMA1J1czEL +MAkGA1UEBwwCWWExDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxEDAOBgNV +BAMMB1Jvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDAM9Pi +KaewiIWqcgcWmpedv0hSWRGvMrrwoeab9a9SbvZcJoZaKla11kg4+SFb9uUoxKYk +sa4VCCHeeIwzdVuChjcliFxi+oIoqXehF+ezqqU5qWczyMM1+Gq6Dvoqc0duzBXD +nB6fYfe6mKchx8GRDb8XyGiNVTWXZBeLtDmAPFonr5qfsFNaDLWVtAcNdSYxhJSH +Mj4Ua+MdRK9QuNn2YXc37fp7VJ5VK8kHXyRnUXADMMZUPXj6MRbKp6bpMeKYvqUy +KciXB5WIjRCsD4q53/a08wMChwqxGuBNGGrLqmGRHsb0WrJeuzXPvYkOiEJdFDvH +TS3cLS/V/kSKFmfGJ9VAyFrN59zczoxTRvrNKMDa/2d5vYBgnkmGlRqyLMLld/vL +uA7YilOhDrjrf/8JjtFKxTMwrhUrRg036iWbrkwD76aywnY4cE9FEff6aXqVkVnE +EOCPi2GOh/oO5ZlJgK8D+FPdkf7chdoyGFyHHhjSE5j4FzvSzSIuNA5KqRjFnYhm +6fQYVNvoXO4Q53VBvCof1lzDy7jDAeugcIihd+ccJnHuAkfyI7p/3I1ScdL2mKVs +oifQcagf7V7nqpiOaPgpBl6Bm5pgP1oJgaELrBG5L1SiW9BzjBiUCryOspJb6t6R +S1AB3ni4BBm0QXSWM39GZyt5Iucb9l4cn5lpeQIDAQABo1AwTjAdBgNVHQ4EFgQU +xwtpEVC+iu4iPSnIaXFcVLBpQggwHwYDVR0jBBgwFoAUxwtpEVC+iu4iPSnIaXFc +VLBpQggwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAZDBp3To967TU +jg8wy9V18LeP32Fwfd1DR1Q4R0aujEfyYFNyQlDNPCRqN84n8P4Elt3Q6GFeyvRR +Kfp9Yl3/+Hm37K5YYi4YM/Av+jKknHqkmMDnB530o/Plvh2Rfpsc9WgNjbQT8VQ6 +QtReVS7anre9HY8/PMhN9ZZlozvJlnb0Hn9XfprJ3ozALAqVoUcCUnvq8/FyCRuK +NZW1jLWWxxhHiwluONoBnzEj34muzAVZ/3zB0ZIFl7qoIufq+pIDpuV+5j/GqnU/ +WSBHPOFBlbUOnp6MDmJxo1VXgbRn4Xvn+7Q5yAUId6h6cr8VHpFoFsLKoeRXebkO +BCcIOlLvhVMaDQHdLM9BThxe/s+Cr1QJ8I7WBcs1v/DiGCSBzMDlXxv3P8rN4NAJ +kUPT9s7jUPOzlnsEo0lZb8a5HVHJYx903U3ruJKOtNV0S+DvQLGAdIr9kgIB/nuH +mYqYZPYHgXak1TCtkdS1opmM7Y7LCC/mMLBdZ+CqZ6Xz1be4l55z0VKqh8Eia5/O +dk+s9cl/PCKOW8e77Aw9LlDwCoQV8ObidfCGaCp6kGFka3cChc7hu0/y0e3HVdF6 +q0shVparVk1Gu5b//FkkO21kDBK4soL2uayaOVPrpcryT/UInEgaq1i9cYFDsbVF +3aRN6nfiRZ49DH71I/CnhIy6d5/pwD4= +-----END CERTIFICATE----- +)___"; + +const TString ServerCrt = R"___( +-----BEGIN CERTIFICATE----- +MIIFKDCCAxACAQEwDQYJKoZIhvcNAQELBQAwWDELMAkGA1UEBhMCUlUxDDAKBgNV +BAgMA1J1czELMAkGA1UEBwwCWWExDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRl +c3QxEDAOBgNVBAMMB1Jvb3QgQ0EwHhcNMTgwNjA1MTQ0MzMyWhcNMjgwNjAyMTQ0 +MzMyWjBcMQswCQYDVQQGEwJSVTEMMAoGA1UECAwDUnVzMQswCQYDVQQHDAJZYTEN +MAsGA1UECgwEVGVzdDEPMA0GA1UECwwGU2VydmVyMRIwEAYDVQQDDAlsb2NhbGhv +c3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDiq7OYi9moBz3SQ8eW +YHK0L47IC+gmYKm87KZpph7CcDLmtqRhLUVGy1DhE6AdcMJqPD02+ddcqO/XnZCg +/8nTv7jbNyN7FkDLASfmRnzeG3XVWJgC2OKbu7zhwNLg48CHdXuwkxB7HzvQptB8 +3pXg7vccTYG7fm+lpUxy4OZRr43qmcPqXvh5f9G72xm1Q6O2/OSibxxab/xcaAIC +h3UGkXaLJykd8LKLF5kbKxM0rXT9TJ4FNQYhtCD99O5VVaRvtnNfDTG28WrhXD5S +tzDau99x7MWfj6DV6pG29j4udcmadhng8qyhIoYpYKkiKzI3Qz1AZTeX7KDQjTAV +yb3HMRZw6U2dxvX4yZLkKAAjWRTSzCJGf8Y8g4wj4s0NCVEOlJ4vx4PvMh39aQye +ha0pMs+Q0NkmKUYNmXqv71DsbWIkJDjvWqC/X+wREaGZykF1o+kGXFDQnObCEaX3 +Ctz92s7zWXsI2q7Bkw50ksS0LPb9ukD3XWSh6L0tTN+C6Lhxv+gwTqJu1XTv9toJ +Jm52RuS2UWZ25noYshUMas2sVM0Ak83R+XW2RSvOibGSoHXSpcW+bXnkwVxIdgNo +o7E4skSbaGUKYUpk2rvJI/D2HdLgYwnD9zGSe09mbqRfBPlb0KO16L4RsWtrH6EX +L60VoFDikVV6Rw1Ec2cP0dKD+QIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQAehG3o +Axyd7SXlq4QFB1y1062BU2+keI515pRfyAIpaMa8gDA17El6FLYkYhYro6QhfuSD +aNf6l681LiQZr9Xje6dXO77QR5fe1u72ocRzdE9d/XaYeHocHHwBwfAHRt7kSGgt +emYO0mwju9MxgJ3buvUvvJCjyWPEpaE2ot/cIeEU/4DzqVdWtGhvtUD/w1vGUV4d +F6K4k1Q1tl8ZN+mmZKYa9ppHTx92iDC3ZLH0tU8FA1sblmrBZWwDRimUSijuFM/W ++YS0C0+F+4DyDqJ/1oI3Y+BO3BY5A+GPhEbITH+sZLHUAWBIgRKXekmJg/1HuPNu +OsqKsnVMbgtFmitfg5HSUx99maEOlWEjbdFClH22Wpdxudlrq8IV0wCI5ZTvaP+y +pQfeAqYJOSBmFHiF30Rt0jYVRM4CEwqtqP7WCKZlQQQuVW+hnZFDcY0RkynQPJNM +VW9+pdUAXUHlIxn6HxSfZISLJeGihLlnKwnnjIj++oWT8tgQH8pBVmU8PSPLnLxH +zNllxqSFryAgXBmLeVymWu34nEL2MqASDsswDcMiJj0M9w1EHs/seis0PYB/c54g +/Y5Cgk2sRuiRWOr39H0Z49SfxqkMnmbjw0iLlopN4QFBQUqLmitJUoAZiHiwJPJc +4/YLQABjdGmF5lfq8xHflkOa83Wm3H5I5WgWGQ== +-----END CERTIFICATE----- +)___"; + +const TString ServerKey = R"___( +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA4quzmIvZqAc90kPHlmBytC+OyAvoJmCpvOymaaYewnAy5rak +YS1FRstQ4ROgHXDCajw9NvnXXKjv152QoP/J07+42zcjexZAywEn5kZ83ht11ViY +Atjim7u84cDS4OPAh3V7sJMQex870KbQfN6V4O73HE2Bu35vpaVMcuDmUa+N6pnD +6l74eX/Ru9sZtUOjtvzkom8cWm/8XGgCAod1BpF2iycpHfCyixeZGysTNK10/Uye +BTUGIbQg/fTuVVWkb7ZzXw0xtvFq4Vw+Urcw2rvfcezFn4+g1eqRtvY+LnXJmnYZ +4PKsoSKGKWCpIisyN0M9QGU3l+yg0I0wFcm9xzEWcOlNncb1+MmS5CgAI1kU0swi +Rn/GPIOMI+LNDQlRDpSeL8eD7zId/WkMnoWtKTLPkNDZJilGDZl6r+9Q7G1iJCQ4 +71qgv1/sERGhmcpBdaPpBlxQ0JzmwhGl9wrc/drO81l7CNquwZMOdJLEtCz2/bpA +911koei9LUzfgui4cb/oME6ibtV07/baCSZudkbktlFmduZ6GLIVDGrNrFTNAJPN +0fl1tkUrzomxkqB10qXFvm155MFcSHYDaKOxOLJEm2hlCmFKZNq7ySPw9h3S4GMJ +w/cxkntPZm6kXwT5W9Cjtei+EbFrax+hFy+tFaBQ4pFVekcNRHNnD9HSg/kCAwEA +AQKCAgBOWWU5vFVetCoVTOJnQy1CxRGIaj0zTsQ9DluzNv143glqIAWPpXNFti9d +rUfyBTDeQbYzE4bye15z2/3K+L9Nlv6Rn2x/NkDtKpgdC45Lw5gmR3o7ubYSeIEg +U2NQ4siAygYYEa1nsXMeexqjntiVqGP3/35xTZHP4uQa7UwPPixCxCWpFGy0qo7X +bTNkqV2keaOZ1egqBn1nf3f6YdH8lDkyfjXDKJi+ZUjB0FDSK4a9q0cPq7VT3wxs +W8Yp7vFov9r/JvNhNe9ouFa+hp23basdmObycVX5uxvk7xatPn+SCXKGg7tR8zoG +gWfU7LNt0KsSYCooNF2d2L+fOF2FHuXffsHM090bf0GE5J829CN/H6Wa44NI9LDn +wmtTXoHOWLqKL+iPj12HfMsx5zhcdMBY1qo7vIBZfR1mYyeWcuAAApBJWED9Z5tJ +pEpIVBRgKq6Q/Tr66gTmU5FLRCFKlbIjD2sZdMl/Rkp1rskNdjRLw5cDhn2X+MyM +cRdF0YkJjRph/DnPP/z2aQgx3/NfjXpEhWtd7o0Ipkp4zrPqZ6yyTThY2VYgT2Ce +PzBBy1Ib+avIXU1EExkFahaijFALq4h4c60bstoTfoGq12Y2B512FJKtwB8+80Ih +Jlrtr4vAsWzndTxpIIrBSgQxy/F5lDcxPg0J2xFOr9DbGC34pQKCAQEA/JV3hLCV +GhciZPzsZHb7gBfkC1l2dn1y4QdnaTdVO4n2p/l4k5ucvuHjWqrBzEiO/CxcfRAg +bqUqs/IKwJ4TLvQWAk5zwAe4diuy/jV5VY8ruOiFc56QhieYuog0asPkP2LRucVq +skunljIZNoHLUTfs80ggMmEHBuewv2gnrNuWPp1cNbjCstivpeRuUwKmL2bdQfFA +TyQQHWn9eItNhRqX7MNr4fz1QUUSUt7CWYYoDhJC/Oo45320AjQR/Lms2QYvz6iG +CzVmIJxSqC7Oykz6gVcKu+R+RdO8DLRb+fIaA7QgAX8/lxLlJuKVn5F8WfaAzRQ6 +yKmbI9+HFifGDwKCAQEA5byDrObyWTL4BTeRyFlHRYF6cTe2xBv/Kuk4Ec6zHaaV +E39PfTJn2t0BkDvK9FXtF9sjKTmhsjg3mmgMNOz8Ll2/z4df6/u4zjy1dZ309zyh +yDWcP4tW8Hj6kl8PFStdlRXxIBMt/Gpsf4w45q2cNmqdMU4VBgwhycp87fq7EWJX +0zPDJnzOHo/twG+tsl/ZzQsHyxPC54oG3uFKWnT8azJTBgGeAD+huoONGk44TpOj +uV6LQpIS+q+vwRLmSaMeAiEG2N/IbMEU6l1gbz9ohfcASqFIVmxht2AE1GpD0I1o +XlxeoP0uSP9TtSuhSHfXowjaD/IBxzMJRtsA+X5ddwKCAQEAy5JbpaH8Se7jValT +jRUoVnDq5wrPo2gwMpWZDv/9veLP3Un/mFgO2PmOGAEP+Olx9GR8ln9s5EBSTn2B +lQTSSUGIi4tXVynhzbwioyfOBttBTeJ5zFm7+aPoQE6OkI4ZY8ztY2BtQg4fn7n5 +AClUCL2eR+WVrYTt+O67UUlM0NCaIxUIwHOM2EA0MOwOzvCPqByrrv4V6rMSGeLW +21TKwcBROg224YjS0iwtPIU09ppdphmpy9Wqz0hM0InPBXVQjgmidydIAbij+xyC +sfIn0HyCWcQhbpYV/4lLQqIKj0RFGz8NnKdGRSiBb/mmxdin9Inr/V2Uky2/UAZU +BdNAmQKCAQAYZPcacf+D5zyc2TS6sHg38jK9OOxIUKy8sr0Ibwln+ZtU0azwH10V +yWf0F9VKMqGVaeiG5R69XOjSlX/OUufISJ4ofDh8R2NtStb94UL0ydRn/QFVYgde +S4pX5o4kclFilkzfWgoFBov00z/rhr3SrWl5pc/nr3wbAExZvMkGZIns3E85lAET +D7dwOquYCEOJWUV/k96bVXW7TvLlPgzbmSFlvuA3KIqU0ok2JN4nwdedxGNHM1me +ku83sjkP0qlKEpW2i7Stj6cX58hop7QCnaLDSfLzcljB7wk0QQBoccuGUYqez6ON +jsclsrdSiZ81Kah2Dv2PWGUAyBqHY5qrAoIBAQCtjsqk71UkhTi5LB2ism1aGAme +pKMY459g57wmSEWE/+rBWOBY5CZ7FMV1W+P1bXqydZOr6ptBEUjM4hW6YcsKmDtk +g/LriBtDnPkd/QvrrpOZnj8rrtFyOF/wMx2jFwp6Nes+QtRqQk0RXerWDSbsn9vL +xSN/nSB6IoYs5rlC2TRoxCHqEGPSGImi4hhpLhIrArpqTH8+OhuRRVwtMO8Qo2fS +LEbMU1ARtp9TVPVktwsdNOnCeyynJnQiB6fL6QnpQYFY3xyO1MuBSjruTKLqorPM +pqUJavKfAG3qyVCaGn4msSaPDNnG5QH7eCpbuemWgPDmx/br4fdx/U7VH8HE +-----END RSA PRIVATE KEY----- +)___"; + +}; // namespace NYdbSslTestData diff --git a/ydb/services/ydb/ydb_logstore.cpp b/ydb/services/ydb/ydb_logstore.cpp index 2db6baf954..607b4f2508 100644 --- a/ydb/services/ydb/ydb_logstore.cpp +++ b/ydb/services/ydb/ydb_logstore.cpp @@ -2,7 +2,7 @@ #include <ydb/core/grpc_services/service_logstore.h> #include <ydb/core/grpc_services/base/base.h> - + #include <ydb/core/grpc_services/grpc_helper.h> namespace NKikimr { @@ -34,30 +34,30 @@ void TGRpcYdbLogStoreService::DecRequest() { } void TGRpcYdbLogStoreService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - using namespace Ydb; + using namespace Ydb; auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); #ifdef ADD_REQUEST #error ADD_REQUEST macro already defined #endif -#define ADD_REQUEST(NAME, CB) \ - MakeIntrusive<TGRpcRequest<LogStore::NAME##Request, LogStore::NAME##Response, TGRpcYdbLogStoreService>> \ - (this, &Service_, CQ_, [this](NGrpc::IRequestContextBase *ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ActorSystem_->Send(GRpcRequestProxyId_, \ - new TGrpcRequestOperationCall<LogStore::NAME##Request, LogStore::NAME##Response> \ - (ctx, &CB)); \ - }, &Ydb::LogStore::V1::LogStoreService::AsyncService::Request ## NAME, \ +#define ADD_REQUEST(NAME, CB) \ + MakeIntrusive<TGRpcRequest<LogStore::NAME##Request, LogStore::NAME##Response, TGRpcYdbLogStoreService>> \ + (this, &Service_, CQ_, [this](NGrpc::IRequestContextBase *ctx) { \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ActorSystem_->Send(GRpcRequestProxyId_, \ + new TGrpcRequestOperationCall<LogStore::NAME##Request, LogStore::NAME##Response> \ + (ctx, &CB)); \ + }, &Ydb::LogStore::V1::LogStoreService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("logstore", #NAME))->Run(); - ADD_REQUEST(CreateLogStore, DoCreateLogStoreRequest) - ADD_REQUEST(DescribeLogStore, DoDescribeLogStoreRequest) - ADD_REQUEST(DropLogStore, DoDropLogStoreRequest) + ADD_REQUEST(CreateLogStore, DoCreateLogStoreRequest) + ADD_REQUEST(DescribeLogStore, DoDescribeLogStoreRequest) + ADD_REQUEST(DropLogStore, DoDropLogStoreRequest) - ADD_REQUEST(CreateLogTable, DoCreateLogTableRequest) - ADD_REQUEST(DescribeLogTable, DoDescribeLogTableRequest) - ADD_REQUEST(DropLogTable, DoDropLogTableRequest) - ADD_REQUEST(AlterLogTable, DoAlterLogTableRequest) + ADD_REQUEST(CreateLogTable, DoCreateLogTableRequest) + ADD_REQUEST(DescribeLogTable, DoDescribeLogTableRequest) + ADD_REQUEST(DropLogTable, DoDropLogTableRequest) + ADD_REQUEST(AlterLogTable, DoAlterLogTableRequest) #undef ADD_REQUEST } diff --git a/ydb/services/ydb/ydb_operation.cpp b/ydb/services/ydb/ydb_operation.cpp index ff2e95fbbb..602980027c 100644 --- a/ydb/services/ydb/ydb_operation.cpp +++ b/ydb/services/ydb/ydb_operation.cpp @@ -1,52 +1,52 @@ #include "ydb_operation.h" - + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + TGRpcOperationService::TGRpcOperationService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ } - + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ } + void TGRpcOperationService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; + CQ_ = cq; SetupIncomingRequests(std::move(logger)); -} - +} + void TGRpcOperationService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter *limiter) { - Limiter_ = limiter; -} - + Limiter_ = limiter; +} + bool TGRpcOperationService::IncRequest() { - return Limiter_->Inc(); -} - + return Limiter_->Inc(); +} + void TGRpcOperationService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + void TGRpcOperationService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ MakeIntrusive<TGRpcRequest<Ydb::Operations::IN, Ydb::Operations::OUT, TGRpcOperationService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Ydb::Operation::V1::OperationService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("operation", #NAME))->Run(); - - ADD_REQUEST(GetOperation, GetOperationRequest, GetOperationResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvGetOperationRequest(ctx)); - }) + + ADD_REQUEST(GetOperation, GetOperationRequest, GetOperationResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvGetOperationRequest(ctx)); + }) ADD_REQUEST(CancelOperation, CancelOperationRequest, CancelOperationResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvCancelOperationRequest(ctx)); }) @@ -56,8 +56,8 @@ void TGRpcOperationService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { ADD_REQUEST(ListOperations, ListOperationsRequest, ListOperationsResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvListOperationsRequest(ctx)); }) -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_operation.h b/ydb/services/ydb/ydb_operation.h index 94d53dbd99..9f7e8646b1 100644 --- a/ydb/services/ydb/ydb_operation.h +++ b/ydb/services/ydb/ydb_operation.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <library/cpp/grpc/server/grpc_server.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + class TGRpcOperationService : public NGrpc::TGrpcServiceBase<Ydb::Operation::V1::OperationService> -{ -public: +{ +public: TGRpcOperationService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); - + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: + + bool IncRequest(); + void DecRequest(); +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_scheme.cpp b/ydb/services/ydb/ydb_scheme.cpp index b271a80c60..ac339f54ab 100644 --- a/ydb/services/ydb/ydb_scheme.cpp +++ b/ydb/services/ydb/ydb_scheme.cpp @@ -1,61 +1,61 @@ #include "ydb_scheme.h" - + #include <ydb/core/grpc_services/grpc_helper.h> - + #include <ydb/core/grpc_services/service_scheme.h> #include <ydb/core/grpc_services/base/base.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + TGRpcYdbSchemeService::TGRpcYdbSchemeService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ } - + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ } + void TGRpcYdbSchemeService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; + CQ_ = cq; SetupIncomingRequests(std::move(logger)); -} - +} + void TGRpcYdbSchemeService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - + Limiter_ = limiter; +} + bool TGRpcYdbSchemeService::IncRequest() { - return Limiter_->Inc(); -} - + return Limiter_->Inc(); +} + void TGRpcYdbSchemeService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + void TGRpcYdbSchemeService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, CB) \ - MakeIntrusive<TGRpcRequest<Ydb::Scheme::NAME##Request, Ydb::Scheme::NAME##Response, TGRpcYdbSchemeService>> \ - (this, &Service_, CQ_, \ - [this](NGrpc::IRequestContextBase *ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ActorSystem_->Send(GRpcRequestProxyId_, \ - new TGrpcRequestOperationCall<Ydb::Scheme::NAME##Request, Ydb::Scheme::NAME##Response> \ - (ctx, &CB, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); \ - }, &Ydb::Scheme::V1::SchemeService::AsyncService::Request ## NAME, \ - #NAME, logger, getCounterBlock("scheme", #NAME))->Run(); - - ADD_REQUEST(MakeDirectory, DoMakeDirectoryRequest) - ADD_REQUEST(RemoveDirectory, DoRemoveDirectoryRequest) - ADD_REQUEST(ListDirectory, DoListDirectoryRequest) - ADD_REQUEST(DescribePath, DoDescribePathRequest) - ADD_REQUEST(ModifyPermissions, DoModifyPermissionsRequest) -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, CB) \ + MakeIntrusive<TGRpcRequest<Ydb::Scheme::NAME##Request, Ydb::Scheme::NAME##Response, TGRpcYdbSchemeService>> \ + (this, &Service_, CQ_, \ + [this](NGrpc::IRequestContextBase *ctx) { \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ActorSystem_->Send(GRpcRequestProxyId_, \ + new TGrpcRequestOperationCall<Ydb::Scheme::NAME##Request, Ydb::Scheme::NAME##Response> \ + (ctx, &CB, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); \ + }, &Ydb::Scheme::V1::SchemeService::AsyncService::Request ## NAME, \ + #NAME, logger, getCounterBlock("scheme", #NAME))->Run(); + + ADD_REQUEST(MakeDirectory, DoMakeDirectoryRequest) + ADD_REQUEST(RemoveDirectory, DoRemoveDirectoryRequest) + ADD_REQUEST(ListDirectory, DoListDirectoryRequest) + ADD_REQUEST(DescribePath, DoDescribePathRequest) + ADD_REQUEST(ModifyPermissions, DoModifyPermissionsRequest) +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_scheme.h b/ydb/services/ydb/ydb_scheme.h index 9a3fac0b2c..a510e5f592 100644 --- a/ydb/services/ydb/ydb_scheme.h +++ b/ydb/services/ydb/ydb_scheme.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/ydb_scheme_v1.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + class TGRpcYdbSchemeService : public NGrpc::TGrpcServiceBase<Ydb::Scheme::V1::SchemeService> -{ -public: +{ +public: TGRpcYdbSchemeService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); - + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: + + bool IncRequest(); + void DecRequest(); +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_scripting.cpp b/ydb/services/ydb/ydb_scripting.cpp index b15a6ec174..3596b17fd9 100644 --- a/ydb/services/ydb/ydb_scripting.cpp +++ b/ydb/services/ydb/ydb_scripting.cpp @@ -33,12 +33,12 @@ void TGRpcYdbScriptingService::DecRequest() { } void TGRpcYdbScriptingService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - using Ydb::Scripting::ExecuteYqlRequest; - using Ydb::Scripting::ExecuteYqlResponse; - using Ydb::Scripting::ExecuteYqlPartialResponse; - using Ydb::Scripting::ExplainYqlRequest; - using Ydb::Scripting::ExplainYqlResponse; - + using Ydb::Scripting::ExecuteYqlRequest; + using Ydb::Scripting::ExecuteYqlResponse; + using Ydb::Scripting::ExecuteYqlPartialResponse; + using Ydb::Scripting::ExplainYqlRequest; + using Ydb::Scripting::ExplainYqlResponse; + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); #ifdef ADD_REQUEST @@ -53,21 +53,21 @@ void TGRpcYdbScriptingService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { #NAME, logger, getCounterBlock("scripting", #NAME))->Run(); ADD_REQUEST(ExecuteYql, ExecuteYqlRequest, ExecuteYqlResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, - new TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse> - (ctx, &DoExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Ru, nullptr})); + ActorSystem_->Send(GRpcRequestProxyId_, + new TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse> + (ctx, &DoExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Ru, nullptr})); }) ADD_REQUEST(StreamExecuteYql, ExecuteYqlRequest, ExecuteYqlPartialResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, - new TGrpcRequestNoOperationCall<ExecuteYqlRequest, ExecuteYqlPartialResponse> - (ctx, &DoStreamExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); + ActorSystem_->Send(GRpcRequestProxyId_, + new TGrpcRequestNoOperationCall<ExecuteYqlRequest, ExecuteYqlPartialResponse> + (ctx, &DoStreamExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); }) ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, - new TGrpcRequestOperationCall<ExplainYqlRequest, ExplainYqlResponse> - (ctx, &DoExplainYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); + ActorSystem_->Send(GRpcRequestProxyId_, + new TGrpcRequestOperationCall<ExplainYqlRequest, ExplainYqlResponse> + (ctx, &DoExplainYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); }) #undef ADD_REQUEST } diff --git a/ydb/services/ydb/ydb_scripting_ut.cpp b/ydb/services/ydb/ydb_scripting_ut.cpp index 0905c18e26..58dc29f3f9 100644 --- a/ydb/services/ydb/ydb_scripting_ut.cpp +++ b/ydb/services/ydb/ydb_scripting_ut.cpp @@ -10,59 +10,59 @@ using namespace NYdb; Y_UNIT_TEST_SUITE(YdbScripting) { - void DoBasicTest(bool v1) { + void DoBasicTest(bool v1) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - auto client = NYdb::NScripting::TScriptingClient(connection); - - const TString v1Prefix = R"( - --!syntax_v1 - )"; - - const TString sql = R"( - CREATE TABLE `/Root/TestTable` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - COMMIT; - - REPLACE INTO `/Root/TestTable` (Key, Value) VALUES - (1, "One"), - (2, "Two"); - COMMIT; - - SELECT * FROM `/Root/TestTable`; - )"; - - auto result = client.ExecuteYqlScript((v1 ? v1Prefix : TString()) + sql).GetValueSync(); + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + auto client = NYdb::NScripting::TScriptingClient(connection); + + const TString v1Prefix = R"( + --!syntax_v1 + )"; + + const TString sql = R"( + CREATE TABLE `/Root/TestTable` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + COMMIT; + + REPLACE INTO `/Root/TestTable` (Key, Value) VALUES + (1, "One"), + (2, "Two"); + COMMIT; + + SELECT * FROM `/Root/TestTable`; + )"; + + auto result = client.ExecuteYqlScript((v1 ? v1Prefix : TString()) + sql).GetValueSync(); result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(result.IsSuccess()); - + UNIT_ASSERT(result.IsSuccess()); + TVector<TResultSet> resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + TResultSetParser rsParser(resultSets[0]); - UNIT_ASSERT(rsParser.TryNextRow()); - UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetOptionalUint64(), 1ul); - UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetOptionalString(), "One"); - UNIT_ASSERT(rsParser.TryNextRow()); - UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetOptionalUint64(), 2ul); - UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetOptionalString(), "Two"); - } - - Y_UNIT_TEST(BasicV0) { - DoBasicTest(false); - } - - Y_UNIT_TEST(BasicV1) { - DoBasicTest(true); - } - + UNIT_ASSERT(rsParser.TryNextRow()); + UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetOptionalUint64(), 1ul); + UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetOptionalString(), "One"); + UNIT_ASSERT(rsParser.TryNextRow()); + UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetOptionalUint64(), 2ul); + UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetOptionalString(), "Two"); + } + + Y_UNIT_TEST(BasicV0) { + DoBasicTest(false); + } + + Y_UNIT_TEST(BasicV1) { + DoBasicTest(true); + } + Y_UNIT_TEST(MultiResults) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp index 1afea837c5..b40e22fc42 100644 --- a/ydb/services/ydb/ydb_stats_ut.cpp +++ b/ydb/services/ydb/ydb_stats_ut.cpp @@ -29,7 +29,7 @@ struct TStatCounters { i64 ReadySessionsRatio = 0; ui64 FakeSessions = 0; ui64 CacheMiss = 0; - ui64 RetryOperationDueAborted = 0; + ui64 RetryOperationDueAborted = 0; }; class TMetricEncoder: public NMonitoring::IMetricEncoder { @@ -45,7 +45,7 @@ public: READYSESSIONSRATIO, FAKESESSIONS, CACHEMISS, - RETRYOPERATIONDUEABORTED, + RETRYOPERATIONDUEABORTED, DISCOVERYDUEPESSIMIZATION, DISCOVERYDUEEXPIRATION, @@ -77,34 +77,34 @@ public: return; } - if (value == "Discovery/TooManyBadEndpoints") { + if (value == "Discovery/TooManyBadEndpoints") { State = ECounterType::DISCOVERYDUEPESSIMIZATION; - } else if (value == "Discovery/Regular") { + } else if (value == "Discovery/Regular") { State = ECounterType::DISCOVERYDUEEXPIRATION; - } else if (value == "Request/FailedDiscoveryQueueOverflow") { + } else if (value == "Request/FailedDiscoveryQueueOverflow") { State = ECounterType::REQUESTFAILDUEQUEUEOVERFLOW; - } else if (value == "Request/FailedNoEndpoint") { + } else if (value == "Request/FailedNoEndpoint") { State = ECounterType::REQUESTFAILDUENOENDPOINT; - } else if (value == "Request/FailedTransportError") { + } else if (value == "Request/FailedTransportError") { State = ECounterType::REQUESTFAILDUETRANSPORTERROR; - } else if (value == "Discovery/FailedTransportError") { + } else if (value == "Discovery/FailedTransportError") { State = ECounterType::DISCOVERYFAILDUETRANSPORTERROR; - } else if (value == "Endpoints/Total") { + } else if (value == "Endpoints/Total") { State = ECounterType::ENDPOINTCOUNT; - } else if (value == "Endpoints/BadRatio") { + } else if (value == "Endpoints/BadRatio") { State = ECounterType::ENDPOINTPESSIMIZATIONRATIO; - } else if (value == "Endpoints/Good") { + } else if (value == "Endpoints/Good") { State = ECounterType::ENDPOINTACTIVE; - } else if (value == "Sessions/InUse") { + } else if (value == "Sessions/InUse") { State = ECounterType::ACTIVESESSIONSRATIO; } else if (value == "ready sessions ratio") { State = ECounterType::READYSESSIONSRATIO; - } else if (value == "Sessions/SessionsLimitExceeded") { + } else if (value == "Sessions/SessionsLimitExceeded") { State = ECounterType::FAKESESSIONS; - } else if (value == "Request/ClientQueryCacheMiss") { + } else if (value == "Request/ClientQueryCacheMiss") { State = ECounterType::CACHEMISS; - } else if (value == "RetryOperation/Aborted") { - State = ECounterType::RETRYOPERATIONDUEABORTED; + } else if (value == "RetryOperation/Aborted") { + State = ECounterType::RETRYOPERATIONDUEABORTED; } else if (value == "request latency") { State = ECounterType::REQUESTLATENCY; } else { @@ -138,7 +138,7 @@ public: case ECounterType::FAKESESSIONS: Counters.FakeSessions = value; break; case ECounterType::CACHEMISS: Counters.CacheMiss = value; break; - case ECounterType::RETRYOPERATIONDUEABORTED: Counters.RetryOperationDueAborted = value; break; + case ECounterType::RETRYOPERATIONDUEABORTED: Counters.RetryOperationDueAborted = value; break; default: return; } } @@ -204,11 +204,11 @@ private: TCountersExtractExtension::TCountersExtractExtension(const TParams& params, IApi* api) : MetricRegistry_(new NMonitoring::TMetricRegistry()) - , Api_(api) -{ + , Api_(api) +{ api->SetMetricRegistry(MetricRegistry_.get()); - params.Extractor->Register(this); -} + params.Extractor->Register(this); +} using namespace NYdb::NTable; @@ -306,7 +306,7 @@ Y_UNIT_TEST_SUITE(ClientStatsCollector) { }, retrySelectSettings).IsSuccess() == false); TStatCounters counters = extractor.Extract(); - UNIT_ASSERT_VALUES_EQUAL(counters.RetryOperationDueAborted, retriesCount); + UNIT_ASSERT_VALUES_EQUAL(counters.RetryOperationDueAborted, retriesCount); UNIT_ASSERT(client.RetryOperation([&upsertOperation, settings, txSettings](TSession session){ auto beginResult = session.BeginTransaction(TTxSettings::SerializableRW(), txSettings).GetValueSync(); @@ -328,58 +328,58 @@ Y_UNIT_TEST_SUITE(ClientStatsCollector) { }, retrySelectSettings).GetValueSync().IsSuccess() == false); counters = extractor.Extract(); - // cumulative counter - UNIT_ASSERT_VALUES_EQUAL(counters.RetryOperationDueAborted, retriesCount * 2); + // cumulative counter + UNIT_ASSERT_VALUES_EQUAL(counters.RetryOperationDueAborted, retriesCount * 2); driver.Stop(true); } - + Y_UNIT_TEST(ExternalMetricRegistryByRawPtr) { NMonitoring::TMetricRegistry sensorsRegistry; - NYdb::TKikimrWithGrpcAndRootSchema server; - - auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); - NYdb::TDriver driver(NYdb::TDriverConfig().SetEndpoint(endpoint)); - + NYdb::TKikimrWithGrpcAndRootSchema server; + + auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); + NYdb::TDriver driver(NYdb::TDriverConfig().SetEndpoint(endpoint)); + NSolomonStatExtension::AddMetricRegistry(driver, &sensorsRegistry); - { - NYdb::NTable::TTableClient client(driver); - - auto createSessionResult = client.GetSession().GetValueSync(); - auto session = createSessionResult.GetSession(); - UNIT_ASSERT(SimpleSelect(session, "SELECT 1;").IsSuccess()); - - TStringStream out; + { + NYdb::NTable::TTableClient client(driver); + + auto createSessionResult = client.GetSession().GetValueSync(); + auto session = createSessionResult.GetSession(); + UNIT_ASSERT(SimpleSelect(session, "SELECT 1;").IsSuccess()); + + TStringStream out; NMonitoring::IMetricEncoderPtr encoder = NMonitoring::EncoderJson(&out); - sensorsRegistry.Accept(TInstant::Zero(), encoder.Get()); - } - driver.Stop(true); - } - - template<template<typename...> class TPointer> + sensorsRegistry.Accept(TInstant::Zero(), encoder.Get()); + } + driver.Stop(true); + } + + template<template<typename...> class TPointer> void TestExternalMetricRegistry() { TPointer<NMonitoring::TMetricRegistry> sensorsRegistry(new NMonitoring::TMetricRegistry()); - NYdb::TKikimrWithGrpcAndRootSchema server; - - auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); - NYdb::TDriver driver(NYdb::TDriverConfig().SetEndpoint(endpoint)); - + NYdb::TKikimrWithGrpcAndRootSchema server; + + auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); + NYdb::TDriver driver(NYdb::TDriverConfig().SetEndpoint(endpoint)); + NSolomonStatExtension::AddMetricRegistry(driver, sensorsRegistry); - { - NYdb::NTable::TTableClient client(driver); - - auto createSessionResult = client.GetSession().GetValueSync(); - auto session = createSessionResult.GetSession(); - UNIT_ASSERT(SimpleSelect(session, "SELECT 1;").IsSuccess()); - - TStringStream out; + { + NYdb::NTable::TTableClient client(driver); + + auto createSessionResult = client.GetSession().GetValueSync(); + auto session = createSessionResult.GetSession(); + UNIT_ASSERT(SimpleSelect(session, "SELECT 1;").IsSuccess()); + + TStringStream out; NMonitoring::IMetricEncoderPtr encoder = NMonitoring::EncoderJson(&out); - sensorsRegistry->Accept(TInstant::Zero(), encoder.Get()); - } - driver.Stop(true); - } - + sensorsRegistry->Accept(TInstant::Zero(), encoder.Get()); + } + driver.Stop(true); + } + Y_UNIT_TEST(ExternalMetricRegistryStdSharedPtr) { TestExternalMetricRegistry<std::shared_ptr>(); - } + } } diff --git a/ydb/services/ydb/ydb_table.cpp b/ydb/services/ydb/ydb_table.cpp index 94f5d58a26..7189614b8e 100644 --- a/ydb/services/ydb/ydb_table.cpp +++ b/ydb/services/ydb/ydb_table.cpp @@ -1,84 +1,84 @@ #include "ydb_table.h" - + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + TGRpcYdbTableService::TGRpcYdbTableService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ } - + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ } + void TGRpcYdbTableService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; + CQ_ = cq; SetupIncomingRequests(std::move(logger)); -} - +} + void TGRpcYdbTableService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - + Limiter_ = limiter; +} + bool TGRpcYdbTableService::IncRequest() { - return Limiter_->Inc(); -} - + return Limiter_->Inc(); +} + void TGRpcYdbTableService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + void TGRpcYdbTableService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ MakeIntrusive<TGRpcRequest<Ydb::Table::IN, Ydb::Table::OUT, TGRpcYdbTableService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Ydb::Table::V1::TableService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("table", #NAME))->Run(); - -#define ADD_BYTES_REQUEST(NAME, IN, OUT, ACTION) \ + +#define ADD_BYTES_REQUEST(NAME, IN, OUT, ACTION) \ MakeIntrusive<TGRpcRequest<Ydb::Table::IN, Ydb::Table::OUT, TGRpcYdbTableService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ }, &Ydb::Table::V1::TableService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("table", #NAME))->Run(); - - ADD_REQUEST(CreateSession, CreateSessionRequest, CreateSessionResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvCreateSessionRequest(ctx)); - }) - ADD_REQUEST(DeleteSession, DeleteSessionRequest, DeleteSessionResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDeleteSessionRequest(ctx)); - }) - ADD_REQUEST(KeepAlive, KeepAliveRequest, KeepAliveResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvKeepAliveRequest(ctx)); - }) - ADD_REQUEST(AlterTable, AlterTableRequest, AlterTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvAlterTableRequest(ctx)); - }) - ADD_REQUEST(CreateTable, CreateTableRequest, CreateTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvCreateTableRequest(ctx)); - }) - ADD_REQUEST(DropTable, DropTableRequest, DropTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDropTableRequest(ctx)); - }) - ADD_BYTES_REQUEST(StreamReadTable, ReadTableRequest, ReadTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvReadTableRequest(ctx)); - }) - ADD_REQUEST(DescribeTable, DescribeTableRequest, DescribeTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDescribeTableRequest(ctx)); - }) - ADD_REQUEST(CopyTable, CopyTableRequest, CopyTableResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvCopyTableRequest(ctx)); - }) + + ADD_REQUEST(CreateSession, CreateSessionRequest, CreateSessionResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvCreateSessionRequest(ctx)); + }) + ADD_REQUEST(DeleteSession, DeleteSessionRequest, DeleteSessionResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDeleteSessionRequest(ctx)); + }) + ADD_REQUEST(KeepAlive, KeepAliveRequest, KeepAliveResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvKeepAliveRequest(ctx)); + }) + ADD_REQUEST(AlterTable, AlterTableRequest, AlterTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvAlterTableRequest(ctx)); + }) + ADD_REQUEST(CreateTable, CreateTableRequest, CreateTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvCreateTableRequest(ctx)); + }) + ADD_REQUEST(DropTable, DropTableRequest, DropTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDropTableRequest(ctx)); + }) + ADD_BYTES_REQUEST(StreamReadTable, ReadTableRequest, ReadTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvReadTableRequest(ctx)); + }) + ADD_REQUEST(DescribeTable, DescribeTableRequest, DescribeTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDescribeTableRequest(ctx)); + }) + ADD_REQUEST(CopyTable, CopyTableRequest, CopyTableResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvCopyTableRequest(ctx)); + }) ADD_REQUEST(CopyTables, CopyTablesRequest, CopyTablesResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvCopyTablesRequest(ctx)); }) @@ -115,9 +115,9 @@ void TGRpcYdbTableService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { ADD_REQUEST(StreamExecuteScanQuery, ExecuteScanQueryRequest, ExecuteScanQueryPartialResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvStreamExecuteScanQueryRequest(ctx)); }) - -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr + +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_table.h b/ydb/services/ydb/ydb_table.h index 9fd2e2d8c3..77756bf023 100644 --- a/ydb/services/ydb/ydb_table.h +++ b/ydb/services/ydb/ydb_table.h @@ -1,35 +1,35 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> - - -namespace NKikimr { -namespace NGRpcService { - + + +namespace NKikimr { +namespace NGRpcService { + class TGRpcYdbTableService : public NGrpc::TGrpcServiceBase<Ydb::Table::V1::TableService> -{ -public: +{ +public: TGRpcYdbTableService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); - + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: + + bool IncRequest(); + void DecRequest(); +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index 6704daafa2..4861c06b92 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -1,5 +1,5 @@ #include "ydb_common_ut.h" - + #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> @@ -14,26 +14,26 @@ #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> - + #include <util/thread/factory.h> - + using namespace NYdb; using namespace NYdb::NTable; - -TSession CreateSession(TDriver driver, const TString& token = "", const TString& discoveryEndpoint = "") { - NYdb::NTable::TClientSettings settings; - if (token) + +TSession CreateSession(TDriver driver, const TString& token = "", const TString& discoveryEndpoint = "") { + NYdb::NTable::TClientSettings settings; + if (token) settings.AuthToken(token); - if (discoveryEndpoint) - settings.DiscoveryEndpoint(discoveryEndpoint); - NYdb::NTable::TTableClient client(driver, settings); - auto session = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(session.IsTransportError(), false); - return session.GetSession(); -} - + if (discoveryEndpoint) + settings.DiscoveryEndpoint(discoveryEndpoint); + NYdb::NTable::TTableClient client(driver, settings); + auto session = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(session.IsTransportError(), false); + return session.GetSession(); +} + void EnsureTablePartitions(NYdb::NTable::TTableClient& client, TString table, ui32 expectedPartitions) { auto session = client.CreateSession().ExtractValueSync().GetSession(); @@ -46,945 +46,945 @@ void EnsureTablePartitions(NYdb::NTable::TTableClient& client, TString table, ui UNIT_ASSERT_VALUES_EQUAL(description.GetTableDescription().GetPartitionStats().size(), expectedPartitions); } -static void MultiTenantSDK(bool asyncDiscovery) { +static void MultiTenantSDK(bool asyncDiscovery) { TKikimrWithGrpcAndRootSchemaWithAuthAndSsl server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetAuthToken("badguy@builtin") - .UseSecureConnection(NYdbSslTestData::CaCrt) - .SetEndpoint(location) - .SetDiscoveryMode(asyncDiscovery ? EDiscoveryMode::Async : EDiscoveryMode::Sync)); - - - NYdb::NTable::TClientSettings settings; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetAuthToken("badguy@builtin") + .UseSecureConnection(NYdbSslTestData::CaCrt) + .SetEndpoint(location) + .SetDiscoveryMode(asyncDiscovery ? EDiscoveryMode::Async : EDiscoveryMode::Sync)); + + + NYdb::NTable::TClientSettings settings; settings.AuthToken("root@builtin"); - - NYdb::NTable::TTableClient clientgood(driver, settings); - NYdb::NTable::TTableClient clientbad(driver); -//TODO: No discovery in ut -/* - NYdb::NTable::TClientSettings settings2; + + NYdb::NTable::TTableClient clientgood(driver, settings); + NYdb::NTable::TTableClient clientbad(driver); +//TODO: No discovery in ut +/* + NYdb::NTable::TClientSettings settings2; settings2.AuthToken("root@builtin"); - settings2.Database_ = "/balabla"; - NYdb::NTable::TTableClient clientbad2(driver, settings2); -*/ - const TString sql = R"__( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - );)__"; - - clientbad.CreateSession().Apply([sql](const TAsyncCreateSessionResult& future) { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - auto session = sessionValue.GetSession(); - session.ExecuteSchemeQuery(sql).Apply([](const TAsyncStatus& future) { - const auto& status = future.GetValue(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::UNAUTHORIZED); - }).Wait(); - }).Wait(); - - - clientgood.CreateSession().Apply([sql](const TAsyncCreateSessionResult& future) { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - auto session = sessionValue.GetSession(); - session.ExecuteSchemeQuery(sql).Apply([](const TAsyncStatus& future) { - const auto& status = future.GetValue(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - }).Wait(); - }).Wait(); -/* - clientbad2.CreateSession().Subscribe([sql](const TAsyncCreateSessionResult& future) { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::CLIENT_DISCOVERY_FAILED); - UNIT_ASSERT_EQUAL(sessionValue.GetIssues().ToString(), "<main>: Error: Endpoint list is empty for database /balabla"); - UNIT_ASSERT(sessionValue.IsTransportError()); - }); -*/ + settings2.Database_ = "/balabla"; + NYdb::NTable::TTableClient clientbad2(driver, settings2); +*/ + const TString sql = R"__( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + );)__"; + + clientbad.CreateSession().Apply([sql](const TAsyncCreateSessionResult& future) { + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + auto session = sessionValue.GetSession(); + session.ExecuteSchemeQuery(sql).Apply([](const TAsyncStatus& future) { + const auto& status = future.GetValue(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::UNAUTHORIZED); + }).Wait(); + }).Wait(); + + + clientgood.CreateSession().Apply([sql](const TAsyncCreateSessionResult& future) { + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + auto session = sessionValue.GetSession(); + session.ExecuteSchemeQuery(sql).Apply([](const TAsyncStatus& future) { + const auto& status = future.GetValue(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + }).Wait(); + }).Wait(); +/* + clientbad2.CreateSession().Subscribe([sql](const TAsyncCreateSessionResult& future) { + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::CLIENT_DISCOVERY_FAILED); + UNIT_ASSERT_EQUAL(sessionValue.GetIssues().ToString(), "<main>: Error: Endpoint list is empty for database /balabla"); + UNIT_ASSERT(sessionValue.IsTransportError()); + }); +*/ driver.Stop(true); -} - +} + Y_UNIT_TEST_SUITE(YdbYqlClient) { - Y_UNIT_TEST(TestYqlWrongTable) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + Y_UNIT_TEST(TestYqlWrongTable) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Json, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Yson, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - } - - Y_UNIT_TEST(TestYqlIssues) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + .SetEndpoint(location)); + auto session = CreateSession(connection); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Json, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + } + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Yson, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + } + } + + Y_UNIT_TEST(TestYqlIssues) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - auto result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) - VALUES("foo", "bar"); + .SetEndpoint(location)); + auto session = CreateSession(connection); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + auto result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) + VALUES("foo", "bar"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - auto ref = R"___(<main>: Error: Type annotation, code: 1030 - <main>:2:25: Error: At function: KiWriteTable! - <main>:2:43: Error: Failed to convert type: Struct<'Key':String,'Value':String> to Struct<'Key':Uint32?,'Value':String?> + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + auto ref = R"___(<main>: Error: Type annotation, code: 1030 + <main>:2:25: Error: At function: KiWriteTable! + <main>:2:43: Error: Failed to convert type: Struct<'Key':String,'Value':String> to Struct<'Key':Uint32?,'Value':String?> <main>:2:43: Error: Failed to convert input columns types to scheme types, code: 2031 -)___"; - UNIT_ASSERT_EQUAL(result.GetIssues().Size(), 1); - UNIT_ASSERT_NO_DIFF(result.GetIssues().ToString(), ref); - } - - Y_UNIT_TEST(TestYqlSessionClosed) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - +)___"; + UNIT_ASSERT_EQUAL(result.GetIssues().Size(), 1); + UNIT_ASSERT_NO_DIFF(result.GetIssues().ToString(), ref); + } + + Y_UNIT_TEST(TestYqlSessionClosed) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - auto status = session.Close().ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - + .SetEndpoint(location)); + auto session = CreateSession(connection); + auto status = session.Close().ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + auto result = session.ExecuteDataQuery("SELECT 42;", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::BAD_SESSION); - } - - Y_UNIT_TEST(DiscoveryLocationOverride) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint("wrongLocation")); - auto session = CreateSession(connection, "", location); - auto status = session.Close().ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - - Y_UNIT_TEST(TestSessionPool) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - const TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::BAD_SESSION); + } + + Y_UNIT_TEST(DiscoveryLocationOverride) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint("wrongLocation")); + auto session = CreateSession(connection, "", location); + auto status = session.Close().ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + + Y_UNIT_TEST(TestSessionPool) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + const TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(driver); - int count = 10; - - THashSet<TString> sids; - while (count--) { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - auto session = sessionResponse.GetSession(); - sids.insert(session.GetId()); - auto result = session.ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - + int count = 10; + + THashSet<TString> sids; + while (count--) { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + auto session = sessionResponse.GetSession(); + sids.insert(session.GetId()); + auto result = session.ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetEndpoint(), location); - } - // All requests used one session + UNIT_ASSERT_VALUES_EQUAL(result.GetEndpoint(), location); + } + // All requests used one session UNIT_ASSERT_VALUES_EQUAL(sids.size(), 1); - // No more session captured by client - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - } - - Y_UNIT_TEST(TestMultipleSessions) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - NYdb::NTable::TTableClient client(driver); - int count = 10; - - TVector<TSession> sids; - TVector<TAsyncDataQueryResult> results; - while (count--) { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - auto session = sessionResponse.GetSession(); - sids.push_back(session); - results.push_back(session.ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); - } - + // No more session captured by client + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + } + + Y_UNIT_TEST(TestMultipleSessions) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + NYdb::NTable::TTableClient client(driver); + int count = 10; + + TVector<TSession> sids; + TVector<TAsyncDataQueryResult> results; + while (count--) { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + auto session = sessionResponse.GetSession(); + sids.push_back(session); + results.push_back(session.ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); + } + NThreading::WaitExceptionOrAll(results).Wait(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 10); - - for (auto& result : results) { - UNIT_ASSERT_EQUAL(result.GetValue().GetStatus(), EStatus::SUCCESS); - } - sids.clear(); - results.clear(); - - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - } - - Y_UNIT_TEST(TestActiveSessionCountAfterBadSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - NYdb::NTable::TTableClient client(driver); - int count = 10; - - TVector<TSession> sids; - TVector<TAsyncDataQueryResult> results; - while (count--) { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - auto session = sessionResponse.GetSession(); - sids.push_back(session); - if (count == 0) { - // Force BAD session server response for ExecuteDataQuery - UNIT_ASSERT_EQUAL(session.Close().GetValueSync().GetStatus(), EStatus::SUCCESS); - results.push_back(session.ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); - } else { - results.push_back(session.ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); - } - } - + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 10); + + for (auto& result : results) { + UNIT_ASSERT_EQUAL(result.GetValue().GetStatus(), EStatus::SUCCESS); + } + sids.clear(); + results.clear(); + + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + } + + Y_UNIT_TEST(TestActiveSessionCountAfterBadSession) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + NYdb::NTable::TTableClient client(driver); + int count = 10; + + TVector<TSession> sids; + TVector<TAsyncDataQueryResult> results; + while (count--) { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + auto session = sessionResponse.GetSession(); + sids.push_back(session); + if (count == 0) { + // Force BAD session server response for ExecuteDataQuery + UNIT_ASSERT_EQUAL(session.Close().GetValueSync().GetStatus(), EStatus::SUCCESS); + results.push_back(session.ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); + } else { + results.push_back(session.ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); + } + } + NThreading::WaitExceptionOrAll(results).Wait(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 10); - - for (size_t i = 0; i < results.size(); i++) { - if (i == 9) { - UNIT_ASSERT_EQUAL(results[i].GetValue().GetStatus(), EStatus::BAD_SESSION); - } else { - UNIT_ASSERT_EQUAL(results[i].GetValue().GetStatus(), EStatus::SUCCESS); - } - } - sids.clear(); - results.clear(); - - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - } - - Y_UNIT_TEST(TestActiveSessionCountAfterTransportError) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - NYdb::NTable::TTableClient client(driver); - int count = 100; - - { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT(sessionResponse.IsSuccess()); - auto session = sessionResponse.GetSession(); - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - } - - while (count--) { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - auto session = sessionResponse.GetSession(); - - // Assume 10us is too small to execute query and get response - auto res = session.ExecuteDataQuery("SELECT COUNT(*) FROM [Root/Test];", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::MicroSeconds(10))).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - } - - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - driver.Stop(true); - } - - Y_UNIT_TEST(MultiThreadSync) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - NYdb::NTable::TTableClient client(driver); - const int nThreads = 10; - const int nRequests = 1000; - auto job = [client]() mutable { - for (int i = 0; i < nRequests; i++) { - auto sessionResponse = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.GetStatus(), EStatus::SUCCESS); - } - }; + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 10); + + for (size_t i = 0; i < results.size(); i++) { + if (i == 9) { + UNIT_ASSERT_EQUAL(results[i].GetValue().GetStatus(), EStatus::BAD_SESSION); + } else { + UNIT_ASSERT_EQUAL(results[i].GetValue().GetStatus(), EStatus::SUCCESS); + } + } + sids.clear(); + results.clear(); + + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + } + + Y_UNIT_TEST(TestActiveSessionCountAfterTransportError) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + NYdb::NTable::TTableClient client(driver); + int count = 100; + + { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT(sessionResponse.IsSuccess()); + auto session = sessionResponse.GetSession(); + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + } + + while (count--) { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + auto session = sessionResponse.GetSession(); + + // Assume 10us is too small to execute query and get response + auto res = session.ExecuteDataQuery("SELECT COUNT(*) FROM [Root/Test];", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::MicroSeconds(10))).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + } + + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + driver.Stop(true); + } + + Y_UNIT_TEST(MultiThreadSync) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + NYdb::NTable::TTableClient client(driver); + const int nThreads = 10; + const int nRequests = 1000; + auto job = [client]() mutable { + for (int i = 0; i < nRequests; i++) { + auto sessionResponse = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.GetStatus(), EStatus::SUCCESS); + } + }; IThreadFactory* pool = SystemThreadFactory(); - + TVector<TAutoPtr<IThreadFactory::IThread>> threads; - threads.resize(nThreads); - for (int i = 0; i < nThreads; i++) { - threads[i] = pool->Run(job); - } - for (int i = 0; i < nThreads; i++) { - threads[i]->Join(); - } - UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 0); - driver.Stop(true); - } - - Y_UNIT_TEST(MultiThreadSessionPoolLimitSync) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - const int maxActiveSessions = 45; - NYdb::NTable::TTableClient client(driver, - TClientSettings() - .SessionPoolSettings( - TSessionPoolSettings().MaxActiveSessions(maxActiveSessions))); - - constexpr int nThreads = 100; - NYdb::EStatus statuses[nThreads]; - TVector<TMaybe<NYdb::NTable::TSession>> sessions; - sessions.resize(nThreads); - TAtomic t = 0; - auto job = [client, &t, &statuses, &sessions]() mutable { - auto sessionResponse = client.GetSession().ExtractValueSync(); - int i = AtomicIncrement(t); - statuses[--i] = sessionResponse.GetStatus(); - if (statuses[i] == EStatus::SUCCESS) { - auto execStatus = sessionResponse.GetSession().ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync().GetStatus(); - UNIT_ASSERT_EQUAL(execStatus, EStatus::SUCCESS); - sessions[i] = sessionResponse.GetSession(); - } - }; + threads.resize(nThreads); + for (int i = 0; i < nThreads; i++) { + threads[i] = pool->Run(job); + } + for (int i = 0; i < nThreads; i++) { + threads[i]->Join(); + } + UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 0); + driver.Stop(true); + } + + Y_UNIT_TEST(MultiThreadSessionPoolLimitSync) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + const int maxActiveSessions = 45; + NYdb::NTable::TTableClient client(driver, + TClientSettings() + .SessionPoolSettings( + TSessionPoolSettings().MaxActiveSessions(maxActiveSessions))); + + constexpr int nThreads = 100; + NYdb::EStatus statuses[nThreads]; + TVector<TMaybe<NYdb::NTable::TSession>> sessions; + sessions.resize(nThreads); + TAtomic t = 0; + auto job = [client, &t, &statuses, &sessions]() mutable { + auto sessionResponse = client.GetSession().ExtractValueSync(); + int i = AtomicIncrement(t); + statuses[--i] = sessionResponse.GetStatus(); + if (statuses[i] == EStatus::SUCCESS) { + auto execStatus = sessionResponse.GetSession().ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync().GetStatus(); + UNIT_ASSERT_EQUAL(execStatus, EStatus::SUCCESS); + sessions[i] = sessionResponse.GetSession(); + } + }; IThreadFactory* pool = SystemThreadFactory(); - + TVector<TAutoPtr<IThreadFactory::IThread>> threads; - threads.resize(nThreads); - for (int i = 0; i < nThreads; i++) { - threads[i] = pool->Run(job); - } - for (int i = 0; i < nThreads; i++) { - threads[i]->Join(); - } - - sessions.clear(); - - int successCount = 0; - int exhaustedCount = 0; - for (int i = 0; i < nThreads; i++) { - switch (statuses[i]) { - case EStatus::SUCCESS: - successCount++; - break; - case EStatus::CLIENT_RESOURCE_EXHAUSTED: - exhaustedCount++; - break; - default: - UNIT_ASSERT(false); - } - } - - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - UNIT_ASSERT_VALUES_EQUAL(successCount, maxActiveSessions); - UNIT_ASSERT_VALUES_EQUAL(exhaustedCount, nThreads - maxActiveSessions); - driver.Stop(true); - } - - Y_UNIT_TEST(MultiThreadMultipleRequestsOnSharedSessions) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto driver = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - const int maxActiveSessions = 10; - NYdb::NTable::TTableClient client(driver, - TClientSettings() - .SessionPoolSettings( - TSessionPoolSettings().MaxActiveSessions(maxActiveSessions))); - - constexpr int nThreads = 20; - constexpr int nRequests = 50; - std::array<TVector<TAsyncDataQueryResult>, nThreads> results; - TAtomic t = 0; - TAtomic validSessions = 0; - auto job = [client, &t, &results, &validSessions]() mutable { - auto sessionResponse = client.GetSession().ExtractValueSync(); - - int i = AtomicIncrement(t); - TVector<TAsyncDataQueryResult>& r = results[--i]; - - if (sessionResponse.GetStatus() != EStatus::SUCCESS) { - return; - } - AtomicIncrement(validSessions); - - for (int i = 0; i < nRequests; i++) { - r.push_back(sessionResponse.GetSession().ExecuteDataQuery("SELECT 42;", - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); - } - }; + threads.resize(nThreads); + for (int i = 0; i < nThreads; i++) { + threads[i] = pool->Run(job); + } + for (int i = 0; i < nThreads; i++) { + threads[i]->Join(); + } + + sessions.clear(); + + int successCount = 0; + int exhaustedCount = 0; + for (int i = 0; i < nThreads; i++) { + switch (statuses[i]) { + case EStatus::SUCCESS: + successCount++; + break; + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + exhaustedCount++; + break; + default: + UNIT_ASSERT(false); + } + } + + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(successCount, maxActiveSessions); + UNIT_ASSERT_VALUES_EQUAL(exhaustedCount, nThreads - maxActiveSessions); + driver.Stop(true); + } + + Y_UNIT_TEST(MultiThreadMultipleRequestsOnSharedSessions) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto driver = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + const int maxActiveSessions = 10; + NYdb::NTable::TTableClient client(driver, + TClientSettings() + .SessionPoolSettings( + TSessionPoolSettings().MaxActiveSessions(maxActiveSessions))); + + constexpr int nThreads = 20; + constexpr int nRequests = 50; + std::array<TVector<TAsyncDataQueryResult>, nThreads> results; + TAtomic t = 0; + TAtomic validSessions = 0; + auto job = [client, &t, &results, &validSessions]() mutable { + auto sessionResponse = client.GetSession().ExtractValueSync(); + + int i = AtomicIncrement(t); + TVector<TAsyncDataQueryResult>& r = results[--i]; + + if (sessionResponse.GetStatus() != EStatus::SUCCESS) { + return; + } + AtomicIncrement(validSessions); + + for (int i = 0; i < nRequests; i++) { + r.push_back(sessionResponse.GetSession().ExecuteDataQuery("SELECT 42;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())); + } + }; IThreadFactory* pool = SystemThreadFactory(); - + TVector<TAutoPtr<IThreadFactory::IThread>> threads; - threads.resize(nThreads); - for (int i = 0; i < nThreads; i++) { - threads[i] = pool->Run(job); - } - for (int i = 0; i < nThreads; i++) { - threads[i]->Join(); - } - - for (auto& r : results) { + threads.resize(nThreads); + for (int i = 0; i < nThreads; i++) { + threads[i] = pool->Run(job); + } + for (int i = 0; i < nThreads; i++) { + threads[i]->Join(); + } + + for (auto& r : results) { NThreading::WaitExceptionOrAll(r).Wait(); - } - for (auto& r : results) { - if (!r.empty()) { - int ok = 0; - int bad = 0; - for (auto& asyncStatus : r) { - auto res = asyncStatus.GetValue(); - if (res.IsSuccess()) { - ok++; - } else { + } + for (auto& r : results) { + if (!r.empty()) { + int ok = 0; + int bad = 0; + for (auto& asyncStatus : r) { + auto res = asyncStatus.GetValue(); + if (res.IsSuccess()) { + ok++; + } else { UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), EStatus::SESSION_BUSY); - bad++; - } - } - //UNIT_ASSERT_VALUES_EQUAL(ok, 1); - //UNIT_ASSERT_VALUES_EQUAL(bad, nRequests - 1); - } - } - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), maxActiveSessions); - auto curExpectedActive = maxActiveSessions; - auto empty = 0; - for (auto& r : results) { - if (!r.empty()) { - r.clear(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), --curExpectedActive); - } else { - empty++; - } - } - UNIT_ASSERT_VALUES_EQUAL(empty, nThreads - maxActiveSessions); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - driver.Stop(true); - } - - Y_UNIT_TEST(TestColumnOrder) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + bad++; + } + } + //UNIT_ASSERT_VALUES_EQUAL(ok, 1); + //UNIT_ASSERT_VALUES_EQUAL(bad, nRequests - 1); + } + } + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), maxActiveSessions); + auto curExpectedActive = maxActiveSessions; + auto empty = 0; + for (auto& r : results) { + if (!r.empty()) { + r.clear(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), --curExpectedActive); + } else { + empty++; + } + } + UNIT_ASSERT_VALUES_EQUAL(empty, nThreads - maxActiveSessions); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + driver.Stop(true); + } + + Y_UNIT_TEST(TestColumnOrder) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - { - auto status = session.ExecuteSchemeQuery(R"__( - CREATE TABLE [Root/Test] ( - Column1 Uint32, - Column2 Uint32, - Column3 Uint32, - Column4 Uint32, - PRIMARY KEY (Column1) - );)__").ExtractValueSync(); - - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - - session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Column1, Column2, Column3, Column4) - VALUES(1u, 12u, 13u, 14u); - UPSERT INTO [Root/Test] (Column1, Column2, Column3, Column4) - VALUES(2u, 22u, 23u, 24u); + .SetEndpoint(location)); + auto session = CreateSession(connection); + + { + auto status = session.ExecuteSchemeQuery(R"__( + CREATE TABLE [Root/Test] ( + Column1 Uint32, + Column2 Uint32, + Column3 Uint32, + Column4 Uint32, + PRIMARY KEY (Column1) + );)__").ExtractValueSync(); + + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + + session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Column1, Column2, Column3, Column4) + VALUES(1u, 12u, 13u, 14u); + UPSERT INTO [Root/Test] (Column1, Column2, Column3, Column4) + VALUES(2u, 22u, 23u, 24u); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - auto result = session.ExecuteDataQuery(R"___( - SELECT Column4, Column2, Column3, Column1 FROM [Root/Test]; + + auto result = session.ExecuteDataQuery(R"___( + SELECT Column4, Column2, Column3, Column1 FROM [Root/Test]; )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); TVector<TResultSet> resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); + UNIT_ASSERT_EQUAL(resultSets.size(), 1); UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 4); - auto columnMeta = resultSets[0].GetColumnsMeta(); - const TString ref[] = { "Column4", "Column2", "Column3", "Column1" }; - for (size_t i = 0; i < columnMeta.size(); ++i) { - UNIT_ASSERT_NO_DIFF(columnMeta[i].Name, ref[i]); - } - } - - Y_UNIT_TEST(TestDecimal) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + auto columnMeta = resultSets[0].GetColumnsMeta(); + const TString ref[] = { "Column4", "Column2", "Column3", "Column1" }; + for (size_t i = 0; i < columnMeta.size(); ++i) { + UNIT_ASSERT_NO_DIFF(columnMeta[i].Name, ref[i]); + } + } + + Y_UNIT_TEST(TestDecimal) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - auto result = session.ExecuteDataQuery(R"___( + .SetEndpoint(location)); + auto session = CreateSession(connection); + + auto result = session.ExecuteDataQuery(R"___( SELECT CAST("184467440737.12345678" as Decimal(22,9)); - )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector<TResultSet> resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); - auto column = resultSets[0].GetColumnsMeta()[0]; - TTypeParser typeParser(column.Type); - typeParser.OpenOptional(); - UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Decimal); - - TResultSetParser rsParser(resultSets[0]); - while (rsParser.TryNextRow()) { - auto columnParser = std::move(rsParser.ColumnParser(0)); - columnParser.OpenOptional(); - auto decimalString = columnParser.GetDecimal().ToString(); - UNIT_ASSERT_EQUAL(decimalString, "184467440737.12345678"); - } - } - - Y_UNIT_TEST(TestDecimalFullStack) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + TVector<TResultSet> resultSets = result.GetResultSets(); + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); + auto column = resultSets[0].GetColumnsMeta()[0]; + TTypeParser typeParser(column.Type); + typeParser.OpenOptional(); + UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Decimal); + + TResultSetParser rsParser(resultSets[0]); + while (rsParser.TryNextRow()) { + auto columnParser = std::move(rsParser.ColumnParser(0)); + columnParser.OpenOptional(); + auto decimalString = columnParser.GetDecimal().ToString(); + UNIT_ASSERT_EQUAL(decimalString, "184467440737.12345678"); + } + } + + Y_UNIT_TEST(TestDecimalFullStack) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); - auto sessionResponse = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - - auto session = sessionResponse.GetSession(); - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Int32) + auto sessionResponse = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + + auto session = sessionResponse.GetSession(); + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Int32) .AddNullableColumn("Value", TDecimalType(22,9)); - tableBuilder.SetPrimaryKeyColumn("Key"); - auto result = session.CreateTable("/Root/FooTable", tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - TString query = R"___( + tableBuilder.SetPrimaryKeyColumn("Key"); + auto result = session.CreateTable("/Root/FooTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + TString query = R"___( DECLARE $Value AS "Decimal(22,9)"; - DECLARE $Key AS Int32; - UPSERT INTO [Root/FooTable] (Key, Value) VALUES - ($Key, $Value); - )___"; - - constexpr int records = 5; - int count = records; - const TString decimalParams[records] = { - "123", - "4.56", - "0", - "-4.56", - "-123" - }; - while (count--) { - auto paramsBuilder = client.GetParamsBuilder(); - auto params = paramsBuilder - .AddParam("$Key") - .Int32(count) - .Build() - .AddParam("$Value") - .Decimal(TDecimalValue(decimalParams[count])) - .Build() - .Build(); - auto result = session - .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) - .CommitTx(), std::move(params)) - .ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - } - - { - TString query = R"___(SELECT SUM(Value),MIN(Value),MAX(Value) FROM [Root/FooTable])___"; - auto result = session - .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) - .CommitTx()) - .ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector<TResultSet> resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 3); - UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 3); - - for (auto column : resultSets[0].GetColumnsMeta()) { - TTypeParser typeParser(column.Type); - UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Optional); - typeParser.OpenOptional(); - UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Decimal); - } - - TResultSetParser rsParser(resultSets[0]); - const TString expected[3] = { - "0", - "-123", - "123" - }; - while (rsParser.TryNextRow()) { - for (size_t i = 0; i < resultSets[0].ColumnsCount(); i++) { - auto columnParser = std::move(rsParser.ColumnParser(i)); - columnParser.OpenOptional(); - auto decimal = columnParser.GetDecimal(); - - UNIT_ASSERT_EQUAL(decimal.ToString(), expected[i]); - } - } - } - { - auto res = session.DescribeTable("Root/FooTable").ExtractValueSync(); - UNIT_ASSERT_EQUAL(res.IsTransportError(), false); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_EQUAL(res.GetTableDescription().GetColumns().size(), 2); - - TTypeParser::ETypeKind kinds[2] = {TTypeParser::ETypeKind::Primitive, TTypeParser::ETypeKind::Decimal}; - int i = 0; - for (const auto& column : res.GetTableDescription().GetColumns()) { - auto tParser = TTypeParser(column.Type); - tParser.OpenOptional(); - UNIT_ASSERT_EQUAL(kinds[i++], tParser.GetKind()); - } - } - } - - Y_UNIT_TEST(TestTzTypesFullStack) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - - NYdb::NTable::TTableClient client(connection); - auto sessionResponse = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); - - auto session = sessionResponse.GetSession(); - - { - TString query = R"___( - DECLARE $x AS TzDate; - DECLARE $y AS TzDatetime; - DECLARE $z AS TzTimestamp; - SELECT $x, $y, $z; - )___"; - - auto paramsBuilder = client.GetParamsBuilder(); - auto params = paramsBuilder - .AddParam("$x") - .TzDate("2020-09-22,Europe/Moscow") - .Build() - .AddParam("$y") - .TzDatetime("2020-09-22T15:00:00,Europe/Moscow") - .Build() - .AddParam("$z") - .TzTimestamp("2020-09-22T15:00:00,Europe/Moscow") - .Build() - .Build(); - auto result = session - .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) - .CommitTx(), std::move(params)) - .ExtractValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto yson = FormatResultSetYson(result.GetResultSet(0)); - - UNIT_ASSERT_VALUES_EQUAL(yson, "[[\"2020-09-22,Europe/Moscow\";\"2020-09-22T15:00:00,Europe/Moscow\";\"2020-09-22T15:00:00,Europe/Moscow\"]]"); - } - } - - Y_UNIT_TEST(TestVariant) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + DECLARE $Key AS Int32; + UPSERT INTO [Root/FooTable] (Key, Value) VALUES + ($Key, $Value); + )___"; + + constexpr int records = 5; + int count = records; + const TString decimalParams[records] = { + "123", + "4.56", + "0", + "-4.56", + "-123" + }; + while (count--) { + auto paramsBuilder = client.GetParamsBuilder(); + auto params = paramsBuilder + .AddParam("$Key") + .Int32(count) + .Build() + .AddParam("$Value") + .Decimal(TDecimalValue(decimalParams[count])) + .Build() + .Build(); + auto result = session + .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) + .CommitTx(), std::move(params)) + .ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + } + + { + TString query = R"___(SELECT SUM(Value),MIN(Value),MAX(Value) FROM [Root/FooTable])___"; + auto result = session + .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) + .CommitTx()) + .ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + TVector<TResultSet> resultSets = result.GetResultSets(); + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 3); + UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 3); + + for (auto column : resultSets[0].GetColumnsMeta()) { + TTypeParser typeParser(column.Type); + UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Optional); + typeParser.OpenOptional(); + UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Decimal); + } + + TResultSetParser rsParser(resultSets[0]); + const TString expected[3] = { + "0", + "-123", + "123" + }; + while (rsParser.TryNextRow()) { + for (size_t i = 0; i < resultSets[0].ColumnsCount(); i++) { + auto columnParser = std::move(rsParser.ColumnParser(i)); + columnParser.OpenOptional(); + auto decimal = columnParser.GetDecimal(); + + UNIT_ASSERT_EQUAL(decimal.ToString(), expected[i]); + } + } + } + { + auto res = session.DescribeTable("Root/FooTable").ExtractValueSync(); + UNIT_ASSERT_EQUAL(res.IsTransportError(), false); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_EQUAL(res.GetTableDescription().GetColumns().size(), 2); + + TTypeParser::ETypeKind kinds[2] = {TTypeParser::ETypeKind::Primitive, TTypeParser::ETypeKind::Decimal}; + int i = 0; + for (const auto& column : res.GetTableDescription().GetColumns()) { + auto tParser = TTypeParser(column.Type); + tParser.OpenOptional(); + UNIT_ASSERT_EQUAL(kinds[i++], tParser.GetKind()); + } + } + } + + Y_UNIT_TEST(TestTzTypesFullStack) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + + NYdb::NTable::TTableClient client(connection); + auto sessionResponse = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_EQUAL(sessionResponse.IsTransportError(), false); + + auto session = sessionResponse.GetSession(); + + { + TString query = R"___( + DECLARE $x AS TzDate; + DECLARE $y AS TzDatetime; + DECLARE $z AS TzTimestamp; + SELECT $x, $y, $z; + )___"; + + auto paramsBuilder = client.GetParamsBuilder(); + auto params = paramsBuilder + .AddParam("$x") + .TzDate("2020-09-22,Europe/Moscow") + .Build() + .AddParam("$y") + .TzDatetime("2020-09-22T15:00:00,Europe/Moscow") + .Build() + .AddParam("$z") + .TzTimestamp("2020-09-22T15:00:00,Europe/Moscow") + .Build() + .Build(); + auto result = session + .ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()) + .CommitTx(), std::move(params)) + .ExtractValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto yson = FormatResultSetYson(result.GetResultSet(0)); + + UNIT_ASSERT_VALUES_EQUAL(yson, "[[\"2020-09-22,Europe/Moscow\";\"2020-09-22T15:00:00,Europe/Moscow\";\"2020-09-22T15:00:00,Europe/Moscow\"]]"); + } + } + + Y_UNIT_TEST(TestVariant) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - const TString query = R"___( - $struct = AsStruct(5 as foo, true as bar); - $var_type = VariantType(TypeOf($struct)); - select Variant(42,"foo",$var_type) as Variant1; - )___"; - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx(TTxSettings::SerializableRW() - ).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - TVector<TResultSet> resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); - auto column = resultSets[0].GetColumnsMeta()[0]; - TTypeParser typeParser(column.Type); - UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Variant); - - TResultSetParser rsParser(resultSets[0]); - while (rsParser.TryNextRow()) { - auto columnParser = std::move(rsParser.ColumnParser(0)); - columnParser.OpenVariant(); - UNIT_ASSERT_EQUAL(columnParser.GetInt32(), 42); - } - } - - Y_UNIT_TEST(TestDescribeDirectory) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + .SetEndpoint(location)); + auto session = CreateSession(connection); + + const TString query = R"___( + $struct = AsStruct(5 as foo, true as bar); + $var_type = VariantType(TypeOf($struct)); + select Variant(42,"foo",$var_type) as Variant1; + )___"; + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW() + ).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + TVector<TResultSet> resultSets = result.GetResultSets(); + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + UNIT_ASSERT_EQUAL(resultSets[0].ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(resultSets[0].GetColumnsMeta().size(), 1); + auto column = resultSets[0].GetColumnsMeta()[0]; + TTypeParser typeParser(column.Type); + UNIT_ASSERT_EQUAL(typeParser.GetKind(), TTypeParser::ETypeKind::Variant); + + TResultSetParser rsParser(resultSets[0]); + while (rsParser.TryNextRow()) { + auto columnParser = std::move(rsParser.ColumnParser(0)); + columnParser.OpenVariant(); + UNIT_ASSERT_EQUAL(columnParser.GetInt32(), 42); + } + } + + Y_UNIT_TEST(TestDescribeDirectory) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); auto scheme = NYdb::NScheme::TSchemeClient(connection); - auto session = CreateSession(connection); - { - auto status = scheme.MakeDirectory("Root/Foo").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - { - auto status = session.ExecuteSchemeQuery(R"__( - CREATE TABLE [Root/Foo/Test] ( - Column1 Uint32, - Column2 Uint32, - Column3 Uint32, - Column4 Uint32, - PRIMARY KEY (Column1) - );)__").ExtractValueSync(); - - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - - { - auto status = scheme.ListDirectory("Root/Foo").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - auto children = status.GetChildren(); - UNIT_ASSERT_EQUAL(children[0].Name, "Test"); - UNIT_ASSERT_EQUAL(children[0].Type, NYdb::NScheme::ESchemeEntryType::Table); - UNIT_ASSERT_EQUAL(children[0].Owner, "root@builtin"); - } - { - auto status = scheme.ListDirectory("Root/BadPath").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SCHEME_ERROR); - const TString expected = R"___(<main>: Error: Path not found -)___"; - UNIT_ASSERT_EQUAL(status.GetIssues().ToString(), expected); - } - - } - - Y_UNIT_TEST(SecurityTokenAuth) { + auto session = CreateSession(connection); + { + auto status = scheme.MakeDirectory("Root/Foo").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + { + auto status = session.ExecuteSchemeQuery(R"__( + CREATE TABLE [Root/Foo/Test] ( + Column1 Uint32, + Column2 Uint32, + Column3 Uint32, + Column4 Uint32, + PRIMARY KEY (Column1) + );)__").ExtractValueSync(); + + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + + { + auto status = scheme.ListDirectory("Root/Foo").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + auto children = status.GetChildren(); + UNIT_ASSERT_EQUAL(children[0].Name, "Test"); + UNIT_ASSERT_EQUAL(children[0].Type, NYdb::NScheme::ESchemeEntryType::Table); + UNIT_ASSERT_EQUAL(children[0].Owner, "root@builtin"); + } + { + auto status = scheme.ListDirectory("Root/BadPath").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SCHEME_ERROR); + const TString expected = R"___(<main>: Error: Path not found +)___"; + UNIT_ASSERT_EQUAL(status.GetIssues().ToString(), expected); + } + + } + + Y_UNIT_TEST(SecurityTokenAuth) { TKikimrWithGrpcAndRootSchemaWithAuthAndSsl server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto connection = NYdb::TDriver( - TDriverConfig() - .SetAuthToken("root@builtin") - .UseSecureConnection(NYdbSslTestData::CaCrt) - .SetEndpoint(location)); - + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( + TDriverConfig() + .SetAuthToken("root@builtin") + .UseSecureConnection(NYdbSslTestData::CaCrt) + .SetEndpoint(location)); + auto& tableSettings = server.GetServer().GetSettings().AppConfig.GetTableServiceConfig(); bool useSchemeCacheMeta = tableSettings.GetUseSchemeCacheMetadata(); - { - auto session = CreateSession(connection, "root@builtin"); - { - auto status = session.ExecuteSchemeQuery(R"__( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - );)__").ExtractValueSync(); - - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - { + { + auto session = CreateSession(connection, "root@builtin"); + { + auto status = session.ExecuteSchemeQuery(R"__( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + );)__").ExtractValueSync(); + + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + { auto scheme = NYdb::NScheme::TSchemeClient(connection); - auto status = scheme.ModifyPermissions("Root/Test", - NYdb::NScheme::TModifyPermissionsSettings() - .AddGrantPermissions( - NYdb::NScheme::TPermissions("pupkin@builtin", TVector<TString>{"ydb.tables.modify"}) - ) - .AddSetPermissions( - NYdb::NScheme::TPermissions("root@builtin", TVector<TString>{"ydb.tables.modify"}) //This permission should be ignored - last set win - ) - .AddSetPermissions( - NYdb::NScheme::TPermissions("root@builtin", TVector<TString>{"ydb.tables.read"}) - ) - ).ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - { + auto status = scheme.ModifyPermissions("Root/Test", + NYdb::NScheme::TModifyPermissionsSettings() + .AddGrantPermissions( + NYdb::NScheme::TPermissions("pupkin@builtin", TVector<TString>{"ydb.tables.modify"}) + ) + .AddSetPermissions( + NYdb::NScheme::TPermissions("root@builtin", TVector<TString>{"ydb.tables.modify"}) //This permission should be ignored - last set win + ) + .AddSetPermissions( + NYdb::NScheme::TPermissions("root@builtin", TVector<TString>{"ydb.tables.read"}) + ) + ).ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + { auto scheme = NYdb::NScheme::TSchemeClient(connection); - auto status = scheme.DescribePath("Root/Test").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - auto entry = status.GetEntry(); - UNIT_ASSERT_EQUAL(entry.Owner, "root@builtin"); - UNIT_ASSERT_EQUAL(entry.Permissions.size(), 2); - UNIT_ASSERT_EQUAL(entry.Permissions[0].PermissionNames.size(), 1); - UNIT_ASSERT_EQUAL(entry.Permissions[0].Subject, "pupkin@builtin"); - UNIT_ASSERT_EQUAL(entry.Permissions[0].PermissionNames[0], "ydb.tables.modify"); - UNIT_ASSERT_EQUAL(entry.Permissions[1].Subject, "root@builtin"); - UNIT_ASSERT_EQUAL(entry.Permissions[1].PermissionNames.size(), 1); - UNIT_ASSERT_EQUAL(entry.Permissions[1].PermissionNames[0], "ydb.tables.read"); - } - } - { - auto session = CreateSession(connection, "test_user@builtin"); - - { - auto status = session.ExecuteDataQuery(R"__( - SELECT * FROM [Root/Test]; - )__",TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + auto status = scheme.DescribePath("Root/Test").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + auto entry = status.GetEntry(); + UNIT_ASSERT_EQUAL(entry.Owner, "root@builtin"); + UNIT_ASSERT_EQUAL(entry.Permissions.size(), 2); + UNIT_ASSERT_EQUAL(entry.Permissions[0].PermissionNames.size(), 1); + UNIT_ASSERT_EQUAL(entry.Permissions[0].Subject, "pupkin@builtin"); + UNIT_ASSERT_EQUAL(entry.Permissions[0].PermissionNames[0], "ydb.tables.modify"); + UNIT_ASSERT_EQUAL(entry.Permissions[1].Subject, "root@builtin"); + UNIT_ASSERT_EQUAL(entry.Permissions[1].PermissionNames.size(), 1); + UNIT_ASSERT_EQUAL(entry.Permissions[1].PermissionNames[0], "ydb.tables.read"); + } + } + { + auto session = CreateSession(connection, "test_user@builtin"); + + { + auto status = session.ExecuteDataQuery(R"__( + SELECT * FROM [Root/Test]; + )__",TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(status.IsTransportError(), false); UNIT_ASSERT_EQUAL(status.GetStatus(), useSchemeCacheMeta ? EStatus::SCHEME_ERROR : EStatus::UNAUTHORIZED); - } - } - } - + } + } + } + Y_UNIT_TEST(ConnectDbAclIsStrictlyChecked) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableFeatureFlags()->SetCheckDatabaseAccessPermission(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableFeatureFlags()->SetCheckDatabaseAccessPermission(true); appConfig.MutableFeatureFlags()->SetAllowYdbRequestsWithoutDatabase(false); appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); appConfig.MutableDomainsConfig()->MutableSecurityConfig()->AddDefaultUserSIDs("test_user_no_rights@builtin"); TKikimrWithGrpcAndRootSchemaWithAuth server(appConfig); - + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_PROXY_NO_CONNECT_ACCESS, NActors::NLog::PRI_DEBUG); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); { // no db TString location = TStringBuilder() << "localhost:" << grpc; @@ -1005,14 +1005,14 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::CLIENT_UNAUTHENTICATED, status.GetIssues().ToString()); } - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = NYdb::TDriver( - TDriverConfig() + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = NYdb::TDriver( + TDriverConfig() .SetEndpoint(location) .SetDatabase("/Root")); - + { // no token - NYdb::NTable::TClientSettings settings; + NYdb::NTable::TClientSettings settings; NYdb::NTable::TTableClient client(driver, settings); auto call = [] (NYdb::NTable::TTableClient& client) -> NYdb::TStatus { Cerr << "Call\n"; @@ -1046,8 +1046,8 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { .SetDatabase("/Root")); NYdb::NTable::TClientSettings settings; - settings.AuthToken("test_user@builtin"); - NYdb::NTable::TTableClient client(driver, settings); + settings.AuthToken("test_user@builtin"); + NYdb::NTable::TTableClient client(driver, settings); auto call = [] (NYdb::NTable::TTableClient& client) -> NYdb::TStatus { return client.CreateSession().ExtractValueSync(); @@ -1055,22 +1055,22 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { auto status = client.RetryOperationSync(call); UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::UNAUTHORIZED, status.GetIssues().ToString()); - } - + } + { // set connect NYdb::TCommonClientSettings settings; settings.AuthToken("root@builtin"); auto scheme = NYdb::NScheme::TSchemeClient(driver, settings); - auto status = scheme.ModifyPermissions("/Root", - NYdb::NScheme::TModifyPermissionsSettings() - .AddGrantPermissions( - NYdb::NScheme::TPermissions("test_user@builtin", TVector<TString>{"ydb.database.connect"}) - ) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); + auto status = scheme.ModifyPermissions("/Root", + NYdb::NScheme::TModifyPermissionsSettings() + .AddGrantPermissions( + NYdb::NScheme::TPermissions("test_user@builtin", TVector<TString>{"ydb.database.connect"}) + ) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL_C(status.IsTransportError(), false, status.GetIssues().ToString()); - } - + } + ui32 attemps = 2; // system is notified asynchronously, so it may see old acl for awhile while (attemps) { // accept connect right --attemps; @@ -1101,7 +1101,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { TKikimrWithGrpcAndRootSchema server(appConfig); ui16 grpc = server.GetPort(); - { + { TString location = TStringBuilder() << "localhost:" << grpc; auto driver = NYdb::TDriver( TDriverConfig() @@ -1109,9 +1109,9 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { .SetDatabase("/Root")); // with db - NYdb::NTable::TClientSettings settings; - settings.AuthToken("test_user@builtin"); - NYdb::NTable::TTableClient client(driver, settings); + NYdb::NTable::TClientSettings settings; + settings.AuthToken("test_user@builtin"); + NYdb::NTable::TTableClient client(driver, settings); auto call = [] (NYdb::NTable::TTableClient& client) -> NYdb::TStatus { return client.CreateSession().ExtractValueSync(); @@ -1119,7 +1119,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { auto status = client.RetryOperationSync(call); UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::UNAUTHORIZED, status.GetIssues().ToString()); - } + } { TString location = TStringBuilder() << "localhost:" << grpc; @@ -1139,7 +1139,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); } - } + } Y_UNIT_TEST(ConnectDbAclIsOffWhenTokenIsOptionalAndNull) { NKikimrConfig::TAppConfig appConfig; @@ -1167,260 +1167,260 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); } } -/* - Y_UNIT_TEST(SecurityTokenError) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); - TKikimrWithGrpcAndRootSchema server(appConfig, true, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto connection = NYdb::TDriver( - TDriverConfig() - .SetAuthToken("@error") - .UseSecureConnection(NYdbSslTestData::CaCrt) - .SetEndpoint(location)); - NYdb::NTable::TTableClient client(connection); - - { - auto session = client.GetSession().GetValueSync(); - UNIT_ASSERT_EQUAL(session.GetStatus(), EStatus::UNAVAILABLE); - UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 1); - } - UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 0); - } -*/ - - Y_UNIT_TEST(SecurityTokenAuthMultiTenantSDK) { - MultiTenantSDK(false); - } - - Y_UNIT_TEST(SecurityTokenAuthMultiTenantSDKAsync) { - MultiTenantSDK(true); - } - - Y_UNIT_TEST(TraceId) { - TStringStream logStream; - TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); - - TString traceId = "CppUtTestQuery"; - - { +/* + Y_UNIT_TEST(SecurityTokenError) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); + TKikimrWithGrpcAndRootSchema server(appConfig, true, true); + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( + TDriverConfig() + .SetAuthToken("@error") + .UseSecureConnection(NYdbSslTestData::CaCrt) + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); + + { + auto session = client.GetSession().GetValueSync(); + UNIT_ASSERT_EQUAL(session.GetStatus(), EStatus::UNAVAILABLE); + UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 1); + } + UNIT_ASSERT_EQUAL(client.GetActiveSessionCount(), 0); + } +*/ + + Y_UNIT_TEST(SecurityTokenAuthMultiTenantSDK) { + MultiTenantSDK(false); + } + + Y_UNIT_TEST(SecurityTokenAuthMultiTenantSDKAsync) { + MultiTenantSDK(true); + } + + Y_UNIT_TEST(TraceId) { + TStringStream logStream; + TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); + + TString traceId = "CppUtTestQuery"; + + { NKikimrConfig::TAppConfig appConfig; auto& logConfig = *appConfig.MutableLogConfig(); auto& entry = *logConfig.AddEntry(); entry.SetComponent(NKikimrServices::EServiceKikimr_Name(NKikimrServices::KQP_YQL)); entry.SetLevel(NActors::NLog::PRI_DEBUG); - + TKikimrWithGrpcAndRootSchema server(appConfig, {}, logBackend); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); - - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); + + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - + .SetEndpoint(location)); + auto session = CreateSession(connection); + auto result = session - .ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - ); + .ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + ); )___", TExecSchemeQuerySettings().TraceId(traceId)).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + result = session - .ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) - VALUES(2u, "Two"); + .ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) + VALUES(2u, "Two"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), TExecDataQuerySettings().TraceId(traceId)).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - bool grpcHasTraceId = false; - bool proxyHasTraceId = false; - bool workerHasTraceId = false; - bool yqlHasTraceId = false; - - TString line; - while (logStream.ReadLine(line)) { - if (line.Contains(traceId)) { - grpcHasTraceId = grpcHasTraceId || line.Contains("GRPC_SERVER"); - proxyHasTraceId = proxyHasTraceId || line.Contains("KQP_PROXY"); - workerHasTraceId = workerHasTraceId || line.Contains("KQP_WORKER"); - yqlHasTraceId = yqlHasTraceId || line.Contains("KQP_YQL"); - } - } - - UNIT_ASSERT(grpcHasTraceId); - UNIT_ASSERT(proxyHasTraceId); - UNIT_ASSERT(workerHasTraceId); - UNIT_ASSERT(yqlHasTraceId); - } - - Y_UNIT_TEST(BuildInfo) { - TStringStream logStream; - TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); - - { + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + bool grpcHasTraceId = false; + bool proxyHasTraceId = false; + bool workerHasTraceId = false; + bool yqlHasTraceId = false; + + TString line; + while (logStream.ReadLine(line)) { + if (line.Contains(traceId)) { + grpcHasTraceId = grpcHasTraceId || line.Contains("GRPC_SERVER"); + proxyHasTraceId = proxyHasTraceId || line.Contains("KQP_PROXY"); + workerHasTraceId = workerHasTraceId || line.Contains("KQP_WORKER"); + yqlHasTraceId = yqlHasTraceId || line.Contains("KQP_YQL"); + } + } + + UNIT_ASSERT(grpcHasTraceId); + UNIT_ASSERT(proxyHasTraceId); + UNIT_ASSERT(workerHasTraceId); + UNIT_ASSERT(yqlHasTraceId); + } + + Y_UNIT_TEST(BuildInfo) { + TStringStream logStream; + TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); + + { + TKikimrWithGrpcAndRootSchema server({}, {}, logBackend); + + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); + + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location) + .SetDatabase("/Root")); + Y_UNUSED(connection); + } + bool grpcHasBuildInfo = false; + + TString line; + const TString expectedBuildInfo = Sprintf("ydb-cpp-sdk/%s", GetSdkSemver().c_str()); + while (logStream.ReadLine(line)) { + if (line.Contains(expectedBuildInfo)) { + grpcHasBuildInfo = grpcHasBuildInfo || line.Contains("GRPC_SERVER"); + } + } + UNIT_ASSERT(grpcHasBuildInfo); + } + + Y_UNIT_TEST(Utf8DatabasePassViaHeader) { + TStringStream logStream; + TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); + + TString utf8Database = "/йцукен"; + { TKikimrWithGrpcAndRootSchema server({}, {}, logBackend); - - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); - - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location) - .SetDatabase("/Root")); - Y_UNUSED(connection); - } - bool grpcHasBuildInfo = false; - - TString line; - const TString expectedBuildInfo = Sprintf("ydb-cpp-sdk/%s", GetSdkSemver().c_str()); - while (logStream.ReadLine(line)) { - if (line.Contains(expectedBuildInfo)) { - grpcHasBuildInfo = grpcHasBuildInfo || line.Contains("GRPC_SERVER"); - } - } - UNIT_ASSERT(grpcHasBuildInfo); - } - - Y_UNIT_TEST(Utf8DatabasePassViaHeader) { - TStringStream logStream; - TAutoPtr<TLogBackend> logBackend(new TStreamLogBackend(&logStream)); - - TString utf8Database = "/йцукен"; - { - TKikimrWithGrpcAndRootSchema server({}, {}, logBackend); - - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); - - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location) - .SetDatabase(utf8Database)); - Y_UNUSED(connection); - } - bool found = false; - - TString line; - while (logStream.ReadLine(line)) { - if (line.Contains(utf8Database)) { - found = true; - } - } - UNIT_ASSERT(found); - } - - - Y_UNIT_TEST(TestTransactionQueryError) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); + + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location) + .SetDatabase(utf8Database)); + Y_UNUSED(connection); + } + bool found = false; + + TString line; + while (logStream.ReadLine(line)) { + if (line.Contains(utf8Database)) { + found = true; + } + } + UNIT_ASSERT(found); + } + + + Y_UNIT_TEST(TestTransactionQueryError) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - auto session = CreateSession(connection); - - { - auto status = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] (Key Int32, Value String, PRIMARY KEY (Key)); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - - auto result1 = session.ExecuteDataQuery(R"___( - INSERT INTO [Root/Test] (Key, Value) VALUES(1u, "One"); + .SetEndpoint(location)); + auto session = CreateSession(connection); + + { + auto status = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] (Key Int32, Value String, PRIMARY KEY (Key)); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + + auto result1 = session.ExecuteDataQuery(R"___( + INSERT INTO [Root/Test] (Key, Value) VALUES(1u, "One"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result1.GetStatus(), EStatus::SUCCESS); - - { - auto session2 = CreateSession(connection); - auto result = session2.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES(1u, "Two"); + UNIT_ASSERT_EQUAL(result1.GetStatus(), EStatus::SUCCESS); + + { + auto session2 = CreateSession(connection); + auto result = session2.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES(1u, "Two"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result2 = session.ExecuteDataQuery(R"___( - SELECT 42; + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result2 = session.ExecuteDataQuery(R"___( + SELECT 42; )___", TTxControl::Tx(*result1.GetTransaction()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_EQUAL(result2.GetStatus(), EStatus::NOT_FOUND); - auto issueString = result2.GetIssues().ToString(); - TString expected = -R"___(<main>: Error: Transaction not found: , code: 2015 -)___"; - UNIT_ASSERT_NO_DIFF(issueString, expected); - } - } - - Y_UNIT_TEST(TestExecError) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + auto issueString = result2.GetIssues().ToString(); + TString expected = +R"___(<main>: Error: Transaction not found: , code: 2015 +)___"; + UNIT_ASSERT_NO_DIFF(issueString, expected); + } + } + + Y_UNIT_TEST(TestExecError) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - { - auto status = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); - } - - auto fillQueryResult = session.PrepareDataQuery(R"___( - DECLARE $Data AS 'List<Struct<Key:Uint64, Value:String>>'; - - REPLACE INTO [Root/Test] - SELECT data.Key AS Key, data.Value AS Value FROM (SELECT $Data AS data) FLATTEN BY data; - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(fillQueryResult.GetStatus(), EStatus::SUCCESS); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + { + auto status = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(status.GetStatus(), EStatus::SUCCESS); + } + + auto fillQueryResult = session.PrepareDataQuery(R"___( + DECLARE $Data AS 'List<Struct<Key:Uint64, Value:String>>'; + + REPLACE INTO [Root/Test] + SELECT data.Key AS Key, data.Value AS Value FROM (SELECT $Data AS data) FLATTEN BY data; + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(fillQueryResult.GetStatus(), EStatus::SUCCESS); auto query = fillQueryResult.GetQuery(); - - const ui32 BATCH_NUM = 5; - const ui32 BATCH_ROWS = 100; - const ui32 BLOB_SIZE = 100 * 1024; // 100 Kb - - for (ui64 i = 0; i < BATCH_NUM ; ++i) { + + const ui32 BATCH_NUM = 5; + const ui32 BATCH_ROWS = 100; + const ui32 BLOB_SIZE = 100 * 1024; // 100 Kb + + for (ui64 i = 0; i < BATCH_NUM ; ++i) { TParamsBuilder paramsBuilder = client.GetParamsBuilder(); - + auto& paramBuilder = paramsBuilder.AddParam("$Data"); paramBuilder.BeginList(); - for (ui64 j = 0; j < BATCH_ROWS; ++j) { - auto key = i * BATCH_ROWS + j; - auto val = TString(BLOB_SIZE, '0' + key % 10); + for (ui64 j = 0; j < BATCH_ROWS; ++j) { + auto key = i * BATCH_ROWS + j; + auto val = TString(BLOB_SIZE, '0' + key % 10); paramBuilder.AddListItem() .BeginStruct() .AddMember("Key") @@ -1428,65 +1428,65 @@ R"___(<main>: Error: Transaction not found: , code: 2015 .AddMember("Value") .String(val) .EndStruct(); - } - paramBuilder.EndList(); + } + paramBuilder.EndList(); paramBuilder.Build(); - + auto result = query.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), paramsBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - auto result = session.ExecuteDataQuery(R"___( - SELECT * FROM [Root/Test] WHERE Key != 1; + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + auto result = session.ExecuteDataQuery(R"___( + SELECT * FROM [Root/Test] WHERE Key != 1; )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::UNDETERMINED); - UNIT_ASSERT(result.GetIssues().ToString().Contains("ExecResultUnavailable")); - UNIT_ASSERT(result.GetIssues().ToString().Contains("REPLY_SIZE_EXECEEDED")); - } - - Y_UNIT_TEST(TestDoubleKey) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + UNIT_ASSERT(result.GetIssues().ToString().Contains("ExecResultUnavailable")); + UNIT_ASSERT(result.GetIssues().ToString().Contains("REPLY_SIZE_EXECEEDED")); + } + + Y_UNIT_TEST(TestDoubleKey) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Double, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - - auto ref = R"___(<main>: Error: Execution, code: 1060 - <main>:5:30: Error: Executing CREATE TABLE - <main>: Error: Scheme operation failed, status: ExecError, reason: Column Key has wrong key type Double -)___"; - UNIT_ASSERT_NO_DIFF(result.GetIssues().ToString(), ref); - } - + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Double, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + + auto ref = R"___(<main>: Error: Execution, code: 1060 + <main>:5:30: Error: Executing CREATE TABLE + <main>: Error: Scheme operation failed, status: ExecError, reason: Column Key has wrong key type Double +)___"; + UNIT_ASSERT_NO_DIFF(result.GetIssues().ToString(), ref); + } + Y_UNIT_TEST(TestBusySession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); + auto session = client.CreateSession().ExtractValueSync().GetSession(); TVector<NYdb::NTable::TAsyncDataQueryResult> futures; - - for (ui32 i = 0; i < 10; ++i) { + + for (ui32 i = 0; i < 10; ++i) { auto query = session.ExecuteDataQuery(R"___( SELECT 1; )___", TTxControl::BeginTx().CommitTx()); @@ -1520,146 +1520,166 @@ R"___(<main>: Error: Transaction not found: , code: 2015 TVector<NYdb::TAsyncStatus> futures; for (ui32 i = 0; i < sessionsCount; ++i) { auto query = sessions[i].ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Int, - Value String, - PRIMARY KEY (Key) - ); - )___"); - futures.push_back(query); - } - - for (auto& future : futures) { - auto result = future.ExtractValueSync(); - if (result.GetStatus() != EStatus::SUCCESS) { - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::OVERLOADED); - } - } - - // Make sure table exists + CREATE TABLE [Root/Test] ( + Key Int, + Value String, + PRIMARY KEY (Key) + ); + )___"); + futures.push_back(query); + } + + for (auto& future : futures) { + auto result = future.ExtractValueSync(); + if (result.GetStatus() != EStatus::SUCCESS) { + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::OVERLOADED); + } + } + + // Make sure table exists auto result = sessions[0].ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES (1, "One"); + UPSERT INTO [Root/Test] (Key, Value) VALUES (1, "One"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - Y_UNIT_TEST(TestYqlLongSessionPrepareError) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + Y_UNIT_TEST(TestYqlLongSessionPrepareError) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - auto result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES(1u, "One"); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + auto result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES(1u, "One"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - Cerr << result.GetIssues().ToString() << Endl; - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - auto prepareResult = session.PrepareDataQuery(R"___( - SELECT * FROM [Root/BadTable]; - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(prepareResult.GetStatus(), EStatus::SCHEME_ERROR); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES(2u, "Two"); + Cerr << result.GetIssues().ToString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + auto prepareResult = session.PrepareDataQuery(R"___( + SELECT * FROM [Root/BadTable]; + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(prepareResult.GetStatus(), EStatus::SCHEME_ERROR); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES(2u, "Two"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - SELECT * FROM [Root/Test]; + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + SELECT * FROM [Root/Test]; )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + TResultSet resultSet = result.GetResultSet(0); TResultSetParser rsParser(resultSet); - int c = 0; + int c = 0; while (rsParser.TryNextRow()) { - c++; - } - - UNIT_ASSERT_VALUES_EQUAL(c, 2); - } - - Y_UNIT_TEST(TestYqlLongSessionMultipleErrors) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + c++; + } + + UNIT_ASSERT_VALUES_EQUAL(c, 2); + } + + Y_UNIT_TEST(TestYqlLongSessionMultipleErrors) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint32, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/BadTable1] (Key, Value) VALUES(1u, "One"); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint32, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/BadTable1] (Key, Value) VALUES(1u, "One"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/BadTable2] (Key, Value) VALUES(2u, "Two"); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/BadTable2] (Key, Value) VALUES(2u, "Two"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - Y_UNIT_TEST(TestYqlTypesFromPreparedQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + Y_UNIT_TEST(TestYqlTypesFromPreparedQuery) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto result = session.PrepareDataQuery(R"___( - DECLARE $paramName AS String; - SELECT $paramName; - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto result = session.PrepareDataQuery(R"___( + DECLARE $paramName AS String; + SELECT $paramName; + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto query = result.GetQuery(); auto paramsBuilder = query.GetParamsBuilder(); paramsBuilder.AddParam("$paramName").String("someString").Build(); - auto params = paramsBuilder.Build(); - { + auto params = paramsBuilder.Build(); + { + auto result = query.Execute(TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), + params).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + auto resultSets = result.GetResultSets(); + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + auto& resultSet = resultSets[0]; + UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 1); + auto meta = resultSet.GetColumnsMeta(); + UNIT_ASSERT_EQUAL(meta.size(), 1); + TTypeParser parser(meta[0].Type); + UNIT_ASSERT(parser.GetKind() == TTypeParser::ETypeKind::Primitive); + UNIT_ASSERT(parser.GetPrimitive() == EPrimitiveType::String); + + TResultSetParser rsParser(resultSet); + while (rsParser.TryNextRow()) { + UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetString(), "someString"); + } + } + // Test params is not destructed during previous execution + { auto result = query.Execute(TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), - params).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + params).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - auto& resultSet = resultSets[0]; + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + auto& resultSet = resultSets[0]; UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 1); - auto meta = resultSet.GetColumnsMeta(); - UNIT_ASSERT_EQUAL(meta.size(), 1); + auto meta = resultSet.GetColumnsMeta(); + UNIT_ASSERT_EQUAL(meta.size(), 1); TTypeParser parser(meta[0].Type); UNIT_ASSERT(parser.GetKind() == TTypeParser::ETypeKind::Primitive); UNIT_ASSERT(parser.GetPrimitive() == EPrimitiveType::String); @@ -1667,216 +1687,196 @@ R"___(<main>: Error: Transaction not found: , code: 2015 TResultSetParser rsParser(resultSet); while (rsParser.TryNextRow()) { UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetString(), "someString"); - } - } - // Test params is not destructed during previous execution - { - auto result = query.Execute(TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), - params).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - auto resultSets = result.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - auto& resultSet = resultSets[0]; - UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 1); - auto meta = resultSet.GetColumnsMeta(); - UNIT_ASSERT_EQUAL(meta.size(), 1); - TTypeParser parser(meta[0].Type); - UNIT_ASSERT(parser.GetKind() == TTypeParser::ETypeKind::Primitive); - UNIT_ASSERT(parser.GetPrimitive() == EPrimitiveType::String); - - TResultSetParser rsParser(resultSet); - while (rsParser.TryNextRow()) { - UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetString(), "someString"); - } - } - } - - Y_UNIT_TEST(TestConstraintViolation) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - + } + } + } + + Y_UNIT_TEST(TestConstraintViolation) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - INSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + INSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - INSERT INTO [Root/Test] (Key, Value) VALUES (1u, "Two"); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + INSERT INTO [Root/Test] (Key, Value) VALUES (1u, "Two"); )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); - } - - Y_UNIT_TEST(TestReadTableOneBatch) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); + } + + Y_UNIT_TEST(TestReadTableOneBatch) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); - )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - { - TValueBuilder valueFrom; - valueFrom.BeginTuple() - .AddElement() - .Uint64(1) - .EndTuple(); - - auto settings = TReadTableSettings() - .Ordered() - .From(TKeyBound::Inclusive(valueFrom.Build())); - - auto it = session.ReadTable("Root/Test", settings).ExtractValueSync(); - - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); - } - - { - TValueBuilder valueTo; - valueTo.BeginTuple() - .AddElement() - .Uint64(1000) - .EndTuple(); - - auto settings = TReadTableSettings() - .Ordered() - .To(TKeyBound::Inclusive(valueTo.Build())); - - auto it = session.ReadTable("Root/Test", settings).ExtractValueSync(); - - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); - } - - { - auto it = session.ReadTable("Root/Test").ExtractValueSync(); - bool read = false; - while (true) { - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - if (streamPart.EOS()) { - break; - } - UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); - - auto rsParser = TResultSetParser(streamPart.ExtractPart()); - - while (rsParser.TryNextRow()) { - auto columnParser1 = std::move(rsParser.ColumnParser(0)); - columnParser1.OpenOptional(); - auto key = columnParser1.GetUint64(); - UNIT_ASSERT_EQUAL(key, 1); - auto& columnParser2 = rsParser.ColumnParser(1); - columnParser2.OpenOptional(); - auto val = columnParser2.GetString(); - UNIT_ASSERT_EQUAL(val, "One"); - read = true; - } - } - UNIT_ASSERT(read); - - // Attempt to call ReadNext on finished iterator causes ContractViolation - UNIT_ASSERT_EXCEPTION(it.ReadNext().GetValueSync().EOS(), NYdb::TContractViolation); - } - } - + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); + )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + { + TValueBuilder valueFrom; + valueFrom.BeginTuple() + .AddElement() + .Uint64(1) + .EndTuple(); + + auto settings = TReadTableSettings() + .Ordered() + .From(TKeyBound::Inclusive(valueFrom.Build())); + + auto it = session.ReadTable("Root/Test", settings).ExtractValueSync(); + + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); + } + + { + TValueBuilder valueTo; + valueTo.BeginTuple() + .AddElement() + .Uint64(1000) + .EndTuple(); + + auto settings = TReadTableSettings() + .Ordered() + .To(TKeyBound::Inclusive(valueTo.Build())); + + auto it = session.ReadTable("Root/Test", settings).ExtractValueSync(); + + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); + } + + { + auto it = session.ReadTable("Root/Test").ExtractValueSync(); + bool read = false; + while (true) { + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + if (streamPart.EOS()) { + break; + } + UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); + + auto rsParser = TResultSetParser(streamPart.ExtractPart()); + + while (rsParser.TryNextRow()) { + auto columnParser1 = std::move(rsParser.ColumnParser(0)); + columnParser1.OpenOptional(); + auto key = columnParser1.GetUint64(); + UNIT_ASSERT_EQUAL(key, 1); + auto& columnParser2 = rsParser.ColumnParser(1); + columnParser2.OpenOptional(); + auto val = columnParser2.GetString(); + UNIT_ASSERT_EQUAL(val, "One"); + read = true; + } + } + UNIT_ASSERT(read); + + // Attempt to call ReadNext on finished iterator causes ContractViolation + UNIT_ASSERT_EXCEPTION(it.ReadNext().GetValueSync().EOS(), NYdb::TContractViolation); + } + } + enum class EReadTableMultiShardMode { Normal, UseSnapshot, }; void TestReadTableMultiShard(EReadTableMultiShardMode mode) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .AddNullableColumn("Key2", EPrimitiveType::Uint32) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Key2"}); - - TCreateTableSettings createTableSettings = - TCreateTableSettings() - .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(100)); - - auto result = session.CreateTable("Root/Test", tableBuilder.Build(), createTableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Key2, Value) VALUES - (1u, 2u, "One"), - (1000000000u, 2000000000u, "Two"); - )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - TValueBuilder valueFrom; - valueFrom.BeginTuple() - .AddElement() - .OptionalUint32(1) - .EndTuple(); - - TValueBuilder valueTo; - valueTo.BeginTuple() - .AddElement() - .OptionalUint32(1000000000u) - .AddElement() - .OptionalUint32(2000000000u) - .EndTuple(); - - TReadTableSettings readTableSettings = - TReadTableSettings() - .Ordered() - .From(TKeyBound::Inclusive(valueFrom.Build())) - .To(TKeyBound::Inclusive(valueTo.Build())); - + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Key2", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Key2"}); + + TCreateTableSettings createTableSettings = + TCreateTableSettings() + .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(100)); + + auto result = session.CreateTable("Root/Test", tableBuilder.Build(), createTableSettings).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Key2, Value) VALUES + (1u, 2u, "One"), + (1000000000u, 2000000000u, "Two"); + )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + TValueBuilder valueFrom; + valueFrom.BeginTuple() + .AddElement() + .OptionalUint32(1) + .EndTuple(); + + TValueBuilder valueTo; + valueTo.BeginTuple() + .AddElement() + .OptionalUint32(1000000000u) + .AddElement() + .OptionalUint32(2000000000u) + .EndTuple(); + + TReadTableSettings readTableSettings = + TReadTableSettings() + .Ordered() + .From(TKeyBound::Inclusive(valueFrom.Build())) + .To(TKeyBound::Inclusive(valueTo.Build())); + switch (mode) { case EReadTableMultiShardMode::Normal: break; @@ -1885,49 +1885,49 @@ R"___(<main>: Error: Transaction not found: , code: 2015 break; } - auto it = session.ReadTable("Root/Test", readTableSettings).ExtractValueSync(); - - struct TRows { - ui32 Key; - ui32 Key2; - TString Value; - }; - TVector<TRows> expected; - expected.push_back({1, 2, "One"}); - expected.push_back({1000000000u, 2000000000u, "Two"}); - int row = 0; - while (true) { - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - - if (streamPart.EOS()) { - break; - } - UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); - - auto rsParser = TResultSetParser(streamPart.ExtractPart()); - - while (rsParser.TryNextRow()) { - const TRows& exp = expected[row++]; - - rsParser.ColumnParser(0).OpenOptional(); - auto key = rsParser.ColumnParser(0).GetUint32(); - UNIT_ASSERT_VALUES_EQUAL(key, exp.Key); - - rsParser.ColumnParser(1).OpenOptional(); - auto key2 = rsParser.ColumnParser(1).GetUint32(); - UNIT_ASSERT_VALUES_EQUAL(key2, exp.Key2); - - rsParser.ColumnParser(2).OpenOptional(); - auto val = rsParser.ColumnParser(2).GetString(); - UNIT_ASSERT_VALUES_EQUAL(val, exp.Value); - } - } - UNIT_ASSERT_VALUES_EQUAL(row, 2); - - // Attempt to call ReadNext on finished iterator causes ContractViolation - UNIT_ASSERT_EXCEPTION(it.ReadNext().GetValueSync().EOS(), NYdb::TContractViolation); - } - + auto it = session.ReadTable("Root/Test", readTableSettings).ExtractValueSync(); + + struct TRows { + ui32 Key; + ui32 Key2; + TString Value; + }; + TVector<TRows> expected; + expected.push_back({1, 2, "One"}); + expected.push_back({1000000000u, 2000000000u, "Two"}); + int row = 0; + while (true) { + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + + if (streamPart.EOS()) { + break; + } + UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); + + auto rsParser = TResultSetParser(streamPart.ExtractPart()); + + while (rsParser.TryNextRow()) { + const TRows& exp = expected[row++]; + + rsParser.ColumnParser(0).OpenOptional(); + auto key = rsParser.ColumnParser(0).GetUint32(); + UNIT_ASSERT_VALUES_EQUAL(key, exp.Key); + + rsParser.ColumnParser(1).OpenOptional(); + auto key2 = rsParser.ColumnParser(1).GetUint32(); + UNIT_ASSERT_VALUES_EQUAL(key2, exp.Key2); + + rsParser.ColumnParser(2).OpenOptional(); + auto val = rsParser.ColumnParser(2).GetString(); + UNIT_ASSERT_VALUES_EQUAL(val, exp.Value); + } + } + UNIT_ASSERT_VALUES_EQUAL(row, 2); + + // Attempt to call ReadNext on finished iterator causes ContractViolation + UNIT_ASSERT_EXCEPTION(it.ReadNext().GetValueSync().EOS(), NYdb::TContractViolation); + } + Y_UNIT_TEST(TestReadTableMultiShard) { TestReadTableMultiShard(EReadTableMultiShardMode::Normal); } @@ -1936,144 +1936,144 @@ R"___(<main>: Error: Transaction not found: , code: 2015 TestReadTableMultiShard(EReadTableMultiShardMode::UseSnapshot); } - void TestReadTableMultiShardWithDescribe(bool rowLimit) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .AddNullableColumn("Key2", EPrimitiveType::Uint32) - .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Key2"}); - - TCreateTableSettings createTableSettings = - TCreateTableSettings() - .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(10)); - - auto result = session.CreateTable("Root/Test", tableBuilder.Build(), createTableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Key2, Value) VALUES - (1u, 2u, "A"), - (429496730u, 20000u, "B"), - (858993459u, 20000u, "C"), - (1288490188u, 20000u, "D"), - (3865470565u, 200000000u, "E"), - (3865470565u, 200000001u, "F"); - )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - TDescribeTableSettings describeTableSettings = - TDescribeTableSettings() - .WithKeyShardBoundary(true); - - TDescribeTableResult describeResult = session.DescribeTable("Root/Test", describeTableSettings) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - - TVector<TString> expected; - expected.push_back(R"___([[1u];[2u];["A"]])___"); - expected.push_back(R"___([[429496730u];[20000u];["B"]])___"); - expected.push_back(R"___([[858993459u];[20000u];["C"]])___"); - expected.push_back(R"___([[1288490188u];[20000u];["D"]])___"); - expected.push_back(R"___([[3865470565u];[200000000u];["E"]])___"); - expected.push_back(R"___([[3865470565u];[200000001u];["F"]])___"); - - int row = 0; - for (const auto& range : describeResult.GetTableDescription().GetKeyRanges()) { - TReadTableSettings readTableSettings; - if (auto from = range.From()) { - readTableSettings.From(from.GetRef()); - } - if (auto to = range.To()) { - readTableSettings.To(to.GetRef()); - } - if (rowLimit) { - readTableSettings.RowLimit(1); - } - - auto it = session.ReadTable("Root/Test", readTableSettings).ExtractValueSync(); - - while (true) { - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - - if (streamPart.EOS()) { - break; - } - - UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); - - auto rsParser = TResultSetParser(streamPart.ExtractPart()); - while (rsParser.TryNextRow()) { - auto columns = rsParser.ColumnsCount(); - const auto& expRow = expected[row++]; - TString tmp = "["; - for (size_t c = 0; c < columns; c++) { - auto colYson = FormatValueYson(rsParser.GetValue(c)); - tmp += colYson; - if (c != columns - 1) - tmp += ";"; - } - tmp += "]"; - UNIT_ASSERT_VALUES_EQUAL(tmp, expRow); - } - } - } - UNIT_ASSERT_VALUES_EQUAL(row, rowLimit ? 5 : expected.size()); - } - - Y_UNIT_TEST(TestReadTableMultiShardWithDescribe) { - TestReadTableMultiShardWithDescribe(false); - } - - Y_UNIT_TEST(TestReadTableMultiShardWithDescribeAndRowLimit) { - TestReadTableMultiShardWithDescribe(true); - } - - Y_UNIT_TEST(TestReadWrongTable) { - TKikimrWithGrpcAndRootSchema server; - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - NYdb::NTable::TTableClient client(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - auto it = session.ReadTable("Root/NoTable").ExtractValueSync(); - - // SUCCESS because stream is connected - // TODO: probably we need to change protocol to make one preventive read - // to get real status - UNIT_ASSERT_VALUES_EQUAL(it.GetStatus(), EStatus::SUCCESS); - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - - Cerr << streamPart.GetIssues().ToString() << Endl; - UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), false); - UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); - - UNIT_ASSERT_VALUES_EQUAL(it.ReadNext().GetValueSync().EOS(), true); - } - + void TestReadTableMultiShardWithDescribe(bool rowLimit) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Key2", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "Key2"}); + + TCreateTableSettings createTableSettings = + TCreateTableSettings() + .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(10)); + + auto result = session.CreateTable("Root/Test", tableBuilder.Build(), createTableSettings).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Key2, Value) VALUES + (1u, 2u, "A"), + (429496730u, 20000u, "B"), + (858993459u, 20000u, "C"), + (1288490188u, 20000u, "D"), + (3865470565u, 200000000u, "E"), + (3865470565u, 200000001u, "F"); + )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + TDescribeTableSettings describeTableSettings = + TDescribeTableSettings() + .WithKeyShardBoundary(true); + + TDescribeTableResult describeResult = session.DescribeTable("Root/Test", describeTableSettings) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + + TVector<TString> expected; + expected.push_back(R"___([[1u];[2u];["A"]])___"); + expected.push_back(R"___([[429496730u];[20000u];["B"]])___"); + expected.push_back(R"___([[858993459u];[20000u];["C"]])___"); + expected.push_back(R"___([[1288490188u];[20000u];["D"]])___"); + expected.push_back(R"___([[3865470565u];[200000000u];["E"]])___"); + expected.push_back(R"___([[3865470565u];[200000001u];["F"]])___"); + + int row = 0; + for (const auto& range : describeResult.GetTableDescription().GetKeyRanges()) { + TReadTableSettings readTableSettings; + if (auto from = range.From()) { + readTableSettings.From(from.GetRef()); + } + if (auto to = range.To()) { + readTableSettings.To(to.GetRef()); + } + if (rowLimit) { + readTableSettings.RowLimit(1); + } + + auto it = session.ReadTable("Root/Test", readTableSettings).ExtractValueSync(); + + while (true) { + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + + if (streamPart.EOS()) { + break; + } + + UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), true); + + auto rsParser = TResultSetParser(streamPart.ExtractPart()); + while (rsParser.TryNextRow()) { + auto columns = rsParser.ColumnsCount(); + const auto& expRow = expected[row++]; + TString tmp = "["; + for (size_t c = 0; c < columns; c++) { + auto colYson = FormatValueYson(rsParser.GetValue(c)); + tmp += colYson; + if (c != columns - 1) + tmp += ";"; + } + tmp += "]"; + UNIT_ASSERT_VALUES_EQUAL(tmp, expRow); + } + } + } + UNIT_ASSERT_VALUES_EQUAL(row, rowLimit ? 5 : expected.size()); + } + + Y_UNIT_TEST(TestReadTableMultiShardWithDescribe) { + TestReadTableMultiShardWithDescribe(false); + } + + Y_UNIT_TEST(TestReadTableMultiShardWithDescribeAndRowLimit) { + TestReadTableMultiShardWithDescribe(true); + } + + Y_UNIT_TEST(TestReadWrongTable) { + TKikimrWithGrpcAndRootSchema server; + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + auto it = session.ReadTable("Root/NoTable").ExtractValueSync(); + + // SUCCESS because stream is connected + // TODO: probably we need to change protocol to make one preventive read + // to get real status + UNIT_ASSERT_VALUES_EQUAL(it.GetStatus(), EStatus::SUCCESS); + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + + Cerr << streamPart.GetIssues().ToString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(streamPart.IsSuccess(), false); + UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR); + + UNIT_ASSERT_VALUES_EQUAL(it.ReadNext().GetValueSync().EOS(), true); + } + Y_UNIT_TEST(RetryOperation) { TKikimrWithGrpcAndRootSchema server; NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); @@ -2122,7 +2122,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 TResultSetParser parser(*selectResult); UNIT_ASSERT(parser.TryNextRow()); - driver.Stop(true); + driver.Stop(true); } Y_UNIT_TEST(QueryLimits) { @@ -2181,7 +2181,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); } - Y_UNIT_TEST(SessionsServerLimit) { + Y_UNIT_TEST(SessionsServerLimit) { NKikimrConfig::TAppConfig appConfig; auto& tableServiceConfig = *appConfig.MutableTableServiceConfig(); tableServiceConfig.SetSessionsLimitPerNode(2); @@ -2191,29 +2191,29 @@ R"___(<main>: Error: Transaction not found: , code: 2015 NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); NYdb::NTable::TTableClient client(driver); auto sessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); auto session1 = sessionResult.GetSession(); sessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); auto session2 = sessionResult.GetSession(); sessionResult = client.CreateSession().ExtractValueSync(); sessionResult.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::OVERLOADED); auto status = session1.Close().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); UNIT_ASSERT_VALUES_EQUAL(status.GetStatus(), EStatus::SUCCESS); auto result = session2.ExecuteDataQuery(R"___( SELECT 1; )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); sessionResult = client.CreateSession().ExtractValueSync(); @@ -2222,586 +2222,586 @@ R"___(<main>: Error: Transaction not found: , code: 2015 sessionResult = client.CreateSession().ExtractValueSync(); sessionResult.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); } - - Y_UNIT_TEST(SessionsServerLimitWithSessionPool) { - NKikimrConfig::TAppConfig appConfig; - auto& tableServiceConfig = *appConfig.MutableTableServiceConfig(); - tableServiceConfig.SetSessionsLimitPerNode(2); - - TKikimrWithGrpcAndRootSchema server(appConfig); - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - auto sessionResult1 = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - auto session1 = sessionResult1.GetSession(); - - auto sessionResult2 = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult2.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - auto session2 = sessionResult2.GetSession(); - - { - auto sessionResult3 = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult3.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 3); - } - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - - auto status = session1.Close().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(status.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(status.GetStatus(), EStatus::SUCCESS); - - // Close doesnt free session from user perspective, - // the value of ActiveSessionsCounter will be same after Close() call. - // Probably we want to chenge this contract - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - - auto result = session2.ExecuteDataQuery(R"___( - SELECT 1; - )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - sessionResult1 = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetSession().GetId().empty(), false); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 3); - - auto sessionResult3 = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult3.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 4); - - auto tmp = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 5); - sessionResult1 = tmp; // here we reset previous created session object, - // so perform close rpc call implicitly and delete it - UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::OVERLOADED); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 4); - } - - Y_UNIT_TEST(CloseSessionAfterDriverDtorWithoutSessionPool) { - NKikimrConfig::TAppConfig appConfig; - - TKikimrWithGrpcAndRootSchema server(appConfig); - - TVector<TString> sessionIds; - int iterations = 50; - - while (iterations--) { - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - auto sessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session1 = sessionResult.GetSession(); - sessionIds.push_back(session1.GetId()); - } - - std::shared_ptr<grpc::Channel> channel; - channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - auto stub = Ydb::Table::V1::TableService::NewStub(channel); - for (const auto& sessionId : sessionIds) { - grpc::ClientContext context; - Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); - Ydb::Table::KeepAliveResponse response; - auto status = stub->KeepAlive(&context, request, &response); + + Y_UNIT_TEST(SessionsServerLimitWithSessionPool) { + NKikimrConfig::TAppConfig appConfig; + auto& tableServiceConfig = *appConfig.MutableTableServiceConfig(); + tableServiceConfig.SetSessionsLimitPerNode(2); + + TKikimrWithGrpcAndRootSchema server(appConfig); + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + NYdb::NTable::TTableClient client(driver); + auto sessionResult1 = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + auto session1 = sessionResult1.GetSession(); + + auto sessionResult2 = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult2.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + auto session2 = sessionResult2.GetSession(); + + { + auto sessionResult3 = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult3.GetStatus(), EStatus::OVERLOADED); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 3); + } + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + + auto status = session1.Close().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(status.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(status.GetStatus(), EStatus::SUCCESS); + + // Close doesnt free session from user perspective, + // the value of ActiveSessionsCounter will be same after Close() call. + // Probably we want to chenge this contract + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + + auto result = session2.ExecuteDataQuery(R"___( + SELECT 1; + )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + sessionResult1 = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetSession().GetId().empty(), false); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 3); + + auto sessionResult3 = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult3.GetStatus(), EStatus::OVERLOADED); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 4); + + auto tmp = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 5); + sessionResult1 = tmp; // here we reset previous created session object, + // so perform close rpc call implicitly and delete it + UNIT_ASSERT_VALUES_EQUAL(sessionResult1.GetStatus(), EStatus::OVERLOADED); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 4); + } + + Y_UNIT_TEST(CloseSessionAfterDriverDtorWithoutSessionPool) { + NKikimrConfig::TAppConfig appConfig; + + TKikimrWithGrpcAndRootSchema server(appConfig); + + TVector<TString> sessionIds; + int iterations = 50; + + while (iterations--) { + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + NYdb::NTable::TTableClient client(driver); + auto sessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + auto session1 = sessionResult.GetSession(); + sessionIds.push_back(session1.GetId()); + } + + std::shared_ptr<grpc::Channel> channel; + channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); + auto stub = Ydb::Table::V1::TableService::NewStub(channel); + for (const auto& sessionId : sessionIds) { + grpc::ClientContext context; + Ydb::Table::KeepAliveRequest request; + request.set_session_id(sessionId); + Ydb::Table::KeepAliveResponse response; + auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_SESSION); - } - } - - Y_UNIT_TEST(CloseSessionWithSessionPoolExplicit) { - NKikimrConfig::TAppConfig appConfig; - - TKikimrWithGrpcAndRootSchema server(appConfig); - - TVector<TString> sessionIds; - int iterations = 100; - - while (iterations--) { - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - //TODO: remove this scope after session tracker implementation - { - auto sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session1 = sessionResult.GetSession(); - sessionIds.push_back(session1.GetId()); - - sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - // Here previous created session will be returnet to session pool - session1 = sessionResult.GetSession(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - sessionIds.push_back(session1.GetId()); - } - - if (RandomNumber<ui32>(10) == 5) { - client.Stop().Apply([client](NThreading::TFuture<void> future){ - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); - return future; - }).Wait(); - } else { - client.Stop().Wait(); - } - - if (iterations & 4) { - driver.Stop(true); - } - } - - std::shared_ptr<grpc::Channel> channel; - channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - auto stub = Ydb::Table::V1::TableService::NewStub(channel); - for (const auto& sessionId : sessionIds) { - grpc::ClientContext context; - Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); - Ydb::Table::KeepAliveResponse response; - auto status = stub->KeepAlive(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_SESSION); + } + } + + Y_UNIT_TEST(CloseSessionWithSessionPoolExplicit) { + NKikimrConfig::TAppConfig appConfig; + + TKikimrWithGrpcAndRootSchema server(appConfig); + + TVector<TString> sessionIds; + int iterations = 100; + + while (iterations--) { + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + NYdb::NTable::TTableClient client(driver); + //TODO: remove this scope after session tracker implementation + { + auto sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + auto session1 = sessionResult.GetSession(); + sessionIds.push_back(session1.GetId()); + + sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + // Here previous created session will be returnet to session pool + session1 = sessionResult.GetSession(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + sessionIds.push_back(session1.GetId()); + } + + if (RandomNumber<ui32>(10) == 5) { + client.Stop().Apply([client](NThreading::TFuture<void> future){ + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 0); + return future; + }).Wait(); + } else { + client.Stop().Wait(); + } + + if (iterations & 4) { + driver.Stop(true); + } + } + + std::shared_ptr<grpc::Channel> channel; + channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); + auto stub = Ydb::Table::V1::TableService::NewStub(channel); + for (const auto& sessionId : sessionIds) { + grpc::ClientContext context; + Ydb::Table::KeepAliveRequest request; + request.set_session_id(sessionId); + Ydb::Table::KeepAliveResponse response; + auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); - } - } - - Y_UNIT_TEST(CloseSessionWithSessionPoolExplicitDriverStopOnly) { - NKikimrConfig::TAppConfig appConfig; - - TKikimrWithGrpcAndRootSchema server(appConfig); - - TVector<TString> sessionIds; - int iterations = 100; - - while (iterations--) { - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - //TODO: remove this scope after session tracker implementation - { - auto sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session1 = sessionResult.GetSession(); - sessionIds.push_back(session1.GetId()); - - sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - // Here previous created session will be returnet to session pool - session1 = sessionResult.GetSession(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - sessionIds.push_back(session1.GetId()); - } - driver.Stop(true); - } - - std::shared_ptr<grpc::Channel> channel; - channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - auto stub = Ydb::Table::V1::TableService::NewStub(channel); - for (const auto& sessionId : sessionIds) { - grpc::ClientContext context; - Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); - Ydb::Table::KeepAliveResponse response; - auto status = stub->KeepAlive(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); + } + } + + Y_UNIT_TEST(CloseSessionWithSessionPoolExplicitDriverStopOnly) { + NKikimrConfig::TAppConfig appConfig; + + TKikimrWithGrpcAndRootSchema server(appConfig); + + TVector<TString> sessionIds; + int iterations = 100; + + while (iterations--) { + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + NYdb::NTable::TTableClient client(driver); + //TODO: remove this scope after session tracker implementation + { + auto sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + auto session1 = sessionResult.GetSession(); + sessionIds.push_back(session1.GetId()); + + sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + // Here previous created session will be returnet to session pool + session1 = sessionResult.GetSession(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + sessionIds.push_back(session1.GetId()); + } + driver.Stop(true); + } + + std::shared_ptr<grpc::Channel> channel; + channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); + auto stub = Ydb::Table::V1::TableService::NewStub(channel); + for (const auto& sessionId : sessionIds) { + grpc::ClientContext context; + Ydb::Table::KeepAliveRequest request; + request.set_session_id(sessionId); + Ydb::Table::KeepAliveResponse response; + auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); - } - } - - Y_UNIT_TEST(CloseSessionWithSessionPoolFromDtors) { - NKikimrConfig::TAppConfig appConfig; - - TKikimrWithGrpcAndRootSchema server(appConfig); - - TVector<TString> sessionIds; - int iterations = 100; - - while (iterations--) { - NYdb::TDriver driver(TDriverConfig() - .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - //TODO: remove this scope after session tracker implementation - { - auto sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session1 = sessionResult.GetSession(); - sessionIds.push_back(session1.GetId()); - - sessionResult = client.GetSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - // Here previous created session will be returnet to session pool - session1 = sessionResult.GetSession(); - UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); - sessionIds.push_back(session1.GetId()); - } - } - - std::shared_ptr<grpc::Channel> channel; - channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - auto stub = Ydb::Table::V1::TableService::NewStub(channel); - for (const auto& sessionId : sessionIds) { - grpc::ClientContext context; - Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); - Ydb::Table::KeepAliveResponse response; - auto status = stub->KeepAlive(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); + } + } + + Y_UNIT_TEST(CloseSessionWithSessionPoolFromDtors) { + NKikimrConfig::TAppConfig appConfig; + + TKikimrWithGrpcAndRootSchema server(appConfig); + + TVector<TString> sessionIds; + int iterations = 100; + + while (iterations--) { + NYdb::TDriver driver(TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + NYdb::NTable::TTableClient client(driver); + //TODO: remove this scope after session tracker implementation + { + auto sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + auto session1 = sessionResult.GetSession(); + sessionIds.push_back(session1.GetId()); + + sessionResult = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + // Here previous created session will be returnet to session pool + session1 = sessionResult.GetSession(); + UNIT_ASSERT_VALUES_EQUAL(client.GetActiveSessionCount(), 1); + sessionIds.push_back(session1.GetId()); + } + } + + std::shared_ptr<grpc::Channel> channel; + channel = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); + auto stub = Ydb::Table::V1::TableService::NewStub(channel); + for (const auto& sessionId : sessionIds) { + grpc::ClientContext context; + Ydb::Table::KeepAliveRequest request; + request.set_session_id(sessionId); + Ydb::Table::KeepAliveResponse response; + auto status = stub->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); - } - } - - Y_UNIT_TEST(DeleteTableWithDeletedIndex) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver( - TDriverConfig() - .SetEndpoint( - TStringBuilder() << "localhost:" << server.GetPort()) - ); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - - { - - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .AddNullableColumn("uid", EPrimitiveType::Uint64) - .SetPrimaryKeyColumn("key") - .AddSecondaryIndex("uid", "uid"); - - auto desc = builder.Build(); - - auto result = session.CreateTable("Root/Test", - std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendDropIndexes({"uid"}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_SESSION); + } + } + + Y_UNIT_TEST(DeleteTableWithDeletedIndex) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver( + TDriverConfig() + .SetEndpoint( + TStringBuilder() << "localhost:" << server.GetPort()) + ); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + + { + + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .AddNullableColumn("uid", EPrimitiveType::Uint64) + .SetPrimaryKeyColumn("key") + .AddSecondaryIndex("uid", "uid"); + + auto desc = builder.Build(); + + auto result = session.CreateTable("Root/Test", + std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendDropIndexes({"uid"}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { auto result = session.DropTable("Root/Test").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - } - - Y_UNIT_TEST(AlterTableAddIndex) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver( - TDriverConfig() - .SetEndpoint( - TStringBuilder() << "localhost:" << server.GetPort()) - ); - - { - NYdb::NOperation::TOperationClient operationClient(driver); - auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 0); // No operations in progress - } - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); - )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("", {"Value"})}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("SomeName", TVector<TString>())}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); - - auto result = session.AlterTable("", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - } - - { - - NYdb::NTable::TClientSettings clientSettings; - clientSettings.AuthToken("badguy@builtin"); - NYdb::NTable::TTableClient clientbad(driver, clientSettings); - auto getSessionResult = clientbad.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNAUTHORIZED, result.GetIssues().ToString()); - - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); - - auto result = session.AlterTable("/Root/WrongPath", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); - } - - { - NYdb::NOperation::TOperationClient operationClient(driver); - auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 1); - auto op = result.GetList()[0]; - UNIT_ASSERT_VALUES_EQUAL(op.Ready(), true); - UNIT_ASSERT_VALUES_EQUAL(op.Status().GetStatus(), EStatus::SUCCESS); - auto meta = op.Metadata(); + } + + Y_UNIT_TEST(AlterTableAddIndex) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver( + TDriverConfig() + .SetEndpoint( + TStringBuilder() << "localhost:" << server.GetPort()) + ); + + { + NYdb::NOperation::TOperationClient operationClient(driver); + auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 0); // No operations in progress + } + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); + )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("", {"Value"})}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("SomeName", TVector<TString>())}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); + + auto result = session.AlterTable("", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + } + + { + + NYdb::NTable::TClientSettings clientSettings; + clientSettings.AuthToken("badguy@builtin"); + NYdb::NTable::TTableClient clientbad(driver, clientSettings); + auto getSessionResult = clientbad.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNAUTHORIZED, result.GetIssues().ToString()); + + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); + + auto result = session.AlterTable("/Root/WrongPath", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); + } + + { + NYdb::NOperation::TOperationClient operationClient(driver); + auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 1); + auto op = result.GetList()[0]; + UNIT_ASSERT_VALUES_EQUAL(op.Ready(), true); + UNIT_ASSERT_VALUES_EQUAL(op.Status().GetStatus(), EStatus::SUCCESS); + auto meta = op.Metadata(); UNIT_ASSERT_VALUES_EQUAL(meta.State, NYdb::NTable::EBuildIndexState::Done); - UNIT_ASSERT_DOUBLES_EQUAL(meta.Progress, 100, 0.001); - - UNIT_ASSERT_VALUES_EQUAL(meta.Path, "/Root/Test"); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexName(), "NewIndex"); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns()[0], "Value"); - - - auto result2 = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.GetList()[0].Id()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result2.Status().GetStatus(), EStatus::SUCCESS, result2.Status().GetIssues().ToString()); + UNIT_ASSERT_DOUBLES_EQUAL(meta.Progress, 100, 0.001); + + UNIT_ASSERT_VALUES_EQUAL(meta.Path, "/Root/Test"); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexName(), "NewIndex"); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(meta.Desctiption.GetRef().GetIndexColumns()[0], "Value"); + + + auto result2 = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.GetList()[0].Id()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result2.Status().GetStatus(), EStatus::SUCCESS, result2.Status().GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().State, NYdb::NTable::EBuildIndexState::Done); - UNIT_ASSERT_DOUBLES_EQUAL(result2.Metadata().Progress, 100, 0.001); - - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Path, "/Root/Test"); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexName(), "NewIndex"); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns()[0], "Value"); - - { - // Cancel already finished operation do nothing - auto result3 = operationClient.Cancel(result.GetList()[0].Id()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::PRECONDITION_FAILED, result3.GetIssues().ToString()); - } - - { - auto result3 = operationClient.Forget(result.GetList()[0].Id()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::SUCCESS, result3.GetIssues().ToString()); - } - - { - auto result3 = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.GetList()[0].Id()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result3.Status().GetStatus(), EStatus::PRECONDITION_FAILED, result3.Status().GetIssues().ToString()); - } - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendDropIndexes({"NewIndex"}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - TDescribeTableResult describeResult = session.DescribeTable("Root/Test") - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetIndexDescriptions().size(), 0); - } - } - - Y_UNIT_TEST(AlterTableAddIndexAsyncOp) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver( - TDriverConfig() - .SetEndpoint( - TStringBuilder() << "localhost:" << server.GetPort()) - ); - - { - NYdb::NOperation::TOperationClient operationClient(driver); - auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 0); // No operations in progress - } - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); - )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); - - auto result = session.AlterTableLong("/Root/Test", settings).ExtractValueSync(); - - // Build index is async operation - UNIT_ASSERT(!result.Ready()); - - NYdb::NOperation::TOperationClient operationClient(driver); - - for (;;) { - auto getResult = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.Id()).GetValueSync(); - if (getResult.Ready()) { - UNIT_ASSERT_VALUES_EQUAL_C(getResult.Status().GetStatus(), EStatus::SUCCESS, getResult.Status().GetIssues().ToString()); - break; - } else { - Sleep(TDuration::MilliSeconds(100)); - } - } - } - - // Add column in to table with index - { - auto type = TTypeBuilder().BeginOptional().Primitive(EPrimitiveType::Uint64).EndOptional().Build(); - auto alter = TAlterTableSettings().AppendAddColumns(TColumn("NewColumn", type)); - - auto result = session.AlterTableLong("/Root/Test", alter).ExtractValueSync(); - - // Build index is async operation - UNIT_ASSERT(result.Ready()); - - UNIT_ASSERT_VALUES_EQUAL_C(result.Status().GetStatus(), EStatus::SUCCESS, result.Status().GetIssues().ToString()); - } - } - - Y_UNIT_TEST(AlterTableAddIndexWithDataColumn) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver( - TDriverConfig() - .SetEndpoint( - TStringBuilder() << "localhost:" << server.GetPort()) - ); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - - { - auto result = session.ExecuteSchemeQuery(R"___( - CREATE TABLE [Root/Test] ( - Key Uint64, - Fk Uint64, - Value String, - PRIMARY KEY (Key) - ); - )___").ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Test] (Key, Fk, Value) VALUES (1u, 111u, "One"); - )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendAddIndexes({TIndexDescription("NewIndex", {"Fk"}, {"Value"})}); - - auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto res = session.DescribeTable("Root/Test").ExtractValueSync(); - UNIT_ASSERT_EQUAL(res.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), EStatus::SUCCESS); - auto columns = res.GetTableDescription().GetTableColumns(); - UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); - const auto& indexDesc = res.GetTableDescription().GetIndexDescriptions(); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); - const auto& index = indexDesc[0]; - UNIT_ASSERT_VALUES_EQUAL(index.GetIndexName(), "NewIndex"); - UNIT_ASSERT_VALUES_EQUAL(index.GetIndexColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(index.GetIndexColumns()[0], "Fk"); - UNIT_ASSERT_VALUES_EQUAL(index.GetDataColumns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(index.GetDataColumns()[0], "Value"); - } - - auto it = session.ReadTable("/Root/Test/NewIndex/indexImplTable").ExtractValueSync(); - TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); - auto str = NYdb::FormatResultSetYson(streamPart.ExtractPart()); - - UNIT_ASSERT_VALUES_EQUAL(str, "[[[111u];[1u];[\"One\"]]]"); - } - - - + UNIT_ASSERT_DOUBLES_EQUAL(result2.Metadata().Progress, 100, 0.001); + + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Path, "/Root/Test"); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexName(), "NewIndex"); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result2.Metadata().Desctiption.GetRef().GetIndexColumns()[0], "Value"); + + { + // Cancel already finished operation do nothing + auto result3 = operationClient.Cancel(result.GetList()[0].Id()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::PRECONDITION_FAILED, result3.GetIssues().ToString()); + } + + { + auto result3 = operationClient.Forget(result.GetList()[0].Id()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::SUCCESS, result3.GetIssues().ToString()); + } + + { + auto result3 = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.GetList()[0].Id()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result3.Status().GetStatus(), EStatus::PRECONDITION_FAILED, result3.Status().GetIssues().ToString()); + } + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendDropIndexes({"NewIndex"}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + TDescribeTableResult describeResult = session.DescribeTable("Root/Test") + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetIndexDescriptions().size(), 0); + } + } + + Y_UNIT_TEST(AlterTableAddIndexAsyncOp) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver( + TDriverConfig() + .SetEndpoint( + TStringBuilder() << "localhost:" << server.GetPort()) + ); + + { + NYdb::NOperation::TOperationClient operationClient(driver); + auto result = operationClient.List<NYdb::NTable::TBuildIndexOperation>().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetList().size(), 0); // No operations in progress + } + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Value) VALUES (1u, "One"); + )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Value"})}); + + auto result = session.AlterTableLong("/Root/Test", settings).ExtractValueSync(); + + // Build index is async operation + UNIT_ASSERT(!result.Ready()); + + NYdb::NOperation::TOperationClient operationClient(driver); + + for (;;) { + auto getResult = operationClient.Get<NYdb::NTable::TBuildIndexOperation>(result.Id()).GetValueSync(); + if (getResult.Ready()) { + UNIT_ASSERT_VALUES_EQUAL_C(getResult.Status().GetStatus(), EStatus::SUCCESS, getResult.Status().GetIssues().ToString()); + break; + } else { + Sleep(TDuration::MilliSeconds(100)); + } + } + } + + // Add column in to table with index + { + auto type = TTypeBuilder().BeginOptional().Primitive(EPrimitiveType::Uint64).EndOptional().Build(); + auto alter = TAlterTableSettings().AppendAddColumns(TColumn("NewColumn", type)); + + auto result = session.AlterTableLong("/Root/Test", alter).ExtractValueSync(); + + // Build index is async operation + UNIT_ASSERT(result.Ready()); + + UNIT_ASSERT_VALUES_EQUAL_C(result.Status().GetStatus(), EStatus::SUCCESS, result.Status().GetIssues().ToString()); + } + } + + Y_UNIT_TEST(AlterTableAddIndexWithDataColumn) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver( + TDriverConfig() + .SetEndpoint( + TStringBuilder() << "localhost:" << server.GetPort()) + ); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"___( + CREATE TABLE [Root/Test] ( + Key Uint64, + Fk Uint64, + Value String, + PRIMARY KEY (Key) + ); + )___").ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Test] (Key, Fk, Value) VALUES (1u, 111u, "One"); + )___", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendAddIndexes({TIndexDescription("NewIndex", {"Fk"}, {"Value"})}); + + auto result = session.AlterTable("/Root/Test", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto res = session.DescribeTable("Root/Test").ExtractValueSync(); + UNIT_ASSERT_EQUAL(res.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), EStatus::SUCCESS); + auto columns = res.GetTableDescription().GetTableColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); + const auto& indexDesc = res.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); + const auto& index = indexDesc[0]; + UNIT_ASSERT_VALUES_EQUAL(index.GetIndexName(), "NewIndex"); + UNIT_ASSERT_VALUES_EQUAL(index.GetIndexColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(index.GetIndexColumns()[0], "Fk"); + UNIT_ASSERT_VALUES_EQUAL(index.GetDataColumns().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(index.GetDataColumns()[0], "Value"); + } + + auto it = session.ReadTable("/Root/Test/NewIndex/indexImplTable").ExtractValueSync(); + TReadTableResultPart streamPart = it.ReadNext().GetValueSync(); + auto str = NYdb::FormatResultSetYson(streamPart.ExtractPart()); + + UNIT_ASSERT_VALUES_EQUAL(str, "[[[111u];[1u];[\"One\"]]]"); + } + + + Y_UNIT_TEST(QueryStats) { TKikimrWithGrpcAndRootSchema server; @@ -3703,7 +3703,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 "Status: " << result.GetStatus() << " Issues: " << TPrintableIssues(result.GetIssues())); } } - + Y_UNIT_TEST(ColumnFamiliesDescriptionWithStorageAndIndex) { TKikimrWithGrpcAndRootSchema server; server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_NOTICE); @@ -3831,70 +3831,70 @@ R"___(<main>: Error: Transaction not found: , code: 2015 } } - Y_UNIT_TEST(TestDescribeTableWithShardStats) { - TKikimrWithGrpcAndRootSchema server; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(connection); - - auto sessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session = sessionResult.GetSession(); - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .AddNullableColumn("Value", EPrimitiveType::Utf8); - tableBuilder.SetPrimaryKeyColumn("Key"); - - auto tableSettings = NYdb::NTable::TCreateTableSettings().PartitioningPolicy( - NYdb::NTable::TPartitioningPolicy().UniformPartitions(2)); - - auto result = session.CreateTable("/Root/Foo", tableBuilder.Build(), tableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = session.ExecuteDataQuery(R"___( - UPSERT INTO [Root/Foo] (Key, Value) - VALUES(1, "bar"); - )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - TDescribeTableSettings describeTableSettings = - TDescribeTableSettings() - .WithTableStatistics(true); - - auto res = session.DescribeTable("Root/Foo", describeTableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(res.IsTransportError(), false); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionsCount(), 2); - //only table statistics - UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionStats().size(), 0); - } - - { - TDescribeTableSettings describeTableSettings = - TDescribeTableSettings() - .WithTableStatistics(true) - .WithPartitionStatistics(true); - - auto res = session.DescribeTable("Root/Foo", describeTableSettings).ExtractValueSync(); - UNIT_ASSERT_EQUAL(res.IsTransportError(), false); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionsCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionStats().size(), 2); - } + Y_UNIT_TEST(TestDescribeTableWithShardStats) { + TKikimrWithGrpcAndRootSchema server; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(connection); + + auto sessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); + auto session = sessionResult.GetSession(); + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::Utf8); + tableBuilder.SetPrimaryKeyColumn("Key"); + + auto tableSettings = NYdb::NTable::TCreateTableSettings().PartitioningPolicy( + NYdb::NTable::TPartitioningPolicy().UniformPartitions(2)); + + auto result = session.CreateTable("/Root/Foo", tableBuilder.Build(), tableSettings).ExtractValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = session.ExecuteDataQuery(R"___( + UPSERT INTO [Root/Foo] (Key, Value) + VALUES(1, "bar"); + )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + TDescribeTableSettings describeTableSettings = + TDescribeTableSettings() + .WithTableStatistics(true); + + auto res = session.DescribeTable("Root/Foo", describeTableSettings).ExtractValueSync(); + UNIT_ASSERT_EQUAL(res.IsTransportError(), false); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionsCount(), 2); + //only table statistics + UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionStats().size(), 0); + } + + { + TDescribeTableSettings describeTableSettings = + TDescribeTableSettings() + .WithTableStatistics(true) + .WithPartitionStatistics(true); + + auto res = session.DescribeTable("Root/Foo", describeTableSettings).ExtractValueSync(); + UNIT_ASSERT_EQUAL(res.IsTransportError(), false); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionsCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionStats().size(), 2); + } } - + Y_UNIT_TEST(TestExplicitPartitioning) { TKikimrWithGrpcAndRootSchema server; @@ -3959,7 +3959,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 } EnsureTablePartitions(client, "/Root/Foo", 3); - } + } Y_UNIT_TEST(CreateAndAltertTableWithCompactionPolicy) { TKikimrWithGrpcAndRootSchema server; @@ -4610,4 +4610,4 @@ R"___(<main>: Error: Transaction not found: , code: 2015 UNIT_ASSERT(!keyRanges[0].To()); } } -} +} diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index 424244540e..2d37063079 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -1,45 +1,45 @@ #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - -#include <grpc++/client_context.h> -#include <grpc++/create_channel.h> - + +#include <grpc++/client_context.h> +#include <grpc++/create_channel.h> + #include <ydb/core/base/storage_pools.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/core/testlib/test_client.h> - + #include <ydb/public/api/grpc/ydb_scheme_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/api/grpc/draft/dummy.grpc.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> - + #include <google/protobuf/any.h> - + #include <ydb/library/yql/core/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - -#include "ydb_common_ut.h" - + +#include "ydb_common_ut.h" + #include <util/generic/ymath.h> -namespace NKikimr { - -using namespace Tests; +namespace NKikimr { + +using namespace Tests; using namespace NYdb; using namespace NYdb::NTable; using namespace NYdb::NScheme; - + static bool HasIssue(const NYql::TIssues& issues, ui32 code, std::function<bool(const NYql::TIssue& issue)> predicate = {}) { @@ -59,56 +59,56 @@ static bool HasIssue(const NYql::TIssues& issues, ui32 code, return hasIssue; } -Ydb::Table::DescribeTableResult DescribeTable(std::shared_ptr<grpc::Channel> channel, const TString& sessionId, const TString& path) -{ - Ydb::Table::DescribeTableRequest request; - request.set_session_id(sessionId); - request.set_path(path); - request.set_include_shard_key_bounds(true); - - Ydb::Table::DescribeTableResponse response; - - std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; - stub = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - auto status = stub->DescribeTable(&context, request, &response); - UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - issues.PrintTo(Cerr); - - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - - Ydb::Table::DescribeTableResult result; - Y_VERIFY(deferred.result().UnpackTo(&result)); - return result; -} - - -Ydb::Table::ExecuteQueryResult ExecYql(std::shared_ptr<grpc::Channel> channel, const TString &sessionId, const TString &yql, bool withStat = false); - +Ydb::Table::DescribeTableResult DescribeTable(std::shared_ptr<grpc::Channel> channel, const TString& sessionId, const TString& path) +{ + Ydb::Table::DescribeTableRequest request; + request.set_session_id(sessionId); + request.set_path(path); + request.set_include_shard_key_bounds(true); + + Ydb::Table::DescribeTableResponse response; + + std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; + stub = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + auto status = stub->DescribeTable(&context, request, &response); + UNIT_ASSERT(status.ok()); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + issues.PrintTo(Cerr); + + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + + Ydb::Table::DescribeTableResult result; + Y_VERIFY(deferred.result().UnpackTo(&result)); + return result; +} + + +Ydb::Table::ExecuteQueryResult ExecYql(std::shared_ptr<grpc::Channel> channel, const TString &sessionId, const TString &yql, bool withStat = false); + static Ydb::StatusIds::StatusCode WaitForStatus(std::shared_ptr<grpc::Channel> channel, const TString& opId) { - std::unique_ptr<Ydb::Operation::V1::OperationService::Stub> stub; - stub = Ydb::Operation::V1::OperationService::NewStub(channel); - Ydb::Operations::GetOperationRequest request; - request.set_id(opId); - Ydb::Operations::GetOperationResponse response; - bool run = true; - while (run) { - grpc::ClientContext context; - auto status = stub->GetOperation(&context, request, &response); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - if (response.operation().ready() == false) { - Sleep(ITERATION_DURATION); - } else { - run = false; - } - } - return response.operation().status(); -} - + std::unique_ptr<Ydb::Operation::V1::OperationService::Stub> stub; + stub = Ydb::Operation::V1::OperationService::NewStub(channel); + Ydb::Operations::GetOperationRequest request; + request.set_id(opId); + Ydb::Operations::GetOperationResponse response; + bool run = true; + while (run) { + grpc::ClientContext context; + auto status = stub->GetOperation(&context, request, &response); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + if (response.operation().ready() == false) { + Sleep(ITERATION_DURATION); + } else { + run = false; + } + } + return response.operation().status(); +} + struct TKikimrTestSettings { static constexpr bool SSL = false; static constexpr bool AUTH = false; @@ -117,102 +117,102 @@ struct TKikimrTestSettings { Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { Y_UNIT_TEST(SimpleRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - bool allDoneOk = false; - - { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + bool allDoneOk = false; + + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); - - Ydb::Table::CreateSessionRequest request; - + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); + + Ydb::Table::CreateSessionRequest request; + NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - allDoneOk = true; - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession); - } - UNIT_ASSERT(allDoneOk); - } - - Y_UNIT_TEST(SimpleRequestDummyService) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - bool allDoneOk = false; - - { + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + allDoneOk = true; + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession); + } + UNIT_ASSERT(allDoneOk); + } + + Y_UNIT_TEST(SimpleRequestDummyService) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + bool allDoneOk = false; + + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Draft::Dummy::DummyService>(clientConfig); - - Draft::Dummy::PingRequest request; - request.set_copy(true); - request.set_payload("abc"); - + auto connection = clientLow.CreateGRpcServiceConnection<Draft::Dummy::DummyService>(clientConfig); + + Draft::Dummy::PingRequest request; + request.set_copy(true); + request.set_payload("abc"); + NGrpc::TResponseCallback<Draft::Dummy::PingResponse> responseCb = [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Draft::Dummy::PingResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - UNIT_ASSERT(response.payload() == "abc"); - allDoneOk = true; - }; - - connection->DoRequest(request, std::move(responseCb), &Draft::Dummy::DummyService::Stub::AsyncPing); - } - UNIT_ASSERT(allDoneOk); - } - - Y_UNIT_TEST(GrpcRequestProxy) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); - TKikimrWithGrpcAndRootSchemaWithAuth server(appConfig); - - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - auto doTest = [&](const TString& database) { - NGrpc::TCallMeta meta; - meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, "root@builtin"}); - meta.Aux.push_back({YDB_DATABASE_HEADER, database}); - - NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); - - Ydb::StatusIds::StatusCode status; - int gStatus; - - do { - auto promise = NThreading::NewPromise<void>(); - Ydb::Table::CreateSessionRequest request; - NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + UNIT_ASSERT(response.payload() == "abc"); + allDoneOk = true; + }; + + connection->DoRequest(request, std::move(responseCb), &Draft::Dummy::DummyService::Stub::AsyncPing); + } + UNIT_ASSERT(allDoneOk); + } + + Y_UNIT_TEST(GrpcRequestProxy) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); + TKikimrWithGrpcAndRootSchemaWithAuth server(appConfig); + + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + auto doTest = [&](const TString& database) { + NGrpc::TCallMeta meta; + meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, "root@builtin"}); + meta.Aux.push_back({YDB_DATABASE_HEADER, database}); + + NGrpc::TGRpcClientLow clientLow; + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); + + Ydb::StatusIds::StatusCode status; + int gStatus; + + do { + auto promise = NThreading::NewPromise<void>(); + Ydb::Table::CreateSessionRequest request; + NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = [&status, &gStatus, promise](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) mutable { - UNIT_ASSERT(!grpcStatus.InternalError); - gStatus = grpcStatus.GRpcStatusCode; - auto deferred = response.operation(); - status = deferred.status(); - promise.SetValue(); - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession, meta); + UNIT_ASSERT(!grpcStatus.InternalError); + gStatus = grpcStatus.GRpcStatusCode; + auto deferred = response.operation(); + status = deferred.status(); + promise.SetValue(); + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession, meta); promise.GetFuture().Wait(); - } while (status == Ydb::StatusIds::UNAVAILABLE); - return std::make_pair(status, gStatus); - }; - + } while (status == Ydb::StatusIds::UNAVAILABLE); + return std::make_pair(status, gStatus); + }; + UNIT_ASSERT_VALUES_EQUAL(doTest("/Root"), std::make_pair(Ydb::StatusIds::SUCCESS, 0)); UNIT_ASSERT_VALUES_EQUAL(doTest("/blabla"), std::make_pair(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED, 16)); UNIT_ASSERT_VALUES_EQUAL(doTest("blabla"), std::make_pair(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED, 16)); - } - + } + Y_UNIT_TEST(GrpcRequestProxyWithoutToken) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); @@ -254,64 +254,64 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { UNIT_ASSERT_EQUAL(doTest("blabla").second, grpc::StatusCode::UNAUTHENTICATED); } - Y_UNIT_TEST(BiStreamPing) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); + Y_UNIT_TEST(BiStreamPing) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); TKikimrWithGrpcAndRootSchemaWithAuth server(appConfig); - - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - NGrpc::TCallMeta meta; - meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, "root@builtin"}); - { - using TRequest = Draft::Dummy::PingRequest; - using TResponse = Draft::Dummy::PingResponse; - using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr; - NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Draft::Dummy::DummyService>(clientConfig); - - auto promise = NThreading::NewPromise<void>(); - auto finishCb = [promise](NGrpc::TGrpcStatus&& status) mutable { - UNIT_ASSERT_EQUAL(status.GRpcStatusCode, grpc::StatusCode::UNAUTHENTICATED); - promise.SetValue(); - }; - - TResponse response; - auto getReadCb = [finishCb](TProcessor processor) { - auto readCb = [finishCb, processor](NGrpc::TGrpcStatus&& status) { - UNIT_ASSERT(status.Ok()); - processor->Finish(finishCb); - }; - return readCb; - }; - - auto lowCallback = [getReadCb, &response] - (NGrpc::TGrpcStatus grpcStatus, TProcessor processor) mutable { - UNIT_ASSERT(grpcStatus.Ok()); - Draft::Dummy::PingRequest request; - request.set_copy(true); - request.set_payload("abc"); - - processor->Write(std::move(request)); - - processor->Read(&response, getReadCb(processor)); - }; - - connection->DoStreamRequest<TRequest, TResponse>( - std::move(lowCallback), - &Draft::Dummy::DummyService::Stub::AsyncBiStreamPing, - std::move(meta)); - - promise.GetFuture().Wait(); - } - } - + + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + NGrpc::TCallMeta meta; + meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, "root@builtin"}); + { + using TRequest = Draft::Dummy::PingRequest; + using TResponse = Draft::Dummy::PingResponse; + using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr; + NGrpc::TGRpcClientLow clientLow; + auto connection = clientLow.CreateGRpcServiceConnection<Draft::Dummy::DummyService>(clientConfig); + + auto promise = NThreading::NewPromise<void>(); + auto finishCb = [promise](NGrpc::TGrpcStatus&& status) mutable { + UNIT_ASSERT_EQUAL(status.GRpcStatusCode, grpc::StatusCode::UNAUTHENTICATED); + promise.SetValue(); + }; + + TResponse response; + auto getReadCb = [finishCb](TProcessor processor) { + auto readCb = [finishCb, processor](NGrpc::TGrpcStatus&& status) { + UNIT_ASSERT(status.Ok()); + processor->Finish(finishCb); + }; + return readCb; + }; + + auto lowCallback = [getReadCb, &response] + (NGrpc::TGrpcStatus grpcStatus, TProcessor processor) mutable { + UNIT_ASSERT(grpcStatus.Ok()); + Draft::Dummy::PingRequest request; + request.set_copy(true); + request.set_payload("abc"); + + processor->Write(std::move(request)); + + processor->Read(&response, getReadCb(processor)); + }; + + connection->DoStreamRequest<TRequest, TResponse>( + std::move(lowCallback), + &Draft::Dummy::DummyService::Stub::AsyncBiStreamPing, + std::move(meta)); + + promise.GetFuture().Wait(); + } + } + Y_UNIT_TEST(BiStreamCancelled) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableDomainsConfig()->MutableSecurityConfig()->SetEnforceUserTokenRequirement(true); TKikimrWithGrpcAndRootSchemaWithAuth server(appConfig); - + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); @@ -353,160 +353,160 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { } Y_UNIT_TEST(MultipleSimpleRequests) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - TAtomic okResponseCounter = 0; - const size_t numRequests = 1000; - - { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + TAtomic okResponseCounter = 0; + const size_t numRequests = 1000; + + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); - - Ydb::Table::CreateSessionRequest request; - for (size_t i = 0; i < numRequests; i++) { + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); + + Ydb::Table::CreateSessionRequest request; + for (size_t i = 0; i < numRequests; i++) { NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = [&okResponseCounter](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - AtomicIncrement(okResponseCounter); - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession); - } - } - UNIT_ASSERT(numRequests == AtomicGet(okResponseCounter)); - } - - Y_UNIT_TEST(ChangeAcl) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); - - { + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + AtomicIncrement(okResponseCounter); + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncCreateSession); + } + } + UNIT_ASSERT(numRequests == AtomicGet(okResponseCounter)); + } + + Y_UNIT_TEST(ChangeAcl) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); + + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); - - Ydb::Scheme::MakeDirectoryRequest request; - TString scheme( - "path: \"/Root/TheDirectory\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); + + Ydb::Scheme::MakeDirectoryRequest request; + TString scheme( + "path: \"/Root/TheDirectory\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + NGrpc::TResponseCallback<Ydb::Scheme::MakeDirectoryResponse> responseCb = [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::MakeDirectoryResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncMakeDirectory); - } - { + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncMakeDirectory); + } + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); - - Ydb::Scheme::ModifyPermissionsRequest request; - request.set_path("/Root/TheDirectory"); - request.Addactions()->Setchange_owner("qqq"); - - auto perm = request.Addactions()->mutable_set(); - perm->set_subject("qqq"); - perm->add_permission_names("ydb.generic.read"); - - + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); + + Ydb::Scheme::ModifyPermissionsRequest request; + request.set_path("/Root/TheDirectory"); + request.Addactions()->Setchange_owner("qqq"); + + auto perm = request.Addactions()->mutable_set(); + perm->set_subject("qqq"); + perm->add_permission_names("ydb.generic.read"); + + NGrpc::TResponseCallback<Ydb::Scheme::ModifyPermissionsResponse> responseCb = [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::ModifyPermissionsResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncModifyPermissions); - } - { + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncModifyPermissions); + } + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); - - Ydb::Scheme::DescribePathRequest request; - TString scheme( - "path: \"/Root/TheDirectory\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); + + Ydb::Scheme::DescribePathRequest request; + TString scheme( + "path: \"/Root/TheDirectory\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + NGrpc::TResponseCallback<Ydb::Scheme::DescribePathResponse> responseCb = [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::DescribePathResponse&& response) -> void { - UNIT_ASSERT(!grpcStatus.InternalError); - UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Scheme::DescribePathResult result; - deferred.result().UnpackTo(&result); - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = R"___(self { - name: "TheDirectory" - owner: "qqq" - type: DIRECTORY - effective_permissions { - subject: "qqq" - permission_names: "ydb.generic.read" - } - permissions { - subject: "qqq" - permission_names: "ydb.generic.read" - } -} -)___"; - UNIT_ASSERT_NO_DIFF(expected, tmp); - - }; - - connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncDescribePath); - } - } -} - + UNIT_ASSERT(!grpcStatus.InternalError); + UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Scheme::DescribePathResult result; + deferred.result().UnpackTo(&result); + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = R"___(self { + name: "TheDirectory" + owner: "qqq" + type: DIRECTORY + effective_permissions { + subject: "qqq" + permission_names: "ydb.generic.read" + } + permissions { + subject: "qqq" + permission_names: "ydb.generic.read" + } +} +)___"; + UNIT_ASSERT_NO_DIFF(expected, tmp); + + }; + + connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncDescribePath); + } + } +} + Y_UNIT_TEST_SUITE(TGRpcNewClient) { Y_UNIT_TEST(SimpleYqlQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - - //TDriver + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + + //TDriver auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - + .SetEndpoint(location)); + auto client = NYdb::NTable::TTableClient(connection); - + std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = [client] (const TAsyncCreateSessionResult& future) mutable { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - auto session = sessionValue.GetSession(); - - auto createTableHandler = - [session, client] (NThreading::TFuture<TStatus> future) mutable { - const auto& createTableResult = future.GetValue(); - UNIT_ASSERT(!createTableResult.IsTransportError()); - auto sqlResultHandler = + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + auto session = sessionValue.GetSession(); + + auto createTableHandler = + [session, client] (NThreading::TFuture<TStatus> future) mutable { + const auto& createTableResult = future.GetValue(); + UNIT_ASSERT(!createTableResult.IsTransportError()); + auto sqlResultHandler = [](TAsyncDataQueryResult future) mutable { - auto sqlResultSets = future.ExtractValue(); - UNIT_ASSERT(!sqlResultSets.IsTransportError()); + auto sqlResultSets = future.ExtractValue(); + UNIT_ASSERT(!sqlResultSets.IsTransportError()); auto resultSets = sqlResultSets.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - auto& resultSet = resultSets[0]; + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + auto& resultSet = resultSets[0]; UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 1); - auto meta = resultSet.GetColumnsMeta(); - UNIT_ASSERT_EQUAL(meta.size(), 1); - UNIT_ASSERT_EQUAL(meta[0].Name, "colName"); + auto meta = resultSet.GetColumnsMeta(); + UNIT_ASSERT_EQUAL(meta.size(), 1); + UNIT_ASSERT_EQUAL(meta[0].Name, "colName"); TTypeParser parser(meta[0].Type); UNIT_ASSERT(parser.GetKind() == TTypeParser::ETypeKind::Primitive); UNIT_ASSERT(parser.GetPrimitive() == EPrimitiveType::Int32); @@ -514,89 +514,89 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { TResultSetParser rsParser(resultSet); while (rsParser.TryNextRow()) { UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetInt32(), 42); - } - }; + } + }; session.ExecuteDataQuery( "SELECT 42 as colName;", TTxControl::BeginTx( TTxSettings::SerializableRW()).CommitTx() - ).Apply(std::move(sqlResultHandler)).Wait(); - }; - - auto tableBuilder = client.GetTableBuilder(); + ).Apply(std::move(sqlResultHandler)).Wait(); + }; + + auto tableBuilder = client.GetTableBuilder(); tableBuilder .AddNullableColumn("Key", EPrimitiveType::Int32) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumn("Key"); - session.CreateTable("/Root/FooTable", tableBuilder.Build()).Apply(createTableHandler).Wait(); - }; - - client.CreateSession().Apply(createSessionHandler).Wait(); - } - - Y_UNIT_TEST(TestAuth) { + tableBuilder.SetPrimaryKeyColumn("Key"); + session.CreateTable("/Root/FooTable", tableBuilder.Build()).Apply(createTableHandler).Wait(); + }; + + client.CreateSession().Apply(createSessionHandler).Wait(); + } + + Y_UNIT_TEST(TestAuth) { TKikimrWithGrpcAndRootSchemaWithAuthAndSsl server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetAuthToken("test_user@builtin") - .UseSecureConnection(NYdbSslTestData::CaCrt) - .SetEndpoint(location)); - + .SetAuthToken("test_user@builtin") + .UseSecureConnection(NYdbSslTestData::CaCrt) + .SetEndpoint(location)); + auto client = NYdb::NTable::TTableClient(connection); - std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = - [client] (const TAsyncCreateSessionResult& future) mutable { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::SUCCESS); - }; - - client.CreateSession().Apply(createSessionHandler).Wait(); + std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = + [client] (const TAsyncCreateSessionResult& future) mutable { + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::SUCCESS); + }; + + client.CreateSession().Apply(createSessionHandler).Wait(); connection.Stop(true); - } - + } + Y_UNIT_TEST(YqlQueryWithParams) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - - //TDriver + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + + //TDriver auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - + .SetEndpoint(location)); + auto client = NYdb::NTable::TTableClient(connection); - + std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = [client] (const TAsyncCreateSessionResult& future) mutable { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - auto session = sessionValue.GetSession(); - - auto prepareQueryHandler = + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + auto session = sessionValue.GetSession(); + + auto prepareQueryHandler = [session, client] (TAsyncPrepareQueryResult future) mutable { - const auto& prepareQueryResult = future.GetValue(); - UNIT_ASSERT(!prepareQueryResult.IsTransportError()); - UNIT_ASSERT_EQUAL(prepareQueryResult.GetStatus(), EStatus::SUCCESS); + const auto& prepareQueryResult = future.GetValue(); + UNIT_ASSERT(!prepareQueryResult.IsTransportError()); + UNIT_ASSERT_EQUAL(prepareQueryResult.GetStatus(), EStatus::SUCCESS); auto query = prepareQueryResult.GetQuery(); - auto paramsBuilder = client.GetParamsBuilder(); + auto paramsBuilder = client.GetParamsBuilder(); auto& param = paramsBuilder.AddParam("$paramName"); param.String("someString").Build(); - - auto sqlResultHandler = + + auto sqlResultHandler = [](TAsyncDataQueryResult future) mutable { - auto sqlResultSets = future.ExtractValue(); - UNIT_ASSERT(!sqlResultSets.IsTransportError()); + auto sqlResultSets = future.ExtractValue(); + UNIT_ASSERT(!sqlResultSets.IsTransportError()); UNIT_ASSERT(sqlResultSets.IsSuccess()); auto resultSets = sqlResultSets.GetResultSets(); - UNIT_ASSERT_EQUAL(resultSets.size(), 1); - auto& resultSet = resultSets[0]; + UNIT_ASSERT_EQUAL(resultSets.size(), 1); + auto& resultSet = resultSets[0]; UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 1); - auto meta = resultSet.GetColumnsMeta(); - UNIT_ASSERT_EQUAL(meta.size(), 1); + auto meta = resultSet.GetColumnsMeta(); + UNIT_ASSERT_EQUAL(meta.size(), 1); TTypeParser parser(meta[0].Type); UNIT_ASSERT(parser.GetKind() == TTypeParser::ETypeKind::Primitive); UNIT_ASSERT(parser.GetPrimitive() == EPrimitiveType::String); @@ -604,134 +604,134 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { TResultSetParser rsParser(resultSet); while (rsParser.TryNextRow()) { UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetString(), "someString"); - } - }; + } + }; query.Execute(TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), - paramsBuilder.Build()).Apply(sqlResultHandler).Wait(); - }; - - session.PrepareDataQuery("DECLARE $paramName AS String; SELECT $paramName;").Apply(prepareQueryHandler).Wait(); - }; - - client.CreateSession().Apply(createSessionHandler).Wait(); - } - + paramsBuilder.Build()).Apply(sqlResultHandler).Wait(); + }; + + session.PrepareDataQuery("DECLARE $paramName AS String; SELECT $paramName;").Apply(prepareQueryHandler).Wait(); + }; + + client.CreateSession().Apply(createSessionHandler).Wait(); + } + Y_UNIT_TEST(YqlExplainDataQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - + .SetEndpoint(location)); + auto client = NYdb::NTable::TTableClient(connection); - bool done = false; - + bool done = false; + std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = [client, &done] (const TAsyncCreateSessionResult& future) mutable { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::SUCCESS); - auto session = sessionValue.GetSession(); - - auto schemeQueryHandler = + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + UNIT_ASSERT_EQUAL(sessionValue.GetStatus(), EStatus::SUCCESS); + auto session = sessionValue.GetSession(); + + auto schemeQueryHandler = [session, &done] (const NYdb::TAsyncStatus& future) mutable { - const auto& schemeQueryResult = future.GetValue(); - UNIT_ASSERT(!schemeQueryResult.IsTransportError()); - UNIT_ASSERT_EQUAL(schemeQueryResult.GetStatus(), EStatus::SUCCESS); - - auto sqlResultHandler = + const auto& schemeQueryResult = future.GetValue(); + UNIT_ASSERT(!schemeQueryResult.IsTransportError()); + UNIT_ASSERT_EQUAL(schemeQueryResult.GetStatus(), EStatus::SUCCESS); + + auto sqlResultHandler = [&done] (const TAsyncExplainDataQueryResult& future) mutable { - const auto& explainResult = future.GetValue(); - UNIT_ASSERT(!explainResult.IsTransportError()); - Cerr << explainResult.GetIssues().ToString() << Endl; - UNIT_ASSERT_EQUAL(explainResult.GetStatus(), EStatus::SUCCESS); - done = true; - }; - const TString query = "UPSERT INTO [Root/TheTable] (Key, Value) VALUES (1, \"One\");"; - session.ExplainDataQuery(query).Apply(std::move(sqlResultHandler)).Wait(); - }; - - const TString query = "CREATE TABLE [Root/TheTable] (Key Uint64, Value Utf8, PRIMARY KEY (Key));"; - session.ExecuteSchemeQuery(query).Apply(schemeQueryHandler).Wait(); - }; - - client.CreateSession().Apply(createSessionHandler).Wait(); - UNIT_ASSERT(done); - } - - Y_UNIT_TEST(CreateAlterUpsertDrop) { + const auto& explainResult = future.GetValue(); + UNIT_ASSERT(!explainResult.IsTransportError()); + Cerr << explainResult.GetIssues().ToString() << Endl; + UNIT_ASSERT_EQUAL(explainResult.GetStatus(), EStatus::SUCCESS); + done = true; + }; + const TString query = "UPSERT INTO [Root/TheTable] (Key, Value) VALUES (1, \"One\");"; + session.ExplainDataQuery(query).Apply(std::move(sqlResultHandler)).Wait(); + }; + + const TString query = "CREATE TABLE [Root/TheTable] (Key Uint64, Value Utf8, PRIMARY KEY (Key));"; + session.ExecuteSchemeQuery(query).Apply(schemeQueryHandler).Wait(); + }; + + client.CreateSession().Apply(createSessionHandler).Wait(); + UNIT_ASSERT(done); + } + + Y_UNIT_TEST(CreateAlterUpsertDrop) { TKikimrWithGrpcAndRootSchemaNoSystemViews server; - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); - { - + .SetEndpoint(location)); + { + auto asyncStatus = NYdb::NScheme::TSchemeClient(connection).MakeDirectory("/Root/TheDir"); - asyncStatus.Wait(); - UNIT_ASSERT_EQUAL(asyncStatus.GetValue().GetStatus(), EStatus::SUCCESS); - } - { + asyncStatus.Wait(); + UNIT_ASSERT_EQUAL(asyncStatus.GetValue().GetStatus(), EStatus::SUCCESS); + } + { auto asyncDescPath = NYdb::NScheme::TSchemeClient(connection).DescribePath("/Root/TheDir"); - asyncDescPath.Wait(); - auto entry = asyncDescPath.GetValue().GetEntry(); - UNIT_ASSERT_EQUAL(entry.Name, "TheDir"); - UNIT_ASSERT_EQUAL(entry.Type, ESchemeEntryType::Directory); - } - { + asyncDescPath.Wait(); + auto entry = asyncDescPath.GetValue().GetEntry(); + UNIT_ASSERT_EQUAL(entry.Name, "TheDir"); + UNIT_ASSERT_EQUAL(entry.Type, ESchemeEntryType::Directory); + } + { auto asyncDescDir = NYdb::NScheme::TSchemeClient(connection).ListDirectory("/Root"); - asyncDescDir.Wait(); - const auto& val = asyncDescDir.GetValue(); - auto entry = val.GetEntry(); - UNIT_ASSERT_EQUAL(entry.Name, "Root"); - UNIT_ASSERT_EQUAL(entry.Type, ESchemeEntryType::Directory); - auto children = val.GetChildren(); - UNIT_ASSERT_EQUAL(children.size(), 1); - UNIT_ASSERT_EQUAL(children[0].Name, "TheDir"); - UNIT_ASSERT_EQUAL(children[0].Type, ESchemeEntryType::Directory); - - } - + asyncDescDir.Wait(); + const auto& val = asyncDescDir.GetValue(); + auto entry = val.GetEntry(); + UNIT_ASSERT_EQUAL(entry.Name, "Root"); + UNIT_ASSERT_EQUAL(entry.Type, ESchemeEntryType::Directory); + auto children = val.GetChildren(); + UNIT_ASSERT_EQUAL(children.size(), 1); + UNIT_ASSERT_EQUAL(children[0].Name, "TheDir"); + UNIT_ASSERT_EQUAL(children[0].Type, ESchemeEntryType::Directory); + + } + auto client = NYdb::NTable::TTableClient(connection); - bool done = false; - + bool done = false; + std::function<void(const TAsyncCreateSessionResult& future)> createSessionHandler = [client, &done] (const TAsyncCreateSessionResult& future) mutable { - const auto& sessionValue = future.GetValue(); - UNIT_ASSERT(!sessionValue.IsTransportError()); - auto session = sessionValue.GetSession(); - - auto createTableHandler = - [session, &done, client] (const NThreading::TFuture<TStatus>& future) mutable { - const auto& createTableResult = future.GetValue(); - UNIT_ASSERT(!createTableResult.IsTransportError()); - auto alterResultHandler = - [session, &done] (const NThreading::TFuture<TStatus>& future) mutable { - const auto& alterStatus = future.GetValue(); - UNIT_ASSERT(!alterStatus.IsTransportError()); - UNIT_ASSERT_EQUAL(alterStatus.GetStatus(), EStatus::SUCCESS); - auto upsertHandler = + const auto& sessionValue = future.GetValue(); + UNIT_ASSERT(!sessionValue.IsTransportError()); + auto session = sessionValue.GetSession(); + + auto createTableHandler = + [session, &done, client] (const NThreading::TFuture<TStatus>& future) mutable { + const auto& createTableResult = future.GetValue(); + UNIT_ASSERT(!createTableResult.IsTransportError()); + auto alterResultHandler = + [session, &done] (const NThreading::TFuture<TStatus>& future) mutable { + const auto& alterStatus = future.GetValue(); + UNIT_ASSERT(!alterStatus.IsTransportError()); + UNIT_ASSERT_EQUAL(alterStatus.GetStatus(), EStatus::SUCCESS); + auto upsertHandler = [session, &done] (const TAsyncDataQueryResult& future) mutable { - const auto& sqlResultSets = future.GetValue(); - UNIT_ASSERT(!sqlResultSets.IsTransportError()); - UNIT_ASSERT_EQUAL(sqlResultSets.GetStatus(), EStatus::SUCCESS); - auto describeHandler = + const auto& sqlResultSets = future.GetValue(); + UNIT_ASSERT(!sqlResultSets.IsTransportError()); + UNIT_ASSERT_EQUAL(sqlResultSets.GetStatus(), EStatus::SUCCESS); + auto describeHandler = [session, &done] (const TAsyncDescribeTableResult& future) mutable { - const auto& value = future.GetValue(); - UNIT_ASSERT(!value.IsTransportError()); - UNIT_ASSERT_EQUAL(value.GetStatus(), EStatus::SUCCESS); - auto desc = value.GetTableDescription(); - UNIT_ASSERT_EQUAL(desc.GetPrimaryKeyColumns().size(), 1); - UNIT_ASSERT_EQUAL(desc.GetPrimaryKeyColumns()[0], "Key"); - auto columns = desc.GetColumns(); - UNIT_ASSERT_EQUAL(columns[0].Name, "Key"); - UNIT_ASSERT_EQUAL(columns[1].Name, "Value"); - UNIT_ASSERT_EQUAL(columns[2].Name, "NewColumn"); + const auto& value = future.GetValue(); + UNIT_ASSERT(!value.IsTransportError()); + UNIT_ASSERT_EQUAL(value.GetStatus(), EStatus::SUCCESS); + auto desc = value.GetTableDescription(); + UNIT_ASSERT_EQUAL(desc.GetPrimaryKeyColumns().size(), 1); + UNIT_ASSERT_EQUAL(desc.GetPrimaryKeyColumns()[0], "Key"); + auto columns = desc.GetColumns(); + UNIT_ASSERT_EQUAL(columns[0].Name, "Key"); + UNIT_ASSERT_EQUAL(columns[1].Name, "Value"); + UNIT_ASSERT_EQUAL(columns[2].Name, "NewColumn"); TTypeParser column0(columns[0].Type); TTypeParser column1(columns[1].Type); TTypeParser column2(columns[2].Type); @@ -744,29 +744,29 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { UNIT_ASSERT_EQUAL(column0.GetPrimitive(), EPrimitiveType::Int32); UNIT_ASSERT_EQUAL(column1.GetPrimitive(), EPrimitiveType::String); UNIT_ASSERT_EQUAL(column2.GetPrimitive(), EPrimitiveType::Utf8); - UNIT_ASSERT_EQUAL( desc.GetOwner(), "root@builtin"); - auto dropHandler = - [&done] (const NThreading::TFuture<TStatus>& future) mutable { - const auto& dropStatus = future.GetValue(); - UNIT_ASSERT(!dropStatus.IsTransportError()); - UNIT_ASSERT_EQUAL(dropStatus.GetStatus(), EStatus::SUCCESS); - done = true; - }; - session.DropTable("/Root/TheDir/FooTable") - .Apply(dropHandler).Wait(); - - }; - session.DescribeTable("/Root/TheDir/FooTable") - .Apply(describeHandler).Wait(); - }; - const TString sql = "UPSERT INTO [Root/TheDir/FooTable] (Key, Value, NewColumn)" - " VALUES (1, \"One\", \"йцукен\")"; + UNIT_ASSERT_EQUAL( desc.GetOwner(), "root@builtin"); + auto dropHandler = + [&done] (const NThreading::TFuture<TStatus>& future) mutable { + const auto& dropStatus = future.GetValue(); + UNIT_ASSERT(!dropStatus.IsTransportError()); + UNIT_ASSERT_EQUAL(dropStatus.GetStatus(), EStatus::SUCCESS); + done = true; + }; + session.DropTable("/Root/TheDir/FooTable") + .Apply(dropHandler).Wait(); + + }; + session.DescribeTable("/Root/TheDir/FooTable") + .Apply(describeHandler).Wait(); + }; + const TString sql = "UPSERT INTO [Root/TheDir/FooTable] (Key, Value, NewColumn)" + " VALUES (1, \"One\", \"йцукен\")"; session.ExecuteDataQuery(sql, TTxControl:: BeginTx(TTxSettings::SerializableRW()).CommitTx() - ).Apply(upsertHandler).Wait(); - }; - - { + ).Apply(upsertHandler).Wait(); + }; + + { auto type = TTypeBuilder() .BeginOptional() .Primitive(EPrimitiveType::Utf8) @@ -774,23 +774,23 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) { .Build(); session.AlterTable("/Root/TheDir/FooTable", TAlterTableSettings() - .AppendAddColumns(TColumn{"NewColumn", type})).Apply(alterResultHandler).Wait(); - } - }; - - auto tableBuilder = client.GetTableBuilder(); + .AppendAddColumns(TColumn{"NewColumn", type})).Apply(alterResultHandler).Wait(); + } + }; + + auto tableBuilder = client.GetTableBuilder(); tableBuilder .AddNullableColumn("Key", EPrimitiveType::Int32) .AddNullableColumn("Value", EPrimitiveType::String); - tableBuilder.SetPrimaryKeyColumn("Key"); - session.CreateTable("/Root/TheDir/FooTable", tableBuilder.Build()).Apply(createTableHandler).Wait(); - }; - - client.CreateSession().Apply(createSessionHandler).Wait(); - UNIT_ASSERT(done); - } -} - + tableBuilder.SetPrimaryKeyColumn("Key"); + session.CreateTable("/Root/TheDir/FooTable", tableBuilder.Build()).Apply(createTableHandler).Wait(); + }; + + client.CreateSession().Apply(createSessionHandler).Wait(); + UNIT_ASSERT(done); + } +} + static TString CreateSession(std::shared_ptr<grpc::Channel> channel) { std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; stub = Ydb::Table::V1::TableService::NewStub(channel); @@ -861,496 +861,496 @@ Y_UNIT_TEST_SUITE(GrpcConnectionStringParserTest) { Y_UNIT_TEST_SUITE(TGRpcYdbTest) { Y_UNIT_TEST(RemoveNotExistedDirecroty) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - - Ydb::Scheme::RemoveDirectoryRequest request; - TString scheme( - "path: \"/Root/TheNotExistedDirectory\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::RemoveDirectoryResponse response; - - auto status = Stub_->RemoveDirectory(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + + Ydb::Scheme::RemoveDirectoryRequest request; + TString scheme( + "path: \"/Root/TheNotExistedDirectory\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::RemoveDirectoryResponse response; + + auto status = Stub_->RemoveDirectory(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SCHEME_ERROR); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - TString tmp = issues.ToString(); - TString expected = "<main>: Error: Path does not exist\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - } - + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + TString tmp = issues.ToString(); + TString expected = "<main>: Error: Path does not exist\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + } + Y_UNIT_TEST(MakeListRemoveDirectory) { TKikimrWithGrpcAndRootSchemaNoSystemViews server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString id; - { - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - - Ydb::Scheme::MakeDirectoryRequest request; - TString scheme( - "path: \"/Root/TheDirectory\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::MakeDirectoryResponse response; - - auto status = Stub_->MakeDirectory(&context, request, &response); - auto operation = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - //UNIT_ASSERT(operation.ready() == false); //Not finished yet - //id = operation.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString id; + { + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + + Ydb::Scheme::MakeDirectoryRequest request; + TString scheme( + "path: \"/Root/TheDirectory\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::MakeDirectoryResponse response; + + auto status = Stub_->MakeDirectory(&context, request, &response); + auto operation = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + //UNIT_ASSERT(operation.ready() == false); //Not finished yet + //id = operation.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - { - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Scheme::ListDirectoryRequest request; - TString scheme( - "path: \"/Roo\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::ListDirectoryResponse response; - - auto status = Stub_->ListDirectory(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SCHEME_ERROR); - } - { - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Scheme::ListDirectoryRequest request; - TString scheme( - "path: \"/Root\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::ListDirectoryResponse response; - - auto status = Stub_->ListDirectory(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Scheme::ListDirectoryResult result; - response.operation().result().UnpackTo(&result); - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = "self {\n" - " name: \"Root\"\n" - " owner: \"root@builtin\"\n" - " type: DIRECTORY\n" - "}\n" - "children {\n" - " name: \"TheDirectory\"\n" - " owner: \"root@builtin\"\n" - " type: DIRECTORY\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - { - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - - Ydb::Scheme::RemoveDirectoryRequest request; - TString scheme( - "path: \"/Root/TheDirectory\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::RemoveDirectoryResponse response; - - auto status = Stub_->RemoveDirectory(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - UNIT_ASSERT(deferred.ready() == true); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + } + */ + { + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Scheme::ListDirectoryRequest request; + TString scheme( + "path: \"/Roo\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::ListDirectoryResponse response; + + auto status = Stub_->ListDirectory(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SCHEME_ERROR); + } + { + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Scheme::ListDirectoryRequest request; + TString scheme( + "path: \"/Root\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::ListDirectoryResponse response; + + auto status = Stub_->ListDirectory(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Scheme::ListDirectoryResult result; + response.operation().result().UnpackTo(&result); + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = "self {\n" + " name: \"Root\"\n" + " owner: \"root@builtin\"\n" + " type: DIRECTORY\n" + "}\n" + "children {\n" + " name: \"TheDirectory\"\n" + " owner: \"root@builtin\"\n" + " type: DIRECTORY\n" + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + { + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + + Ydb::Scheme::RemoveDirectoryRequest request; + TString scheme( + "path: \"/Root/TheDirectory\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::RemoveDirectoryResponse response; + + auto status = Stub_->RemoveDirectory(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + UNIT_ASSERT(deferred.ready() == true); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - } - + } + */ + } + Y_UNIT_TEST(GetOperationBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - auto status = WaitForStatus(Channel_, ""); - UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); - } - { - auto status = WaitForStatus(Channel_, "ydb://..."); - UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); - } - { - auto status = WaitForStatus(Channel_, "ydb://operation/1"); - UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); - } - { - auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=42"); - UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); - } - { - auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=aaa&sstid=bbbb"); - UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); - } - - } -/* + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + auto status = WaitForStatus(Channel_, ""); + UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); + } + { + auto status = WaitForStatus(Channel_, "ydb://..."); + UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); + } + { + auto status = WaitForStatus(Channel_, "ydb://operation/1"); + UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); + } + { + auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=42"); + UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); + } + { + auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=aaa&sstid=bbbb"); + UNIT_ASSERT_VALUES_EQUAL(status, Ydb::StatusIds::BAD_REQUEST); + } + + } +/* Y_UNIT_TEST(GetOperationUnknownId) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=42&sstid=66"); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + auto status = WaitForStatus(Channel_, "ydb://operation/1?txid=42&sstid=66"); UNIT_ASSERT(status == Ydb::StatusIds::BAD_REQUEST); - } - } -*/ + } + } +*/ Y_UNIT_TEST(CreateTableBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); //Ready to get status + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); //Ready to get status UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); //But with error - } - - static void CreateTableBadRequest(const TString& scheme, - const TString& expectedMsg, const Ydb::StatusIds::StatusCode expectedStatus) - { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); //Ready to get status - - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - TString tmp = issues.ToString(); - - UNIT_ASSERT_NO_DIFF(tmp, expectedMsg); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), expectedStatus); //But with error - } - - Y_UNIT_TEST(CreateTableBadRequest2) { - TString scheme(R"___( - path: "/Root/TheTable" - columns { name: "Key" type: { optional_type { item { type_id: UINT64 } } } } - columns { name: "Value" type: { optional_type { item { type_id: UTF8 } } } } - primary_key: ["BlaBla"] - )___"); - - TString expected("<main>: Error: Unknown column 'BlaBla' specified in key column list\n"); - - CreateTableBadRequest(scheme, expected, Ydb::StatusIds::SCHEME_ERROR); - } - - Y_UNIT_TEST(CreateTableBadRequest3) { - TString scheme(R"___( - path: "/Root/TheTable" - columns { name: "Key" type: { optional_type { item { type_id: UINT64 } } } } - columns { name: "Value" type: { optional_type { item { type_id: UTF8 } } } } - )___"); - - TString expected("<main>: Error: At least one primary key should be specified\n"); - - CreateTableBadRequest(scheme, expected, Ydb::StatusIds::BAD_REQUEST); - } - + } + + static void CreateTableBadRequest(const TString& scheme, + const TString& expectedMsg, const Ydb::StatusIds::StatusCode expectedStatus) + { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); //Ready to get status + + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + TString tmp = issues.ToString(); + + UNIT_ASSERT_NO_DIFF(tmp, expectedMsg); + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), expectedStatus); //But with error + } + + Y_UNIT_TEST(CreateTableBadRequest2) { + TString scheme(R"___( + path: "/Root/TheTable" + columns { name: "Key" type: { optional_type { item { type_id: UINT64 } } } } + columns { name: "Value" type: { optional_type { item { type_id: UTF8 } } } } + primary_key: ["BlaBla"] + )___"); + + TString expected("<main>: Error: Unknown column 'BlaBla' specified in key column list\n"); + + CreateTableBadRequest(scheme, expected, Ydb::StatusIds::SCHEME_ERROR); + } + + Y_UNIT_TEST(CreateTableBadRequest3) { + TString scheme(R"___( + path: "/Root/TheTable" + columns { name: "Key" type: { optional_type { item { type_id: UINT64 } } } } + columns { name: "Value" type: { optional_type { item { type_id: UTF8 } } } } + )___"); + + TString expected("<main>: Error: At least one primary key should be specified\n"); + + CreateTableBadRequest(scheme, expected, Ydb::StatusIds::BAD_REQUEST); + } + Y_UNIT_TEST(DropTableBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - - Ydb::Table::DropTableRequest request; - Ydb::Table::DropTableResponse response; - - auto status = Stub_->DropTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); //Ready to get status + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + + Ydb::Table::DropTableRequest request; + Ydb::Table::DropTableResponse response; + + auto status = Stub_->DropTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); //Ready to get status UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); //But with error - } - { - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - - Ydb::Table::DropTableRequest request; - TString scheme( - "path: \"/Root/NotExists\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::Table::DropTableResponse response; - - auto status = Stub_->DropTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); //Ready to get status + } + { + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + + Ydb::Table::DropTableRequest request; + TString scheme( + "path: \"/Root/NotExists\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::Table::DropTableResponse response; + + auto status = Stub_->DropTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); //Ready to get status UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SCHEME_ERROR); //But with error - } - - } - - Y_UNIT_TEST(AlterTableAddIndexBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::AlterTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "add_indexes { name: \"ByValue\" index_columns: \"Value\" }" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::AlterTableResponse response; - - auto status = Stub_->AlterTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - UNIT_ASSERT(issues.ToString().Contains("invalid or unset index type")); - } - } - + } + + } + + Y_UNIT_TEST(AlterTableAddIndexBadRequest) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::AlterTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "add_indexes { name: \"ByValue\" index_columns: \"Value\" }" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::AlterTableResponse response; + + auto status = Stub_->AlterTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + UNIT_ASSERT(issues.ToString().Contains("invalid or unset index type")); + } + } + Y_UNIT_TEST(CreateAlterCopyAndDropTable) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString id; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString id; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::DescribeTableRequest request; - TString scheme( - "path: \"/Root/TheTable\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::DescribeTableResponse response; - - auto status = Stub_->DescribeTable(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::DescribeTableResult result; - response.operation().result().UnpackTo(&result); - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = R"___(self { - name: "TheTable" - owner: "root@builtin" - type: TABLE -} -columns { - name: "Key" - type { - optional_type { - item { - type_id: UINT64 - } - } - } -} -columns { - name: "Value" - type { - optional_type { - item { - type_id: UTF8 - } - } - } -} -primary_key: "Key" + } + */ + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::DescribeTableRequest request; + TString scheme( + "path: \"/Root/TheTable\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::DescribeTableResponse response; + + auto status = Stub_->DescribeTable(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Table::DescribeTableResult result; + response.operation().result().UnpackTo(&result); + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = R"___(self { + name: "TheTable" + owner: "root@builtin" + type: TABLE +} +columns { + name: "Key" + type { + optional_type { + item { + type_id: UINT64 + } + } + } +} +columns { + name: "Value" + type { + optional_type { + item { + type_id: UTF8 + } + } + } +} +primary_key: "Key" partitioning_settings { partitioning_by_size: DISABLED partitioning_by_load: DISABLED } -)___"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - { - std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; - Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Scheme::DescribePathRequest request; - TString scheme( - "path: \"/Root/TheTable\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Scheme::DescribePathResponse response; - - auto status = Stub_->DescribePath(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Scheme::DescribePathResult result; - response.operation().result().UnpackTo(&result); - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = "self {\n" - " name: \"TheTable\"\n" - " owner: \"root@builtin\"\n" - " type: TABLE\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - id.clear(); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::AlterTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "add_columns { name: \"Value2\" type: { optional_type { item { type_id: UTF8 } } } }" - "drop_columns: [\"Value\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::AlterTableResponse response; - - auto status = Stub_->AlterTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); +)___"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + { + std::unique_ptr<Ydb::Scheme::V1::SchemeService::Stub> Stub_; + Stub_ = Ydb::Scheme::V1::SchemeService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Scheme::DescribePathRequest request; + TString scheme( + "path: \"/Root/TheTable\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Scheme::DescribePathResponse response; + + auto status = Stub_->DescribePath(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Scheme::DescribePathResult result; + response.operation().result().UnpackTo(&result); + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = "self {\n" + " name: \"TheTable\"\n" + " owner: \"root@builtin\"\n" + " type: TABLE\n" + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + id.clear(); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::AlterTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "add_columns { name: \"Value2\" type: { optional_type { item { type_id: UTF8 } } } }" + "drop_columns: [\"Value\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::AlterTableResponse response; + + auto status = Stub_->AlterTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - id.clear(); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CopyTableRequest request; - TString scheme( - "source_path: \"/Root/TheTable\"" - "destination_path: \"/Root/TheTable2\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CopyTableResponse response; - auto status = Stub_->CopyTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + } + */ + id.clear(); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CopyTableRequest request; + TString scheme( + "source_path: \"/Root/TheTable\"" + "destination_path: \"/Root/TheTable2\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CopyTableResponse response; + auto status = Stub_->CopyTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - id.clear(); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; + } + */ + id.clear(); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; Ydb::Table::CopyTablesRequest request; TString scheme(R"( tables { @@ -1381,621 +1381,621 @@ partitioning_settings { std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::DropTableRequest request; - TString scheme( - "path: \"/Root/TheTable\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::DropTableResponse response; - auto status = Stub_->DropTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + Ydb::Table::DropTableRequest request; + TString scheme( + "path: \"/Root/TheTable\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::DropTableResponse response; + auto status = Stub_->DropTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - } - Y_UNIT_TEST(CreateTableWithIndex) { - TKikimrWithGrpcAndRootSchema server; - server.Server_->GetRuntime()->GetAppData().AllowPrivateTableDescribeForTest = true; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> channel; - channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString id; -/* - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]" - "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { table_profile { partitioning_policy { uniform_partitions: 16 } } } }" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::GENERIC_ERROR); - } -*/ - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable1\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::TypedValue point; - auto &keyType = *point.mutable_type()->mutable_tuple_type(); - keyType.add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - auto &keyVal = *point.mutable_value(); - keyVal.add_items()->set_text_value("q"); - auto index = request.add_indexes(); - index->set_name("IndexedValue"); - index->add_index_columns("IValue"); -// auto points = index->mutable_global_index()->mutable_table_profile()->mutable_partitioning_policy()->mutable_explicit_partitions(); -// points->add_split_points()->CopyFrom(point); - - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::SUCCESS); - } -/* - { - TString sessionId = CreateSession(channel); - auto result = DescribeTable(channel, sessionId, "/Root/TheTable1/IndexedValue/indexImplTable"); - const TString expected = R"__(type { - tuple_type { - elements { - optional_type { - item { - type_id: UTF8 - } - } - } - elements { - optional_type { - item { - type_id: UINT64 - } - } - } - } -} -value { - items { - text_value: "q" - } - items { - null_flag_value: NULL_VALUE - } -} -)__"; - TString tmp; - google::protobuf::TextFormat::PrintToString(result.shard_key_bounds(0), &tmp); - UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(tmp, expected); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable2\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"IValue\" type: { optional_type { item { type_id: UINT32 } } } }" - "primary_key: [\"Key\"]" - "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { table_profile { partitioning_policy { uniform_partitions: 16 } } } }" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::SUCCESS); - } - - { - TString sessionId = CreateSession(channel); - auto result = DescribeTable(channel, sessionId, "/Root/TheTable2/IndexedValue/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds().size(), 15); - UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds(0).value().items(0).uint32_value(), ((1ull << 32) - 1) / 16); - } -*/ - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]" - "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { } }" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(channel); - grpc::ClientContext context; - Ydb::Table::DescribeTableRequest request; - TString scheme( - "path: \"/Root/TheTable\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::DescribeTableResponse response; - - auto status = Stub_->DescribeTable(&context, request, &response); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready() == true); - UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::DescribeTableResult result; - response.operation().result().UnpackTo(&result); - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = R"___(self { - name: "TheTable" - owner: "root@builtin" - type: TABLE -} -columns { - name: "Key" - type { - optional_type { - item { - type_id: UINT64 - } - } - } -} -columns { - name: "IValue" - type { - optional_type { - item { - type_id: UTF8 - } - } - } -} -primary_key: "Key" -indexes { - name: "IndexedValue" - index_columns: "IValue" - global_index { - } - status: STATUS_READY -} + } + */ + } + Y_UNIT_TEST(CreateTableWithIndex) { + TKikimrWithGrpcAndRootSchema server; + server.Server_->GetRuntime()->GetAppData().AllowPrivateTableDescribeForTest = true; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> channel; + channel = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString id; +/* + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]" + "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { table_profile { partitioning_policy { uniform_partitions: 16 } } } }" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::GENERIC_ERROR); + } +*/ + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable1\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::TypedValue point; + auto &keyType = *point.mutable_type()->mutable_tuple_type(); + keyType.add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); + auto &keyVal = *point.mutable_value(); + keyVal.add_items()->set_text_value("q"); + auto index = request.add_indexes(); + index->set_name("IndexedValue"); + index->add_index_columns("IValue"); +// auto points = index->mutable_global_index()->mutable_table_profile()->mutable_partitioning_policy()->mutable_explicit_partitions(); +// points->add_split_points()->CopyFrom(point); + + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::SUCCESS); + } +/* + { + TString sessionId = CreateSession(channel); + auto result = DescribeTable(channel, sessionId, "/Root/TheTable1/IndexedValue/indexImplTable"); + const TString expected = R"__(type { + tuple_type { + elements { + optional_type { + item { + type_id: UTF8 + } + } + } + elements { + optional_type { + item { + type_id: UINT64 + } + } + } + } +} +value { + items { + text_value: "q" + } + items { + null_flag_value: NULL_VALUE + } +} +)__"; + TString tmp; + google::protobuf::TextFormat::PrintToString(result.shard_key_bounds(0), &tmp); + UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(tmp, expected); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable2\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"IValue\" type: { optional_type { item { type_id: UINT32 } } } }" + "primary_key: [\"Key\"]" + "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { table_profile { partitioning_policy { uniform_partitions: 16 } } } }" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::SUCCESS); + } + + { + TString sessionId = CreateSession(channel); + auto result = DescribeTable(channel, sessionId, "/Root/TheTable2/IndexedValue/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds().size(), 15); + UNIT_ASSERT_VALUES_EQUAL(result.shard_key_bounds(0).value().items(0).uint32_value(), ((1ull << 32) - 1) / 16); + } +*/ + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"IValue\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]" + "indexes { name: \"IndexedValue\" index_columns: [\"IValue\"] global_index { } }" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(channel); + grpc::ClientContext context; + Ydb::Table::DescribeTableRequest request; + TString scheme( + "path: \"/Root/TheTable\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::DescribeTableResponse response; + + auto status = Stub_->DescribeTable(&context, request, &response); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready() == true); + UNIT_ASSERT(response.operation().status() == Ydb::StatusIds::SUCCESS); + Ydb::Table::DescribeTableResult result; + response.operation().result().UnpackTo(&result); + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = R"___(self { + name: "TheTable" + owner: "root@builtin" + type: TABLE +} +columns { + name: "Key" + type { + optional_type { + item { + type_id: UINT64 + } + } + } +} +columns { + name: "IValue" + type { + optional_type { + item { + type_id: UTF8 + } + } + } +} +primary_key: "Key" +indexes { + name: "IndexedValue" + index_columns: "IValue" + global_index { + } + status: STATUS_READY +} partitioning_settings { partitioning_by_size: DISABLED partitioning_by_load: DISABLED } -)___"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - { - TString sessionId = CreateSession(channel); - const TString query1(R"( - UPSERT INTO [/Root/TheTable] (Key, IValue) VALUES - (1, "Secondary1"), - (2, "Secondary2"), - (3, "Secondary3"); - )"); - ExecYql(channel, sessionId, query1); - } - - { - TString sessionId = CreateSession(channel); - ExecYql(channel, sessionId, - "SELECT * FROM [/Root/TheTable];"); - } - - } - +)___"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + { + TString sessionId = CreateSession(channel); + const TString query1(R"( + UPSERT INTO [/Root/TheTable] (Key, IValue) VALUES + (1, "Secondary1"), + (2, "Secondary2"), + (3, "Secondary3"); + )"); + ExecYql(channel, sessionId, query1); + } + + { + TString sessionId = CreateSession(channel); + ExecYql(channel, sessionId, + "SELECT * FROM [/Root/TheTable];"); + } + + } + Y_UNIT_TEST(CreateYqlSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - } - } - + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + } + } + Y_UNIT_TEST(CreateDeleteYqlSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::DeleteSessionRequest request; - request.set_session_id(sessionId); - Ydb::Table::DeleteSessionResponse response; - - auto status = Stub_->DeleteSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - } - - } - + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::DeleteSessionRequest request; + request.set_session_id(sessionId); + Ydb::Table::DeleteSessionResponse response; + + auto status = Stub_->DeleteSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + } + + } + Y_UNIT_TEST(ExecuteQueryBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(status.ok()); UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; + Ydb::Table::CreateSessionResult result; deferred.result().UnpackTo(&result); sessionId = result.session_id(); } { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_REQUEST); - } - } - + } + } + Y_UNIT_TEST(ExecuteQueryImplicitSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; request.mutable_query()->set_yql_text("SELECT 1 as a, 'qwerty' as b, 43.5 as c UNION ALL SELECT 11 as a, 'asdfgg' as b, Null as c;"); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::BAD_REQUEST); - } - } - + } + } + Y_UNIT_TEST(ExecuteQueryExplicitSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::KeepAliveRequest request; - request.set_session_id(sessionId); - Ydb::Table::KeepAliveResponse response; - auto status = Stub_->KeepAlive(&context, request, &response); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::KeepAliveRequest request; + request.set_session_id(sessionId); + Ydb::Table::KeepAliveResponse response; + auto status = Stub_->KeepAlive(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("SELECT 1 as a, 'qwerty' as b, 43.5 as c UNION ALL SELECT 11 as a, 'asdfgg' as b, Null as c;"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - - Ydb::Table::ExecuteQueryResult result; - deferred.result().UnpackTo(&result); - { - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = - "result_sets {\n" + + Ydb::Table::ExecuteQueryResult result; + deferred.result().UnpackTo(&result); + { + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = + "result_sets {\n" " columns {\n" - " name: \"a\"\n" - " type {\n" - " type_id: INT32\n" - " }\n" - " }\n" + " name: \"a\"\n" + " type {\n" + " type_id: INT32\n" + " }\n" + " }\n" " columns {\n" - " name: \"b\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" + " name: \"b\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" " columns {\n" - " name: \"c\"\n" - " type {\n" - " optional_type {\n" - " item {\n" - " type_id: DOUBLE\n" - " }\n" - " }\n" - " }\n" - " }\n" - " rows {\n" - " items {\n" - " int32_value: 1\n" - " }\n" - " items {\n" - " bytes_value: \"qwerty\"\n" - " }\n" - " items {\n" - " double_value: 43.5\n" - " }\n" - " }\n" - " rows {\n" - " items {\n" - " int32_value: 11\n" - " }\n" - " items {\n" - " bytes_value: \"asdfgg\"\n" - " }\n" - " items {\n" - " null_flag_value: NULL_VALUE\n" - " }\n" - " }\n" + " name: \"c\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " type_id: DOUBLE\n" + " }\n" + " }\n" + " }\n" + " }\n" + " rows {\n" + " items {\n" + " int32_value: 1\n" + " }\n" + " items {\n" + " bytes_value: \"qwerty\"\n" + " }\n" + " items {\n" + " double_value: 43.5\n" + " }\n" + " }\n" + " rows {\n" + " items {\n" + " int32_value: 11\n" + " }\n" + " items {\n" + " bytes_value: \"asdfgg\"\n" + " }\n" + " items {\n" + " null_flag_value: NULL_VALUE\n" + " }\n" + " }\n" "}\n" "tx_meta {\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); TResultSet resultSet(result.result_sets(0)); UNIT_ASSERT_EQUAL(resultSet.ColumnsCount(), 3); - int row = 0; + int row = 0; TResultSetParser rsParser(resultSet); while (rsParser.TryNextRow()) { - switch (row) { - case 0: { + switch (row) { + case 0: { UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetInt32(), 1); UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetString(), "qwerty"); UNIT_ASSERT_EQUAL(rsParser.ColumnParser(2).GetOptionalDouble(), 43.5); - } - break; - case 1: { + } + break; + case 1: { UNIT_ASSERT_EQUAL(rsParser.ColumnParser(0).GetInt32(), 11); UNIT_ASSERT_EQUAL(rsParser.ColumnParser(1).GetString(), "asdfgg"); rsParser.ColumnParser(2).OpenOptional(); UNIT_ASSERT_EQUAL(rsParser.ColumnParser(2).IsNull(), true); - } - break; - default: { - UNIT_ASSERT(false); - } - } - row++; - } - } - } - } - - Y_UNIT_TEST(ExecuteQueryWithUuid) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); - request.mutable_query()->set_yql_text(R"___(SELECT CAST("5ca32c22-841b-11e8-adc0-fa7ae01bbebc" AS Uuid);)___"); - request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); - request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; - auto status = Stub_->ExecuteDataQuery(&context, request, &response); + } + break; + default: { + UNIT_ASSERT(false); + } + } + row++; + } + } + } + } + + Y_UNIT_TEST(ExecuteQueryWithUuid) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); + request.mutable_query()->set_yql_text(R"___(SELECT CAST("5ca32c22-841b-11e8-adc0-fa7ae01bbebc" AS Uuid);)___"); + request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); + request.mutable_tx_control()->set_commit_tx(true); + Ydb::Table::ExecuteDataQueryResponse response; + auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - - Ydb::Table::ExecuteQueryResult result; - deferred.result().UnpackTo(&result); - { - TString expected = R"___(result_sets { + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + + Ydb::Table::ExecuteQueryResult result; + deferred.result().UnpackTo(&result); + { + TString expected = R"___(result_sets { columns { - name: "column0" - type { - optional_type { - item { - type_id: UUID - } - } - } - } - rows { - items { - low_128: 1290426546294828066 - high_128: 13600338575655354541 - } - } -} -tx_meta { -} -)___"; - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - } - } - + name: "column0" + type { + optional_type { + item { + type_id: UUID + } + } + } + } + rows { + items { + low_128: 1290426546294828066 + high_128: 13600338575655354541 + } + } +} +tx_meta { +} +)___"; + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + } + } + Y_UNIT_TEST(ExecuteQueryWithParametersBadRequest) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); request.mutable_query()->set_yql_text("DECLARE $param1 AS \"Tuple<Int32,Bool>\"; SELECT $param1 AS Tuple;"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - { + { // Bad type Kind - ::google::protobuf::Map<TString, Ydb::TypedValue> parameters; + ::google::protobuf::Map<TString, Ydb::TypedValue> parameters; const TString type = R"( type_id: TYPE_UNDEFINED @@ -2009,7 +2009,7 @@ tx_meta { *request.mutable_parameters() = parameters; Ydb::Table::ExecuteDataQueryResponse response; - grpc::ClientContext context; + grpc::ClientContext context; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); auto deferred = response.operation(); @@ -2019,10 +2019,10 @@ tx_meta { issues.PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_REQUEST); } - + { // Value mismatch - ::google::protobuf::Map<TString, Ydb::TypedValue> parameters; + ::google::protobuf::Map<TString, Ydb::TypedValue> parameters; const TString type = R"( tuple_type { @@ -2042,65 +2042,65 @@ tx_meta { google::protobuf::TextFormat::ParseFromString(value, parameters["$param1"].mutable_value()); *request.mutable_parameters() = parameters; - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; grpc::ClientContext context; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); NYql::TIssues issues; NYql::IssuesFromMessage(deferred.issues(), issues); issues.PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_REQUEST); - } - } - + UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::BAD_REQUEST); + } + } + Y_UNIT_TEST(ExecuteQueryWithParametersExplicitSession) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; request.mutable_query()->set_yql_text("DECLARE $paramName AS String; SELECT $paramName;"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::TypedValue parameter; - { - const TString type = - "type_id: STRING\n"; - google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()); - const TString value = "bytes_value: \"Paul\"\n"; - google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()); - } - - auto& map = *request.mutable_parameters(); - map["$paramName"] = parameter; - { - TString tmp; - google::protobuf::TextFormat::PrintToString(request, &tmp); - const TString expected = + Ydb::TypedValue parameter; + { + const TString type = + "type_id: STRING\n"; + google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()); + const TString value = "bytes_value: \"Paul\"\n"; + google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()); + } + + auto& map = *request.mutable_parameters(); + map["$paramName"] = parameter; + { + TString tmp; + google::protobuf::TextFormat::PrintToString(request, &tmp); + const TString expected = "tx_control {\n" " begin_tx {\n" " serializable_read_write {\n" @@ -2111,173 +2111,173 @@ tx_meta { "query {\n" " yql_text: \"DECLARE $paramName AS String; SELECT $paramName;\"\n" "}\n" - "parameters {\n" - " key: \"$paramName\"\n" - " value {\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " value {\n" - " bytes_value: \"Paul\"\n" - " }\n" - " }\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - request.set_session_id(sessionId); - Ydb::Table::ExecuteDataQueryResponse response; + "parameters {\n" + " key: \"$paramName\"\n" + " value {\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " value {\n" + " bytes_value: \"Paul\"\n" + " }\n" + " }\n" + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + request.set_session_id(sessionId); + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - - Ydb::Table::ExecuteQueryResult result; - deferred.result().UnpackTo(&result); - { - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = - "result_sets {\n" + + Ydb::Table::ExecuteQueryResult result; + deferred.result().UnpackTo(&result); + { + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = + "result_sets {\n" " columns {\n" - " name: \"column0\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" - " rows {\n" - " items {\n" - " bytes_value: \"Paul\"\n" - " }\n" - " }\n" + " name: \"column0\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " rows {\n" + " items {\n" + " bytes_value: \"Paul\"\n" + " }\n" + " }\n" "}\n" "tx_meta {\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - } - // Check Uuid protos as parametr - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.mutable_query()->set_yql_text("DECLARE $paramName AS Uuid; SELECT $paramName;"); - request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); - request.mutable_tx_control()->set_commit_tx(true); - Ydb::TypedValue parameter; - { - const TString type = - "type_id: UUID\n"; + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + } + // Check Uuid protos as parametr + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.mutable_query()->set_yql_text("DECLARE $paramName AS Uuid; SELECT $paramName;"); + request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); + request.mutable_tx_control()->set_commit_tx(true); + Ydb::TypedValue parameter; + { + const TString type = + "type_id: UUID\n"; UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type())); const TString value = R"(low_128: 1290426546294828066 high_128: 13600338575655354541)"; UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value())); - } - - auto& map = *request.mutable_parameters(); - map["$paramName"] = parameter; - request.set_session_id(sessionId); - Ydb::Table::ExecuteDataQueryResponse response; - auto status = Stub_->ExecuteDataQuery(&context, request, &response); + } + + auto& map = *request.mutable_parameters(); + map["$paramName"] = parameter; + request.set_session_id(sessionId); + Ydb::Table::ExecuteDataQueryResponse response; + auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - - Ydb::Table::ExecuteQueryResult result; - deferred.result().UnpackTo(&result); - { - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = R"___(result_sets { + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + + Ydb::Table::ExecuteQueryResult result; + deferred.result().UnpackTo(&result); + { + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = R"___(result_sets { columns { - name: "column0" - type { - type_id: UUID - } - } - rows { - items { - low_128: 1290426546294828066 - high_128: 13600338575655354541 - } - } -} -tx_meta { -} -)___"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - } - - } - + name: "column0" + type { + type_id: UUID + } + } + rows { + items { + low_128: 1290426546294828066 + high_128: 13600338575655354541 + } + } +} +tx_meta { +} +)___"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + } + + } + Y_UNIT_TEST(ExecuteDmlQuery) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); - + std::shared_ptr<grpc::Channel> Channel_; Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); TString sessionId; { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(status.ok()); UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; + Ydb::Table::CreateSessionResult result; deferred.result().UnpackTo(&result); sessionId = result.session_id(); } { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExecuteSchemeQueryRequest request; + Ydb::Table::ExecuteSchemeQueryRequest request; request.set_session_id(sessionId); request.set_yql_text(R"( CREATE TABLE [Root/TheTable] ( - Key UINT64, - Value UTF8, + Key UINT64, + Value UTF8, PRIMARY KEY (Key) ); )"); - Ydb::Table::ExecuteSchemeQueryResponse response; + Ydb::Table::ExecuteSchemeQueryResponse response; auto status = Stub_->ExecuteSchemeQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); } { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExplainDataQueryRequest request; + Ydb::Table::ExplainDataQueryRequest request; request.set_session_id(sessionId); request.set_yql_text(R"( UPSERT INTO [Root/TheTable] (Key, Value) VALUES (1, "One"); )"); - Ydb::Table::ExplainDataQueryResponse response; + Ydb::Table::ExplainDataQueryResponse response; auto status = Stub_->ExplainDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); @@ -2285,11 +2285,11 @@ tx_meta { TString txId; { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); request.mutable_query()->set_yql_text(R"( UPSERT INTO [Root/TheTable] (Key, Value) @@ -2299,15 +2299,15 @@ tx_meta { auto& txControl = *request.mutable_tx_control(); txControl.mutable_begin_tx()->mutable_serializable_read_write(); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::ExecuteQueryResult result; + Ydb::Table::ExecuteQueryResult result; deferred.result().UnpackTo(&result); txId = result.tx_meta().id(); @@ -2315,11 +2315,11 @@ tx_meta { } { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); request.mutable_query()->set_yql_text(R"( UPSERT INTO [Root/TheTable] (Key, Value) @@ -2330,10 +2330,10 @@ tx_meta { txControl.set_tx_id(txId); txControl.set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); @@ -2341,36 +2341,36 @@ tx_meta { TString queryId; { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::PrepareDataQueryRequest request; + Ydb::Table::PrepareDataQueryRequest request; request.set_session_id(sessionId); request.set_yql_text(R"( DECLARE $Key AS Uint64; SELECT * FROM [Root/TheTable] WHERE Key < $Key; )"); - Ydb::Table::PrepareDataQueryResponse response; + Ydb::Table::PrepareDataQueryResponse response; auto status = Stub_->PrepareDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::PrepareQueryResult result; + Ydb::Table::PrepareQueryResult result; deferred.result().UnpackTo(&result); queryId = result.query_id(); } { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; + Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); request.mutable_query()->set_id(queryId); @@ -2378,10 +2378,10 @@ tx_meta { txControl.mutable_begin_tx()->mutable_online_read_only(); txControl.set_commit_tx(true); - Ydb::TypedValue parameter; + Ydb::TypedValue parameter; { const TString type = - "type_id: UINT64\n"; + "type_id: UINT64\n"; google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()); const TString value = "uint64_value: 5\n"; google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()); @@ -2390,15 +2390,15 @@ tx_meta { auto& map = *request.mutable_parameters(); map["$Key"] = parameter; - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::ExecuteQueryResult result; + Ydb::Table::ExecuteQueryResult result; deferred.result().UnpackTo(&result); UNIT_ASSERT(result.tx_meta().id().empty()); @@ -2407,159 +2407,159 @@ tx_meta { } Y_UNIT_TEST(CreateYqlSessionExecuteQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("SELECT 1, \"qq\"; SELECT 2;"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + UNIT_ASSERT(deferred.ready() == true); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("SELECT * from [Root/NotFound]"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); - auto deferred = response.operation(); + auto deferred = response.operation(); UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SCHEME_ERROR); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); UNIT_ASSERT(HasIssue(issues, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR)); - } - - } + } + + } Y_UNIT_TEST(ExecutePreparedQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - TString preparedQueryId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::PrepareDataQueryRequest request; - request.set_session_id(sessionId); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + TString preparedQueryId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::PrepareDataQueryRequest request; + request.set_session_id(sessionId); request.set_yql_text("DECLARE $paramName AS String; SELECT $paramName;"); - Ydb::Table::PrepareDataQueryResponse response; + Ydb::Table::PrepareDataQueryResponse response; auto status = Stub_->PrepareDataQuery(&context, request, &response); - UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(status.ok()); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::PrepareQueryResult result; - - deferred.result().UnpackTo(&result); - preparedQueryId = result.query_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + Ydb::Table::PrepareQueryResult result; + + deferred.result().UnpackTo(&result); + preparedQueryId = result.query_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_id(preparedQueryId); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::TypedValue parameter; - { - const TString type = - "type_id: STRING\n"; - google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()); - const TString value = "bytes_value: \"Paul\"\n"; - google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()); - } - - auto& map = *request.mutable_parameters(); - map["$paramName"] = parameter; - - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::TypedValue parameter; + { + const TString type = + "type_id: STRING\n"; + google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()); + const TString value = "bytes_value: \"Paul\"\n"; + google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()); + } + + auto& map = *request.mutable_parameters(); + map["$paramName"] = parameter; + + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); - UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + UNIT_ASSERT(status.ok()); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - Ydb::Table::ExecuteQueryResult result; - deferred.result().UnpackTo(&result); - { - TString tmp; - google::protobuf::TextFormat::PrintToString(result, &tmp); - const TString expected = - "result_sets {\n" + Ydb::Table::ExecuteQueryResult result; + deferred.result().UnpackTo(&result); + { + TString tmp; + google::protobuf::TextFormat::PrintToString(result, &tmp); + const TString expected = + "result_sets {\n" " columns {\n" - " name: \"column0\"\n" - " type {\n" - " type_id: STRING\n" - " }\n" - " }\n" - " rows {\n" - " items {\n" - " bytes_value: \"Paul\"\n" - " }\n" - " }\n" + " name: \"column0\"\n" + " type {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " rows {\n" + " items {\n" + " bytes_value: \"Paul\"\n" + " }\n" + " }\n" "}\n" "tx_meta {\n" - "}\n"; - UNIT_ASSERT_NO_DIFF(tmp, expected); - } - } - } - + "}\n"; + UNIT_ASSERT_NO_DIFF(tmp, expected); + } + } + } + Y_UNIT_TEST(ExecuteQueryCache) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); @@ -2647,418 +2647,418 @@ tx_meta { } Y_UNIT_TEST(ExplainQuery) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString id; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type: { item: { type_id: UINT64 } } } }" - "columns { name: \"Value\" type: { optional_type: { item: { type_id: UTF8 } } } }" - "primary_key: [\"Key\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - TString tmp = issues.ToString(); - Cerr << tmp << Endl; - - UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString id; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type: { item: { type_id: UINT64 } } } }" + "columns { name: \"Value\" type: { optional_type: { item: { type_id: UTF8 } } } }" + "primary_key: [\"Key\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + TString tmp = issues.ToString(); + Cerr << tmp << Endl; + + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - - TString sessionId; - TString preparedQueryId; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + } + */ + + TString sessionId; + TString preparedQueryId; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("UPSERT INTO [Root/TheTable] (Key, Value) VALUES (42, \"data\");"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExplainDataQueryRequest request; - request.set_session_id(sessionId); + } + + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExplainDataQueryRequest request; + request.set_session_id(sessionId); request.set_yql_text("SELECT COUNT(*) FROM [Root/TheTable];"); - Ydb::Table::ExplainDataQueryResponse response; + Ydb::Table::ExplainDataQueryResponse response; auto status = Stub_->ExplainDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - } - + } + } + Y_UNIT_TEST(DeleteFromAfterCreate) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - - TString id; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\", \"Value\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + + TString id; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\", \"Value\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable2\"" - "columns {\n" - " name: \"id_a\"\n" - " type: { optional_type { item { type_id: INT32 } } }" - "}\n" - "columns {\n" - " name: \"id_b\"\n" - " type: { optional_type { item { type_id: INT64 } } }" - "}\n" - "columns {\n" - " name: \"id_c\"\n" - " type: { optional_type { item { type_id: STRING } } }" - "}\n" - "columns {\n" - " name: \"id_d\"\n" - " type: { optional_type { item { type_id: STRING } } }" - "}\n" - "primary_key: \"id_a\"\n" - "primary_key: \"id_b\"\n" - "primary_key: \"id_c\"\n" - "primary_key: \"id_d\""); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - //id = deferred.id(); - } - /* - { - auto status = WaitForStatus(Channel_, id); + } + */ + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable2\"" + "columns {\n" + " name: \"id_a\"\n" + " type: { optional_type { item { type_id: INT32 } } }" + "}\n" + "columns {\n" + " name: \"id_b\"\n" + " type: { optional_type { item { type_id: INT64 } } }" + "}\n" + "columns {\n" + " name: \"id_c\"\n" + " type: { optional_type { item { type_id: STRING } } }" + "}\n" + "columns {\n" + " name: \"id_d\"\n" + " type: { optional_type { item { type_id: STRING } } }" + "}\n" + "primary_key: \"id_a\"\n" + "primary_key: \"id_b\"\n" + "primary_key: \"id_c\"\n" + "primary_key: \"id_d\""); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + //id = deferred.id(); + } + /* + { + auto status = WaitForStatus(Channel_, id); UNIT_ASSERT(status == Ydb::StatusIds::SUCCESS); - } - */ - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + } + */ + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("DELETE FROM [Root/TheTable];"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); request.mutable_query()->set_yql_text("DELETE FROM [Root/TheTable2];"); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - } - + } + } + Y_UNIT_TEST(ReadTable) { - TKikimrWithGrpcAndRootSchema server; - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); - ui16 grpc = server.GetPort(); - - std::shared_ptr<grpc::Channel> Channel_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); - TString sessionId; - - TVector<std::tuple<ui64, TString>> data = { - {42, "data42"}, - {43, "data43"}, - {44, "data44"}, - {45, "data45"}, - {46, "data46"}, - {47, "data47"}, - {48, "data48"}, - {49, "data49"}, - {50, "data50"}, - {51, "data51"}, - {52, "data52"} - }; - TString id; - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateTableRequest request; - TString scheme( - "path: \"/Root/TheTable\"" - "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" - "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" - "primary_key: [\"Key\", \"Value\"]"); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - Ydb::Table::CreateTableResponse response; - - auto status = Stub_->CreateTable(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); //GRpc layer - OK - UNIT_ASSERT(deferred.ready() == true); //Not finished yet - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::CreateSessionRequest request; - Ydb::Table::CreateSessionResponse response; - - auto status = Stub_->CreateSession(&context, request, &response); - auto deferred = response.operation(); - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(deferred.ready() == true); - Ydb::Table::CreateSessionResult result; - - deferred.result().UnpackTo(&result); - sessionId = result.session_id(); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ExecuteDataQueryRequest request; - request.set_session_id(sessionId); - TStringBuilder requestBuilder; - requestBuilder << "UPSERT INTO [Root/TheTable] (Key, Value) VALUES"; - for (auto pair : data) { - requestBuilder << "(" << std::get<0>(pair) << ", \"" << std::get<1>(pair) << "\"),"; - } - TString req(requestBuilder); - req.back() = ';'; - request.mutable_query()->set_yql_text(req); - + TKikimrWithGrpcAndRootSchema server; + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); + server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::READ_TABLE_API, NLog::PRI_TRACE); + ui16 grpc = server.GetPort(); + + std::shared_ptr<grpc::Channel> Channel_; + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpc), grpc::InsecureChannelCredentials()); + TString sessionId; + + TVector<std::tuple<ui64, TString>> data = { + {42, "data42"}, + {43, "data43"}, + {44, "data44"}, + {45, "data45"}, + {46, "data46"}, + {47, "data47"}, + {48, "data48"}, + {49, "data49"}, + {50, "data50"}, + {51, "data51"}, + {52, "data52"} + }; + TString id; + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateTableRequest request; + TString scheme( + "path: \"/Root/TheTable\"" + "columns { name: \"Key\" type: { optional_type { item { type_id: UINT64 } } } }" + "columns { name: \"Value\" type: { optional_type { item { type_id: UTF8 } } } }" + "primary_key: [\"Key\", \"Value\"]"); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + Ydb::Table::CreateTableResponse response; + + auto status = Stub_->CreateTable(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); //GRpc layer - OK + UNIT_ASSERT(deferred.ready() == true); //Not finished yet + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::CreateSessionRequest request; + Ydb::Table::CreateSessionResponse response; + + auto status = Stub_->CreateSession(&context, request, &response); + auto deferred = response.operation(); + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(deferred.ready() == true); + Ydb::Table::CreateSessionResult result; + + deferred.result().UnpackTo(&result); + sessionId = result.session_id(); + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ExecuteDataQueryRequest request; + request.set_session_id(sessionId); + TStringBuilder requestBuilder; + requestBuilder << "UPSERT INTO [Root/TheTable] (Key, Value) VALUES"; + for (auto pair : data) { + requestBuilder << "(" << std::get<0>(pair) << ", \"" << std::get<1>(pair) << "\"),"; + } + TString req(requestBuilder); + req.back() = ';'; + request.mutable_query()->set_yql_text(req); + request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - Ydb::Table::ExecuteDataQueryResponse response; + Ydb::Table::ExecuteDataQueryResponse response; auto status = Stub_->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); - auto deferred = response.operation(); - UNIT_ASSERT(deferred.ready() == true); + auto deferred = response.operation(); + UNIT_ASSERT(deferred.ready() == true); UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ReadTableRequest request; - Ydb::Table::ReadTableResponse response; - - auto reader = Stub_->StreamReadTable(&context, request); - bool res = true; - // Empty request - we expect to get BAD_REQUEST response - while (res) { - res = reader->Read(&response); - if (res) { - UNIT_ASSERT(response.status() == Ydb::StatusIds::BAD_REQUEST); - } - } - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ReadTableRequest request; - Ydb::Table::ReadTableResponse response; - - TString scheme( - "path: \"/Root/TheTable\"" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - auto reader = Stub_->StreamReadTable(&context, request); - bool res = true; - while (res) { - res = reader->Read(&response); - // Expect all data in first response message - if (res) { - UNIT_ASSERT_EQUAL(response.status(), Ydb::StatusIds::SUCCESS); - if (response.result().has_result_set()) { - size_t i = 0; - UNIT_ASSERT_EQUAL((size_t)response.result().result_set().rows_size(), data.size()); - for (const auto& row : response.result().result_set().rows()) { - const auto& pair = data[i++]; - UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); - UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); - } - } - } - } - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ReadTableRequest request; - Ydb::Table::ReadTableResponse response; - - TString scheme( - "path: \"/Root/TheTable\"" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - auto keyRange = request.mutable_key_range(); - auto greater = keyRange->mutable_greater(); - greater->mutable_type()->mutable_tuple_type()->add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UINT64); - greater->mutable_value()->add_items()->set_uint64_value(50); - auto reader = Stub_->StreamReadTable(&context, request); - bool res = true; - while (res) { - res = reader->Read(&response); - if (res) { - UNIT_ASSERT(response.status() == Ydb::StatusIds::SUCCESS); - if (response.result().has_result_set()) { - size_t i = 9; - UNIT_ASSERT(response.result().result_set().rows_size() == 2); - for (const auto& row : response.result().result_set().rows()) { - const auto& pair = data[i++]; - UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); - UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); - } - } - } - } - } - { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); - grpc::ClientContext context; - Ydb::Table::ReadTableRequest request; - Ydb::Table::ReadTableResponse response; - - TString scheme( - "path: \"/Root/TheTable\"" - ); - ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - auto keyRange = request.mutable_key_range(); - auto less = keyRange->mutable_less_or_equal(); - less->mutable_type()->mutable_tuple_type()->add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UINT64); - less->mutable_value()->add_items()->set_uint64_value(50); - auto reader = Stub_->StreamReadTable(&context, request); - bool res = true; - while (res) { - res = reader->Read(&response); - if (res) { - UNIT_ASSERT(response.status() == Ydb::StatusIds::SUCCESS); - if (response.result().has_result_set()) { - UNIT_ASSERT(response.result().result_set().rows_size() == 9); - size_t i = 0; - for (const auto& row : response.result().result_set().rows()) { - const auto& pair = data[i++]; - UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); - UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); - } - } - } - } - } - } + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ReadTableRequest request; + Ydb::Table::ReadTableResponse response; + + auto reader = Stub_->StreamReadTable(&context, request); + bool res = true; + // Empty request - we expect to get BAD_REQUEST response + while (res) { + res = reader->Read(&response); + if (res) { + UNIT_ASSERT(response.status() == Ydb::StatusIds::BAD_REQUEST); + } + } + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ReadTableRequest request; + Ydb::Table::ReadTableResponse response; + + TString scheme( + "path: \"/Root/TheTable\"" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + auto reader = Stub_->StreamReadTable(&context, request); + bool res = true; + while (res) { + res = reader->Read(&response); + // Expect all data in first response message + if (res) { + UNIT_ASSERT_EQUAL(response.status(), Ydb::StatusIds::SUCCESS); + if (response.result().has_result_set()) { + size_t i = 0; + UNIT_ASSERT_EQUAL((size_t)response.result().result_set().rows_size(), data.size()); + for (const auto& row : response.result().result_set().rows()) { + const auto& pair = data[i++]; + UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); + UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); + } + } + } + } + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ReadTableRequest request; + Ydb::Table::ReadTableResponse response; + + TString scheme( + "path: \"/Root/TheTable\"" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + auto keyRange = request.mutable_key_range(); + auto greater = keyRange->mutable_greater(); + greater->mutable_type()->mutable_tuple_type()->add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UINT64); + greater->mutable_value()->add_items()->set_uint64_value(50); + auto reader = Stub_->StreamReadTable(&context, request); + bool res = true; + while (res) { + res = reader->Read(&response); + if (res) { + UNIT_ASSERT(response.status() == Ydb::StatusIds::SUCCESS); + if (response.result().has_result_set()) { + size_t i = 9; + UNIT_ASSERT(response.result().result_set().rows_size() == 2); + for (const auto& row : response.result().result_set().rows()) { + const auto& pair = data[i++]; + UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); + UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); + } + } + } + } + } + { + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + grpc::ClientContext context; + Ydb::Table::ReadTableRequest request; + Ydb::Table::ReadTableResponse response; + + TString scheme( + "path: \"/Root/TheTable\"" + ); + ::google::protobuf::TextFormat::ParseFromString(scheme, &request); + auto keyRange = request.mutable_key_range(); + auto less = keyRange->mutable_less_or_equal(); + less->mutable_type()->mutable_tuple_type()->add_elements()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UINT64); + less->mutable_value()->add_items()->set_uint64_value(50); + auto reader = Stub_->StreamReadTable(&context, request); + bool res = true; + while (res) { + res = reader->Read(&response); + if (res) { + UNIT_ASSERT(response.status() == Ydb::StatusIds::SUCCESS); + if (response.result().has_result_set()) { + UNIT_ASSERT(response.result().result_set().rows_size() == 9); + size_t i = 0; + for (const auto& row : response.result().result_set().rows()) { + const auto& pair = data[i++]; + UNIT_ASSERT_EQUAL(std::get<0>(pair), row.items(0).uint64_value()); + UNIT_ASSERT_EQUAL(std::get<1>(pair), row.items(1).text_value()); + } + } + } + } + } + } Y_UNIT_TEST(OperationTimeout) { TKikimrWithGrpcAndRootSchema server; @@ -3177,8 +3177,8 @@ tx_meta { UNIT_ASSERT_VALUES_EQUAL(deferred.status(), Ydb::StatusIds::NOT_FOUND); } } -} - +} + namespace { NKikimrSchemeOp::TCompactionPolicy DEFAULT_COMPACTION_POLICY; @@ -3689,10 +3689,10 @@ void CreateTable(TKikimrWithGrpcAndRootSchema &server, { std::shared_ptr<grpc::Channel> Channel_; Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; - Ydb::Table::CreateTableResponse response; + Ydb::Table::CreateTableResponse response; auto status = Stub_->CreateTable(&context, request, &response); auto deferred = response.operation(); UNIT_ASSERT(status.ok()); @@ -3719,10 +3719,10 @@ void CreateTable(TKikimrWithGrpcAndRootSchema &server, void CreateTable(TKikimrWithGrpcAndRootSchema &server, const TString &path, - const Ydb::Table::TableProfile &profile, + const Ydb::Table::TableProfile &profile, Ydb::StatusIds::StatusCode code = Ydb::StatusIds::SUCCESS) { - Ydb::Table::CreateTableRequest request; + Ydb::Table::CreateTableRequest request; request.mutable_profile()->CopyFrom(profile); CreateTable(server, path, request, code); } @@ -3779,7 +3779,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("default"); CreateTable(server, "/Root/table-2", profile); @@ -3787,7 +3787,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("default"); profile.mutable_compaction_policy()->set_preset_name("default"); profile.mutable_execution_policy()->set_preset_name("default"); @@ -3804,7 +3804,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile); @@ -3820,7 +3820,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile2"); CreateTable(server, "/Root/ydb_ut_tenant/table-2", profile); @@ -3836,75 +3836,75 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } } - Y_UNIT_TEST(UseTableProfilePresetViaSdk) { + Y_UNIT_TEST(UseTableProfilePresetViaSdk) { TKikimrWithGrpcAndRootSchema server; - InitConfigs(server); - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( + InitConfigs(server); + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( TDriverConfig() - .SetEndpoint(location)); + .SetEndpoint(location)); auto client = NYdb::NTable::TTableClient(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Uint64); - tableBuilder.SetPrimaryKeyColumn("Key"); - auto settings = TCreateTableSettings().PresetName("profile1"); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Uint64); + tableBuilder.SetPrimaryKeyColumn("Key"); + auto settings = TCreateTableSettings().PresetName("profile1"); auto res = session.CreateTable("/Root/ydb_ut_tenant/table-1", tableBuilder.Build(), settings).ExtractValueSync(); - + NKikimrSchemeOp::TTableDescription description; - Apply(COMPACTION_POLICY1, description); - Apply(EXECUTION_POLICY1, description); - Apply(PARTITIONING_POLICY1, description); - Apply(STORAGE_POLICY1, description); + Apply(COMPACTION_POLICY1, description); + Apply(EXECUTION_POLICY1, description); + Apply(PARTITIONING_POLICY1, description); + Apply(STORAGE_POLICY1, description); Apply(REPLICATION_POLICY1, description); Apply(CACHING_POLICY1, description); - + CheckTableSettings(server, "/Root/ydb_ut_tenant/table-1", description); - } - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNullableColumn("Key", EPrimitiveType::Uint64); - tableBuilder.SetPrimaryKeyColumn("Key"); - auto settings = TCreateTableSettings().PresetName("profile2"); + } + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::Uint64); + tableBuilder.SetPrimaryKeyColumn("Key"); + auto settings = TCreateTableSettings().PresetName("profile2"); auto res = session.CreateTable("/Root/ydb_ut_tenant/table-2", tableBuilder.Build(), settings).ExtractValueSync(); - + NKikimrSchemeOp::TTableDescription description; - Apply(COMPACTION_POLICY2, description); - Apply(EXECUTION_POLICY2, description); - Apply(PARTITIONING_POLICY2, description); - Apply(STORAGE_POLICY2, description); + Apply(COMPACTION_POLICY2, description); + Apply(EXECUTION_POLICY2, description); + Apply(PARTITIONING_POLICY2, description); + Apply(STORAGE_POLICY2, description); Apply(REPLICATION_POLICY2, description); Apply(CACHING_POLICY2, description); - + CheckTableSettings(server, "/Root/ydb_ut_tenant/table-2", description); - } - - { - auto res = session.ExecuteSchemeQuery("CREATE TABLE [/Root/ydb_ut_tenant/table-3] (Key Uint64, Value Utf8, PRIMARY KEY (Key))").ExtractValueSync(); + } + + { + auto res = session.ExecuteSchemeQuery("CREATE TABLE [/Root/ydb_ut_tenant/table-3] (Key Uint64, Value Utf8, PRIMARY KEY (Key))").ExtractValueSync(); NKikimrSchemeOp::TTableDescription defaultDescription; - defaultDescription.MutablePartitionConfig()->MutableCompactionPolicy()->CopyFrom(DEFAULT_COMPACTION_POLICY); + defaultDescription.MutablePartitionConfig()->MutableCompactionPolicy()->CopyFrom(DEFAULT_COMPACTION_POLICY); //defaultDescription.MutablePartitionConfig()->SetChannelProfileId(0); - - CheckTableSettings(server, "/Root/ydb_ut_tenant/table-3", defaultDescription); - } - - } - - + + CheckTableSettings(server, "/Root/ydb_ut_tenant/table-3", defaultDescription); + } + + } + + Y_UNIT_TEST(OverwriteCompactionPolicy) { TKikimrWithGrpcAndRootSchema server; InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_compaction_policy()->set_preset_name("compaction2"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile); @@ -3921,7 +3921,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_compaction_policy()->set_preset_name("default"); CreateTable(server, "/Root/ydb_ut_tenant/table-2", profile); @@ -3944,7 +3944,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_execution_policy()->set_preset_name("execution2"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile); @@ -3961,7 +3961,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_execution_policy()->set_preset_name("default"); CreateTable(server, "/Root/ydb_ut_tenant/table-2", profile); @@ -3982,7 +3982,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_partitioning_policy()->set_preset_name("partitioning2"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile); @@ -3999,7 +3999,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_partitioning_policy()->set_preset_name("default"); CreateTable(server, "/Root/ydb_ut_tenant/table-2", profile); @@ -4015,9 +4015,9 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); - profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::DISABLED); + profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::DISABLED); profile.mutable_partitioning_policy()->set_uniform_partitions(5); CreateTable(server, "/Root/ydb_ut_tenant/table-3", profile); @@ -4037,10 +4037,10 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_partitioning_policy()->set_preset_name("partitioning2"); - profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::DISABLED); + profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::DISABLED); profile.mutable_partitioning_policy()->set_uniform_partitions(5); CreateTable(server, "/Root/ydb_ut_tenant/table-4", profile); @@ -4060,10 +4060,10 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_partitioning_policy()->set_preset_name("default"); - profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::AUTO_SPLIT); + profile.mutable_partitioning_policy()->set_auto_partitioning(Ydb::Table::PartitioningPolicy::AUTO_SPLIT); CreateTable(server, "/Root/ydb_ut_tenant/table-5", profile); NKikimrSchemeOp::TTableDescription description; @@ -4082,22 +4082,22 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } - Y_UNIT_TEST(DescribeTableWithPartitioningPolicy) { - TKikimrWithGrpcAndRootSchema server; - InitConfigs(server); - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver( - TDriverConfig() - .SetEndpoint(location)); - auto client = NYdb::NTable::TTableClient(connection); - auto session = client.CreateSession().ExtractValueSync().GetSession(); - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder + Y_UNIT_TEST(DescribeTableWithPartitioningPolicy) { + TKikimrWithGrpcAndRootSchema server; + InitConfigs(server); + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver( + TDriverConfig() + .SetEndpoint(location)); + auto client = NYdb::NTable::TTableClient(connection); + auto session = client.CreateSession().ExtractValueSync().GetSession(); + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder .AddNullableColumn("Data", EPrimitiveType::String) .AddNullableColumn("KeyHash", EPrimitiveType::Uint64) .AddNullableColumn("Version", EPrimitiveType::Uint32) @@ -4105,79 +4105,79 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { .AddNullableColumn("SubKey", EPrimitiveType::Int32) .AddNullableColumn("Key", EPrimitiveType::Utf8); tableBuilder.SetPrimaryKeyColumns({"KeyHash", "Key", "SubKey"}); - auto settings = TCreateTableSettings().PresetName("profile1"); - auto res = session.CreateTable("/Root/ydb_ut_tenant/table-1", tableBuilder.Build(), settings).ExtractValueSync(); - + auto settings = TCreateTableSettings().PresetName("profile1"); + auto res = session.CreateTable("/Root/ydb_ut_tenant/table-1", tableBuilder.Build(), settings).ExtractValueSync(); + NKikimrSchemeOp::TTableDescription description; - Apply(COMPACTION_POLICY1, description); - Apply(EXECUTION_POLICY1, description); - Apply(PARTITIONING_POLICY1, description); - Apply(STORAGE_POLICY1, description); - Apply(REPLICATION_POLICY1, description); - Apply(CACHING_POLICY1, description); - - CheckTableSettings(server, "/Root/ydb_ut_tenant/table-1", description); - - auto descResult = session.DescribeTable( - "/Root/ydb_ut_tenant/table-1", - TDescribeTableSettings().WithKeyShardBoundary(true) - ).GetValueSync(); - - UNIT_ASSERT(descResult.IsSuccess()); - auto ranges = descResult.GetTableDescription().GetKeyRanges(); - UNIT_ASSERT_VALUES_EQUAL(ranges.size(), 10); - - auto extractValue = [](const TValue& val) { - auto parser = TValueParser(val); - parser.OpenTuple(); - UNIT_ASSERT(parser.TryNextElement()); - return parser.GetOptionalUint64().GetRef(); - }; - - int n = 0; - const ui64 expectedRanges[10] = { - 1844674407370955264ul, - 3689348814741910528ul, - 5534023222112865280ul, - 7378697629483821056ul, - 9223372036854775808ul, - 11068046444225730560ul, - 12912720851596685312ul, - 14757395258967642112ul, - 16602069666338596864ul - }; - - for (const auto& range : ranges) { - if (n == 0) { + Apply(COMPACTION_POLICY1, description); + Apply(EXECUTION_POLICY1, description); + Apply(PARTITIONING_POLICY1, description); + Apply(STORAGE_POLICY1, description); + Apply(REPLICATION_POLICY1, description); + Apply(CACHING_POLICY1, description); + + CheckTableSettings(server, "/Root/ydb_ut_tenant/table-1", description); + + auto descResult = session.DescribeTable( + "/Root/ydb_ut_tenant/table-1", + TDescribeTableSettings().WithKeyShardBoundary(true) + ).GetValueSync(); + + UNIT_ASSERT(descResult.IsSuccess()); + auto ranges = descResult.GetTableDescription().GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(ranges.size(), 10); + + auto extractValue = [](const TValue& val) { + auto parser = TValueParser(val); + parser.OpenTuple(); + UNIT_ASSERT(parser.TryNextElement()); + return parser.GetOptionalUint64().GetRef(); + }; + + int n = 0; + const ui64 expectedRanges[10] = { + 1844674407370955264ul, + 3689348814741910528ul, + 5534023222112865280ul, + 7378697629483821056ul, + 9223372036854775808ul, + 11068046444225730560ul, + 12912720851596685312ul, + 14757395258967642112ul, + 16602069666338596864ul + }; + + for (const auto& range : ranges) { + if (n == 0) { UNIT_ASSERT(!range.From()); - } else { + } else { UNIT_ASSERT(range.From()->IsInclusive()); - auto left = extractValue(range.From()->GetValue()); + auto left = extractValue(range.From()->GetValue()); UNIT_ASSERT_VALUES_EQUAL(left, expectedRanges[n - 1]); - } - + } + if (n == 9) { UNIT_ASSERT(!range.To()); } else { UNIT_ASSERT(!range.To()->IsInclusive()); - auto right = extractValue(range.To()->GetValue()); - UNIT_ASSERT_VALUES_EQUAL(right, expectedRanges[n]); - } + auto right = extractValue(range.To()->GetValue()); + UNIT_ASSERT_VALUES_EQUAL(right, expectedRanges[n]); + } ++n; - } - - } - } - + } + + } + } + Y_UNIT_TEST(OverwriteStoragePolicy) { TKikimrWithGrpcAndRootSchema server; server.Server_->GetRuntime()->GetAppData().FeatureFlags.SetEnablePublicApiKeepInMemory(true); InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->set_preset_name("storage2"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile); @@ -4194,7 +4194,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->set_preset_name("default"); CreateTable(server, "/Root/ydb_ut_tenant/table-2", profile); @@ -4212,7 +4212,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->mutable_syslog()->set_media("ssd"); profile.mutable_storage_policy()->mutable_data()->set_media("ssd"); @@ -4237,7 +4237,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->set_preset_name("storage2"); profile.mutable_storage_policy()->mutable_log()->set_media("hdd"); @@ -4263,7 +4263,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->set_preset_name("default"); profile.mutable_storage_policy()->mutable_syslog()->set_media("ssd"); @@ -4333,48 +4333,48 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { InitConfigs(server); { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("unknown"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_compaction_policy(); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_compaction_policy()->set_preset_name("unknown"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_execution_policy(); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_execution_policy()->set_preset_name("unknown"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_partitioning_policy()->set_preset_name("unknown"); CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::BAD_REQUEST); } { - Ydb::Table::TableProfile profile; + Ydb::Table::TableProfile profile; profile.set_preset_name("profile1"); profile.mutable_storage_policy()->set_preset_name("unknown"); CreateTable(server, "/Root/table-1", profile, Ydb::StatusIds::BAD_REQUEST); @@ -4559,8 +4559,8 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { std::shared_ptr<grpc::Channel> Channel_; Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GetPort()), grpc::InsecureChannelCredentials()); - std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; - Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> Stub_; + Stub_ = Ydb::Table::V1::TableService::NewStub(Channel_); grpc::ClientContext context; Ydb::Table::DescribeTableOptionsRequest request; Ydb::Table::DescribeTableOptionsResponse response; @@ -4727,8 +4727,8 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { void ExecSchemeYql(std::shared_ptr<grpc::Channel> channel, const TString &sessionId, const TString &yql) { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; - stub = Ydb::Table::V1::TableService::NewStub(channel); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; + stub = Ydb::Table::V1::TableService::NewStub(channel); grpc::ClientContext context; Ydb::Table::ExecuteSchemeQueryRequest request; @@ -4744,28 +4744,28 @@ void ExecSchemeYql(std::shared_ptr<grpc::Channel> channel, const TString &sessio UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); } -Ydb::Table::ExecuteQueryResult ExecYql(std::shared_ptr<grpc::Channel> channel, const TString &sessionId, const TString &yql, bool withStat) +Ydb::Table::ExecuteQueryResult ExecYql(std::shared_ptr<grpc::Channel> channel, const TString &sessionId, const TString &yql, bool withStat) { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; - stub = Ydb::Table::V1::TableService::NewStub(channel); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; + stub = Ydb::Table::V1::TableService::NewStub(channel); grpc::ClientContext context; Ydb::Table::ExecuteDataQueryRequest request; request.set_session_id(sessionId); request.mutable_query()->set_yql_text(yql); request.mutable_tx_control()->mutable_begin_tx()->mutable_serializable_read_write(); request.mutable_tx_control()->set_commit_tx(true); - if (withStat) { + if (withStat) { request.set_collect_stats(Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC); - } + } Ydb::Table::ExecuteDataQueryResponse response; auto status = stub->ExecuteDataQuery(&context, request, &response); UNIT_ASSERT(status.ok()); auto deferred = response.operation(); UNIT_ASSERT(deferred.ready() == true); - NYql::TIssues issues; - NYql::IssuesFromMessage(deferred.issues(), issues); - issues.PrintTo(Cerr); - + NYql::TIssues issues; + NYql::IssuesFromMessage(deferred.issues(), issues); + issues.PrintTo(Cerr); + UNIT_ASSERT(deferred.status() == Ydb::StatusIds::SUCCESS); Ydb::Table::ExecuteQueryResult result; @@ -4794,15 +4794,15 @@ void CheckYqlDecimalValues(std::shared_ptr<grpc::Channel> channel, const TString UNIT_ASSERT_VALUES_EQUAL(result_set.rows_size(), halves.size()); for (size_t i = 0; i < halves.size(); ++i) { UNIT_ASSERT_VALUES_EQUAL(result_set.rows(i).items(0).low_128(), halves[i].first); - UNIT_ASSERT_VALUES_EQUAL(result_set.rows(i).items(0).high_128(), halves[i].second); + UNIT_ASSERT_VALUES_EQUAL(result_set.rows(i).items(0).high_128(), halves[i].second); } } void CreateTable(std::shared_ptr<grpc::Channel> channel, const Ydb::Table::CreateTableRequest &request) { - std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; - stub = Ydb::Table::V1::TableService::NewStub(channel); + std::unique_ptr<Ydb::Table::V1::TableService::Stub> stub; + stub = Ydb::Table::V1::TableService::NewStub(channel); grpc::ClientContext context; Ydb::Table::CreateTableResponse response; auto status = stub->CreateTable(&context, request, &response); @@ -4818,7 +4818,7 @@ void CreateTable(std::shared_ptr<grpc::Channel> channel) request.set_path("/Root/table-1"); auto &col1 = *request.add_columns(); col1.set_name("key"); - col1.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::INT32); + col1.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::INT32); auto &col2 = *request.add_columns(); col2.set_name("value"); auto &decimalType = *col2.mutable_type()->mutable_optional_type()->mutable_item()->mutable_decimal_type(); @@ -5354,4 +5354,4 @@ Y_UNIT_TEST(LocksFromAnotherTenants) { } } -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/services/yq/private_grpc.cpp b/ydb/services/yq/private_grpc.cpp index e259c92ecf..c4c076b482 100644 --- a/ydb/services/yq/private_grpc.cpp +++ b/ydb/services/yq/private_grpc.cpp @@ -38,18 +38,18 @@ void TGRpcYqPrivateTaskService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) #ifdef ADD_REQUEST #error ADD_REQUEST macro already defined #endif -#define ADD_REQUEST(NAME, CB) \ +#define ADD_REQUEST(NAME, CB) \ MakeIntrusive<TGRpcRequest<Yq::Private::NAME##Request, Yq::Private::NAME##Response, TGRpcYqPrivateTaskService, TSecurityTextFormatPrinter<Yq::Private::NAME##Request>, TSecurityTextFormatPrinter<Yq::Private::NAME##Response>>>( \ - this, &Service_, CQ_, \ - [this](NGrpc::IRequestContextBase *ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ActorSystem_->Send(GRpcRequestProxyId_, \ + this, &Service_, CQ_, \ + [this](NGrpc::IRequestContextBase *ctx) { \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ActorSystem_->Send(GRpcRequestProxyId_, \ new TGrpcRequestOperationCall<Yq::Private::NAME##Request, Yq::Private::NAME##Response> \ - (ctx, &CB)); \ - }, \ + (ctx, &CB)); \ + }, \ &Yq::Private::V1::YqPrivateTaskService::AsyncService::Request##NAME, \ - #NAME, logger, getCounterBlock("yql_internal", #NAME)) \ - ->Run(); \ + #NAME, logger, getCounterBlock("yql_internal", #NAME)) \ + ->Run(); \ ADD_REQUEST(PingTask, DoYqPrivatePingTaskRequest) diff --git a/ydb/tests/functional/api/test_read_table.py b/ydb/tests/functional/api/test_read_table.py index 544332597c..481e37fea6 100644 --- a/ydb/tests/functional/api/test_read_table.py +++ b/ydb/tests/functional/api/test_read_table.py @@ -315,7 +315,7 @@ class TestReadTableWithTabletKills(AbstractReadTableTest): data_chunks.append(data) total_rows += len(data.rows) logger.debug(iteration) - except ydb.Unavailable as e: + except ydb.Unavailable as e: self.logger.debug("received error, %s", e) continue diff --git a/ydb/tests/functional/tenants/common.py b/ydb/tests/functional/tenants/common.py index d37cbfcf30..827005e5ae 100644 --- a/ydb/tests/functional/tenants/common.py +++ b/ydb/tests/functional/tenants/common.py @@ -73,7 +73,7 @@ class DBForStaticSlots(object): 'TX_DATASHARD': LogLevels.DEBUG, 'FLAT_TX_SCHEMESHARD': LogLevels.DEBUG, 'TX_PROXY_SCHEME_CACHE': LogLevels.DEBUG, - 'GRPC_SERVER': LogLevels.DEBUG, + 'GRPC_SERVER': LogLevels.DEBUG, } ), ) @@ -134,7 +134,7 @@ class DBWithDynamicSlot(object): 'NODE_BROKER': LogLevels.DEBUG, 'TX_DATASHARD': LogLevels.DEBUG, 'TX_PROXY': LogLevels.DEBUG, - 'GRPC_SERVER': LogLevels.DEBUG, + 'GRPC_SERVER': LogLevels.DEBUG, } ) ) diff --git a/ydb/tests/functional/tenants/test_dynamic_tenants.py b/ydb/tests/functional/tenants/test_dynamic_tenants.py index 1fbb37a964..bdf7d59782 100644 --- a/ydb/tests/functional/tenants/test_dynamic_tenants.py +++ b/ydb/tests/functional/tenants/test_dynamic_tenants.py @@ -44,58 +44,58 @@ class TestCreateTenantWithCPU(DBWithDynamicSlot): self.cluster.remove_database(database) -class TestCreateTenantThenExecYQLEmptyDatabaseHeader(DBWithDynamicSlot): - def test_case(self): - database = '/Root/users/database' - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - database - ) - - self.cluster.create_database( - database, - storage_pool_units_count={ - 'hdd': 1 - } - ) +class TestCreateTenantThenExecYQLEmptyDatabaseHeader(DBWithDynamicSlot): + def test_case(self): + database = '/Root/users/database' + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + database + ) + + self.cluster.create_database( + database, + storage_pool_units_count={ + 'hdd': 1 + } + ) self.cluster.register_and_start_slots(database, count=1) self.cluster.wait_tenant_up(database) - - def list_endpoints(database): - logger.debug("List endpoints of %s", database) - resolver = ydb.DiscoveryEndpointsResolver(driver_config) - result = resolver.resolve() - if result is not None: - return result.endpoints - return result - - endpoints = list_endpoints(database) - - driver_config2 = ydb.DriverConfig( - "%s" % endpoints[0].endpoint, - None, - credentials=ydb.AuthTokenCredentials("root@builtin") - ) - - table_path = '%s/table-1' % database - with ydb.Driver(driver_config2) as driver: - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - session.execute_scheme( + + def list_endpoints(database): + logger.debug("List endpoints of %s", database) + resolver = ydb.DiscoveryEndpointsResolver(driver_config) + result = resolver.resolve() + if result is not None: + return result.endpoints + return result + + endpoints = list_endpoints(database) + + driver_config2 = ydb.DriverConfig( + "%s" % endpoints[0].endpoint, + None, + credentials=ydb.AuthTokenCredentials("root@builtin") + ) + + table_path = '%s/table-1' % database + with ydb.Driver(driver_config2) as driver: + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + session.execute_scheme( "create table `{}` (key Int32, value String, primary key(key));".format( - table_path - ) - ) - - session.transaction().execute( + table_path + ) + ) + + session.transaction().execute( "upsert into `{}` (key) values (101);".format(table_path), - commit_tx=True - ) - + commit_tx=True + ) + session.transaction().execute("select key from `{}`;".format(table_path), commit_tx=True) - - + + class TestCreateTenantThenExecYQL(DBWithDynamicSlot): def test_case(self): database = '/Root/users/database' @@ -105,11 +105,11 @@ class TestCreateTenantThenExecYQL(DBWithDynamicSlot): database ) - driver_config2 = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - database + "/" - ) - + driver_config2 = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + database + "/" + ) + self.cluster.create_database( database, storage_pool_units_count={ @@ -119,22 +119,22 @@ class TestCreateTenantThenExecYQL(DBWithDynamicSlot): self.cluster.register_and_start_slots(database, count=1) self.cluster.wait_tenant_up(database) - d_configs = [driver_config, driver_config2] - for d_config in d_configs: - table_path = '%s/table-1' % database - with ydb.Driver(d_config) as driver: - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - session.execute_scheme( + d_configs = [driver_config, driver_config2] + for d_config in d_configs: + table_path = '%s/table-1' % database + with ydb.Driver(d_config) as driver: + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + session.execute_scheme( "create table `{}` (key Int32, value String, primary key(key));".format( - table_path - ) + table_path + ) ) - session.transaction().execute( + session.transaction().execute( "upsert into `{}` (key) values (101);".format(table_path), - commit_tx=True - ) + commit_tx=True + ) session.transaction().execute("select key from `{}`;".format(table_path), commit_tx=True) diff --git a/ydb/tests/library/common/protobuf_console.py b/ydb/tests/library/common/protobuf_console.py index 98677f067a..2706cfc1b0 100644 --- a/ydb/tests/library/common/protobuf_console.py +++ b/ydb/tests/library/common/protobuf_console.py @@ -65,8 +65,8 @@ class CreateTenantRequest(AbstractProtobufBuilder): if soft is not None: quotas.data_size_soft_quota = soft - def set_attribute(self, name, value): - self.protobuf.CreateTenantRequest.Request.attributes[name] = value + def set_attribute(self, name, value): + self.protobuf.CreateTenantRequest.Request.attributes[name] = value class AlterTenantRequest(AbstractProtobufBuilder): diff --git a/ydb/tests/library/harness/kikimr_client.py b/ydb/tests/library/harness/kikimr_client.py index 587f176390..d74383ae23 100644 --- a/ydb/tests/library/harness/kikimr_client.py +++ b/ydb/tests/library/harness/kikimr_client.py @@ -19,7 +19,7 @@ from ydb.core.protos import flat_scheme_op_pb2 as flat_scheme_op from ydb.public.api.protos.ydb_status_codes_pb2 import StatusIds from collections import namedtuple - + class FlatTxId(namedtuple('FlatTxId', ['tx_id', 'schemeshard_tablet_id'])): @staticmethod def from_response(response): @@ -172,20 +172,20 @@ class KiKiMRMessageBusClient(object): def ddl_exec_status(self, flat_tx_id): return self.flat_transaction_status(flat_tx_id.tx_id, flat_tx_id.schemeshard_tablet_id) - def add_attr(self, working_dir, name, attributes, token=None): - request = msgbus.TSchemeOperation() - request.Transaction.ModifyScheme.OperationType = flat_scheme_op_pb2.ESchemeOpAlterUserAttributes - request.Transaction.ModifyScheme.WorkingDir = working_dir - request.Transaction.ModifyScheme.AlterUserAttributes.PathName = name - - if attributes is not None: - for name, value in attributes.items(): - request.Transaction.ModifyScheme.AlterUserAttributes.UserAttributes.add(Key=name, Value=value) - - if token: - request.SecurityToken = token - return self.send_and_poll_request(request, 'SchemeOperation') - + def add_attr(self, working_dir, name, attributes, token=None): + request = msgbus.TSchemeOperation() + request.Transaction.ModifyScheme.OperationType = flat_scheme_op_pb2.ESchemeOpAlterUserAttributes + request.Transaction.ModifyScheme.WorkingDir = working_dir + request.Transaction.ModifyScheme.AlterUserAttributes.PathName = name + + if attributes is not None: + for name, value in attributes.items(): + request.Transaction.ModifyScheme.AlterUserAttributes.UserAttributes.add(Key=name, Value=value) + + if token: + request.SecurityToken = token + return self.send_and_poll_request(request, 'SchemeOperation') + def upgrade_subdomain(self, working_dir, name, token=None): request = msgbus.TSchemeOperation() request.Transaction.ModifyScheme.OperationType = flat_scheme_op_pb2.ESchemeOpUpgradeSubDomain diff --git a/ydb/tests/library/harness/kikimr_cluster_interface.py b/ydb/tests/library/harness/kikimr_cluster_interface.py index ee3b75f0da..96a891da11 100644 --- a/ydb/tests/library/harness/kikimr_cluster_interface.py +++ b/ydb/tests/library/harness/kikimr_cluster_interface.py @@ -229,7 +229,7 @@ class KiKiMRClusterInterface(object): if attributes is not None: for name, value in attributes.items(): - req.set_attribute(name, value) + req.set_attribute(name, value) response = self.client.send_request(req.protobuf, method='ConsoleRequest') operation = response.CreateTenantResponse.Response.operation diff --git a/ydb/tests/library/matchers/datashard_matchers.py b/ydb/tests/library/matchers/datashard_matchers.py index e1d178975f..94ac5e9a9a 100644 --- a/ydb/tests/library/matchers/datashard_matchers.py +++ b/ydb/tests/library/matchers/datashard_matchers.py @@ -18,6 +18,6 @@ class TResponseMatcher(AbstractProtobufMatcher): self.expected_protobuf.MiniKQLCompileResults.ProgramCompileErrors = [] self.expected_protobuf.MiniKQLCompileResults.ProgramCompileErrors.append( - FakeProtobuf(message=message) + FakeProtobuf(message=message) ) return self diff --git a/ydb/tests/library/sqs/__init__.py b/ydb/tests/library/sqs/__init__.py index 1ea5f23f11..faa18be5bb 100644 --- a/ydb/tests/library/sqs/__init__.py +++ b/ydb/tests/library/sqs/__init__.py @@ -1,2 +1,2 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python +# -*- coding: utf-8 -*- diff --git a/ydb/tests/library/sqs/tables.py b/ydb/tests/library/sqs/tables.py index c7db27191a..c94241d538 100644 --- a/ydb/tests/library/sqs/tables.py +++ b/ydb/tests/library/sqs/tables.py @@ -1,276 +1,276 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import ydb - -from enum import Enum - - -class QueueType(Enum): - STD = 1 - FIFO = 2 - - -def get_prefix_path(root, queue_type=None): - subdir = ('.{}/'.format(queue_type.name)) if queue_type else '' - return '{}/{}'.format(root, subdir) - - -def get_table_path(root, table_name, queue_type=None): - return '{}/{}'.format(get_prefix_path(root, queue_type), table_name) - - -def _create_table(root, session, table_name, columns, keys_count, queue_type=None): - table_path = get_table_path(root, table_name, queue_type) - keys = [name for name, _ in columns[:keys_count]] - columns = [ydb.Column(name, ydb.OptionalType(column_type)) for name, column_type in columns] - ydb.retry_operation_sync(lambda: session.create_table( - table_path, - ydb.TableDescription() - .with_primary_keys(*keys) - .with_columns(*columns) - .with_profile( - ydb.TableProfile() - .with_partitioning_policy( - ydb.PartitioningPolicy() - .with_auto_partitioning( - ydb.AutoPartitioningPolicy.AUTO_SPLIT - ) - ) - ) - )) - - -def get_table_keys_for_queue(with_shard=False): - if with_shard: - columns = [ - ('QueueIdNumberAndShardHash', ydb.PrimitiveType.Uint64), - ('QueueIdNumber', ydb.PrimitiveType.Uint64), - ('Shard', ydb.PrimitiveType.Uint32) - ] - else: - columns = [ - ('QueueIdNumberHash', ydb.PrimitiveType.Uint64), - ('QueueIdNumber', ydb.PrimitiveType.Uint64) - ] - return columns - - -def create_atomic_counter(root, session): - columns = [ - ('counter_key', ydb.PrimitiveType.Uint64), - ('value', ydb.PrimitiveType.Uint64) - ] - table_name = '.AtomicCounter' - _create_table(root, session, table_name, columns, keys_count=1) - - session.transaction(ydb.SerializableReadWrite()).execute( - "--!syntax_v1\n" - "INSERT INTO `{}` (counter_key, value) VALUES (0, 0);".format(get_table_path(root, table_name)), - commit_tx=True - ) - - -def create_queues_table(root, session): - columns = [ - ('Account', ydb.PrimitiveType.Utf8), - ('QueueName', ydb.PrimitiveType.Utf8), - ('QueueId', ydb.PrimitiveType.String), - ('QueueState', ydb.PrimitiveType.Uint64), - ('FifoQueue', ydb.PrimitiveType.Bool), - ('DeadLetterQueue', ydb.PrimitiveType.Bool), - ('CreatedTimestamp', ydb.PrimitiveType.Uint64), - ('Shards', ydb.PrimitiveType.Uint64), - ('Partitions', ydb.PrimitiveType.Uint64), - ('MasterTabletId', ydb.PrimitiveType.Uint64), - ('CustomQueueName', ydb.PrimitiveType.Utf8), - ('FolderId', ydb.PrimitiveType.Utf8), - ('Version', ydb.PrimitiveType.Uint64), - ('DlqName', ydb.PrimitiveType.Utf8), - ('TablesFormat', ydb.PrimitiveType.Uint32), - ] - _create_table(root, session, '.Queues', columns, keys_count=2) - - -def create_events_table(root, session): - columns = [ - ('Account', ydb.PrimitiveType.Utf8), - ('QueueName', ydb.PrimitiveType.Utf8), - ('EventType', ydb.PrimitiveType.Uint64), - ('CustomQueueName', ydb.PrimitiveType.Utf8), - ('EventTimestamp', ydb.PrimitiveType.Uint64), - ('FolderId', ydb.PrimitiveType.Utf8), - ] - _create_table(root, session, '.Events', columns, keys_count=3) - - -def create_settings_table(root, session): - columns = [ - ('Account', ydb.PrimitiveType.Utf8), - ('Name', ydb.PrimitiveType.Utf8), - ('Value', ydb.PrimitiveType.Utf8), - ] - _create_table(root, session, '.Settings', columns, keys_count=2) - - -def create_attibutes_table(root, session, queue_type): - queue_keys = get_table_keys_for_queue() - columns = queue_keys + [ - ('ContentBasedDeduplication', ydb.PrimitiveType.Bool), - ('DelaySeconds', ydb.PrimitiveType.Uint64), - ('FifoQueue', ydb.PrimitiveType.Bool), - ('MaximumMessageSize', ydb.PrimitiveType.Uint64), - ('MessageRetentionPeriod', ydb.PrimitiveType.Uint64), - ('ReceiveMessageWaitTime', ydb.PrimitiveType.Uint64), - ('VisibilityTimeout', ydb.PrimitiveType.Uint64), - ('DlqName', ydb.PrimitiveType.Utf8), - ('DlqArn', ydb.PrimitiveType.Utf8), - ('MaxReceiveCount', ydb.PrimitiveType.Uint64), - ('ShowDetailedCountersDeadline', ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'Attributes', columns, len(queue_keys), queue_type) - - -def create_state_table(root, session, queue_type): - queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) - columns = queue_keys + [ - ('CleanupTimestamp', ydb.PrimitiveType.Uint64), - ('CreatedTimestamp', ydb.PrimitiveType.Uint64), - ('LastModifiedTimestamp', ydb.PrimitiveType.Uint64), - ('RetentionBoundary', ydb.PrimitiveType.Uint64), - ('InflyCount', ydb.PrimitiveType.Int64), - ('MessageCount', ydb.PrimitiveType.Int64), - ('ReadOffset', ydb.PrimitiveType.Uint64), - ('WriteOffset', ydb.PrimitiveType.Uint64), - ('CleanupVersion', ydb.PrimitiveType.Uint64), - ('InflyVersion', ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'State', columns, len(queue_keys), queue_type) - - -def create_infly_table(root, session): - queue_type = QueueType.STD - queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) - columns = queue_keys + [ - ('Offset', ydb.PrimitiveType.Uint64), - ('RandomId', ydb.PrimitiveType.Uint64), - ('LoadId', ydb.PrimitiveType.Uint64), - ('FirstReceiveTimestamp', ydb.PrimitiveType.Uint64), - ('LockTimestamp', ydb.PrimitiveType.Uint64), - ('ReceiveCount', ydb.PrimitiveType.Uint32), - ('SentTimestamp', ydb.PrimitiveType.Uint64), - ('VisibilityDeadline', ydb.PrimitiveType.Uint64), - ('DelayDeadline', ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'Infly', columns, len(queue_keys) + 1, queue_type) - - -def create_message_data_table(root, session): - queue_type = QueueType.STD - queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) - columns = queue_keys + [ - ('RandomId', ydb.PrimitiveType.Uint64), - ('Offset', ydb.PrimitiveType.Uint64), - ('Attributes', ydb.PrimitiveType.String), - ('Data', ydb.PrimitiveType.String), - ('MessageId', ydb.PrimitiveType.String), - ('SenderId', ydb.PrimitiveType.String), - ] - _create_table(root, session, 'MessageData', columns, len(queue_keys) + 2, queue_type) - - -def create_message_table(root, session, queue_type): - queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) - columns = queue_keys + [ - ('Offset', ydb.PrimitiveType.Uint64), - ('RandomId', ydb.PrimitiveType.Uint64), - ('SentTimestamp', ydb.PrimitiveType.Uint64), - ('DelayDeadline', ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'Messages', columns, len(queue_keys) + 1, queue_type) - - -def create_sent_timestamp_idx_table(root, session, queue_type): - queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) - columns = queue_keys + [ - ('SentTimestamp', ydb.PrimitiveType.Uint64), - ('Offset', ydb.PrimitiveType.Uint64), - ('RandomId', ydb.PrimitiveType.Uint64), - ('DelayDeadline', ydb.PrimitiveType.Uint64) - ] - _create_table(root, session, 'SentTimestampIdx', columns, len(queue_keys) + 2, queue_type) - - -def create_data_table(root, session): - queue_type = QueueType.FIFO - queue_keys = get_table_keys_for_queue() - columns = queue_keys + [ - ("RandomId", ydb.PrimitiveType.Uint64), - ("Offset", ydb.PrimitiveType.Uint64), - ("SenderId", ydb.PrimitiveType.String), - ("DedupId", ydb.PrimitiveType.String), - ("Attributes", ydb.PrimitiveType.String), - ("Data", ydb.PrimitiveType.String), - ("MessageId", ydb.PrimitiveType.String), - ] - _create_table(root, session, 'Data', columns, len(queue_keys) + 1, queue_type) - - -def create_deduplication_table(root, session): - queue_type = QueueType.FIFO - queue_keys = get_table_keys_for_queue() - columns = queue_keys + [ - ("DedupId", ydb.PrimitiveType.String), - ("Deadline", ydb.PrimitiveType.Uint64), - ("Offset", ydb.PrimitiveType.Uint64), - ("MessageId", ydb.PrimitiveType.String), - ] - _create_table(root, session, 'Deduplication', columns, len(queue_keys) + 1, queue_type) - - -def create_groups_table(root, session): - queue_type = QueueType.FIFO - queue_keys = get_table_keys_for_queue() - columns = queue_keys + [ - ("GroupId", ydb.PrimitiveType.String), - ("VisibilityDeadline", ydb.PrimitiveType.Uint64), - ("RandomId", ydb.PrimitiveType.Uint64), - ("Head", ydb.PrimitiveType.Uint64), - ("Tail", ydb.PrimitiveType.Uint64), - ("ReceiveAttemptId", ydb.PrimitiveType.Utf8), - ("LockTimestamp", ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'Groups', columns, len(queue_keys) + 1, queue_type) - - -def create_reads_table(root, session): - queue_type = QueueType.FIFO - queue_keys = get_table_keys_for_queue() - columns = queue_keys + [ - ("ReceiveAttemptId", ydb.PrimitiveType.Utf8), - ("Deadline", ydb.PrimitiveType.Uint64), - ] - _create_table(root, session, 'Reads', columns, len(queue_keys) + 1, queue_type) - - -def create_all_tables(root, driver, session): - create_atomic_counter(root, session) - create_queues_table(root, session) - create_events_table(root, session) - create_settings_table(root, session) - - for queue_type in [QueueType.STD, QueueType.FIFO]: - create_sent_timestamp_idx_table(root, session, queue_type) - create_message_table(root, session, queue_type) - create_state_table(root, session, queue_type) - create_attibutes_table(root, session, queue_type) - - # only STD - create_infly_table(root, session) - create_message_data_table(root, session) - - # only FIFO - create_reads_table(root, session) - create_groups_table(root, session) - create_deduplication_table(root, session) - create_data_table(root, session) +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import ydb + +from enum import Enum + + +class QueueType(Enum): + STD = 1 + FIFO = 2 + + +def get_prefix_path(root, queue_type=None): + subdir = ('.{}/'.format(queue_type.name)) if queue_type else '' + return '{}/{}'.format(root, subdir) + + +def get_table_path(root, table_name, queue_type=None): + return '{}/{}'.format(get_prefix_path(root, queue_type), table_name) + + +def _create_table(root, session, table_name, columns, keys_count, queue_type=None): + table_path = get_table_path(root, table_name, queue_type) + keys = [name for name, _ in columns[:keys_count]] + columns = [ydb.Column(name, ydb.OptionalType(column_type)) for name, column_type in columns] + ydb.retry_operation_sync(lambda: session.create_table( + table_path, + ydb.TableDescription() + .with_primary_keys(*keys) + .with_columns(*columns) + .with_profile( + ydb.TableProfile() + .with_partitioning_policy( + ydb.PartitioningPolicy() + .with_auto_partitioning( + ydb.AutoPartitioningPolicy.AUTO_SPLIT + ) + ) + ) + )) + + +def get_table_keys_for_queue(with_shard=False): + if with_shard: + columns = [ + ('QueueIdNumberAndShardHash', ydb.PrimitiveType.Uint64), + ('QueueIdNumber', ydb.PrimitiveType.Uint64), + ('Shard', ydb.PrimitiveType.Uint32) + ] + else: + columns = [ + ('QueueIdNumberHash', ydb.PrimitiveType.Uint64), + ('QueueIdNumber', ydb.PrimitiveType.Uint64) + ] + return columns + + +def create_atomic_counter(root, session): + columns = [ + ('counter_key', ydb.PrimitiveType.Uint64), + ('value', ydb.PrimitiveType.Uint64) + ] + table_name = '.AtomicCounter' + _create_table(root, session, table_name, columns, keys_count=1) + + session.transaction(ydb.SerializableReadWrite()).execute( + "--!syntax_v1\n" + "INSERT INTO `{}` (counter_key, value) VALUES (0, 0);".format(get_table_path(root, table_name)), + commit_tx=True + ) + + +def create_queues_table(root, session): + columns = [ + ('Account', ydb.PrimitiveType.Utf8), + ('QueueName', ydb.PrimitiveType.Utf8), + ('QueueId', ydb.PrimitiveType.String), + ('QueueState', ydb.PrimitiveType.Uint64), + ('FifoQueue', ydb.PrimitiveType.Bool), + ('DeadLetterQueue', ydb.PrimitiveType.Bool), + ('CreatedTimestamp', ydb.PrimitiveType.Uint64), + ('Shards', ydb.PrimitiveType.Uint64), + ('Partitions', ydb.PrimitiveType.Uint64), + ('MasterTabletId', ydb.PrimitiveType.Uint64), + ('CustomQueueName', ydb.PrimitiveType.Utf8), + ('FolderId', ydb.PrimitiveType.Utf8), + ('Version', ydb.PrimitiveType.Uint64), + ('DlqName', ydb.PrimitiveType.Utf8), + ('TablesFormat', ydb.PrimitiveType.Uint32), + ] + _create_table(root, session, '.Queues', columns, keys_count=2) + + +def create_events_table(root, session): + columns = [ + ('Account', ydb.PrimitiveType.Utf8), + ('QueueName', ydb.PrimitiveType.Utf8), + ('EventType', ydb.PrimitiveType.Uint64), + ('CustomQueueName', ydb.PrimitiveType.Utf8), + ('EventTimestamp', ydb.PrimitiveType.Uint64), + ('FolderId', ydb.PrimitiveType.Utf8), + ] + _create_table(root, session, '.Events', columns, keys_count=3) + + +def create_settings_table(root, session): + columns = [ + ('Account', ydb.PrimitiveType.Utf8), + ('Name', ydb.PrimitiveType.Utf8), + ('Value', ydb.PrimitiveType.Utf8), + ] + _create_table(root, session, '.Settings', columns, keys_count=2) + + +def create_attibutes_table(root, session, queue_type): + queue_keys = get_table_keys_for_queue() + columns = queue_keys + [ + ('ContentBasedDeduplication', ydb.PrimitiveType.Bool), + ('DelaySeconds', ydb.PrimitiveType.Uint64), + ('FifoQueue', ydb.PrimitiveType.Bool), + ('MaximumMessageSize', ydb.PrimitiveType.Uint64), + ('MessageRetentionPeriod', ydb.PrimitiveType.Uint64), + ('ReceiveMessageWaitTime', ydb.PrimitiveType.Uint64), + ('VisibilityTimeout', ydb.PrimitiveType.Uint64), + ('DlqName', ydb.PrimitiveType.Utf8), + ('DlqArn', ydb.PrimitiveType.Utf8), + ('MaxReceiveCount', ydb.PrimitiveType.Uint64), + ('ShowDetailedCountersDeadline', ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'Attributes', columns, len(queue_keys), queue_type) + + +def create_state_table(root, session, queue_type): + queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) + columns = queue_keys + [ + ('CleanupTimestamp', ydb.PrimitiveType.Uint64), + ('CreatedTimestamp', ydb.PrimitiveType.Uint64), + ('LastModifiedTimestamp', ydb.PrimitiveType.Uint64), + ('RetentionBoundary', ydb.PrimitiveType.Uint64), + ('InflyCount', ydb.PrimitiveType.Int64), + ('MessageCount', ydb.PrimitiveType.Int64), + ('ReadOffset', ydb.PrimitiveType.Uint64), + ('WriteOffset', ydb.PrimitiveType.Uint64), + ('CleanupVersion', ydb.PrimitiveType.Uint64), + ('InflyVersion', ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'State', columns, len(queue_keys), queue_type) + + +def create_infly_table(root, session): + queue_type = QueueType.STD + queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) + columns = queue_keys + [ + ('Offset', ydb.PrimitiveType.Uint64), + ('RandomId', ydb.PrimitiveType.Uint64), + ('LoadId', ydb.PrimitiveType.Uint64), + ('FirstReceiveTimestamp', ydb.PrimitiveType.Uint64), + ('LockTimestamp', ydb.PrimitiveType.Uint64), + ('ReceiveCount', ydb.PrimitiveType.Uint32), + ('SentTimestamp', ydb.PrimitiveType.Uint64), + ('VisibilityDeadline', ydb.PrimitiveType.Uint64), + ('DelayDeadline', ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'Infly', columns, len(queue_keys) + 1, queue_type) + + +def create_message_data_table(root, session): + queue_type = QueueType.STD + queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) + columns = queue_keys + [ + ('RandomId', ydb.PrimitiveType.Uint64), + ('Offset', ydb.PrimitiveType.Uint64), + ('Attributes', ydb.PrimitiveType.String), + ('Data', ydb.PrimitiveType.String), + ('MessageId', ydb.PrimitiveType.String), + ('SenderId', ydb.PrimitiveType.String), + ] + _create_table(root, session, 'MessageData', columns, len(queue_keys) + 2, queue_type) + + +def create_message_table(root, session, queue_type): + queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) + columns = queue_keys + [ + ('Offset', ydb.PrimitiveType.Uint64), + ('RandomId', ydb.PrimitiveType.Uint64), + ('SentTimestamp', ydb.PrimitiveType.Uint64), + ('DelayDeadline', ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'Messages', columns, len(queue_keys) + 1, queue_type) + + +def create_sent_timestamp_idx_table(root, session, queue_type): + queue_keys = get_table_keys_for_queue(with_shard=(queue_type == QueueType.STD)) + columns = queue_keys + [ + ('SentTimestamp', ydb.PrimitiveType.Uint64), + ('Offset', ydb.PrimitiveType.Uint64), + ('RandomId', ydb.PrimitiveType.Uint64), + ('DelayDeadline', ydb.PrimitiveType.Uint64) + ] + _create_table(root, session, 'SentTimestampIdx', columns, len(queue_keys) + 2, queue_type) + + +def create_data_table(root, session): + queue_type = QueueType.FIFO + queue_keys = get_table_keys_for_queue() + columns = queue_keys + [ + ("RandomId", ydb.PrimitiveType.Uint64), + ("Offset", ydb.PrimitiveType.Uint64), + ("SenderId", ydb.PrimitiveType.String), + ("DedupId", ydb.PrimitiveType.String), + ("Attributes", ydb.PrimitiveType.String), + ("Data", ydb.PrimitiveType.String), + ("MessageId", ydb.PrimitiveType.String), + ] + _create_table(root, session, 'Data', columns, len(queue_keys) + 1, queue_type) + + +def create_deduplication_table(root, session): + queue_type = QueueType.FIFO + queue_keys = get_table_keys_for_queue() + columns = queue_keys + [ + ("DedupId", ydb.PrimitiveType.String), + ("Deadline", ydb.PrimitiveType.Uint64), + ("Offset", ydb.PrimitiveType.Uint64), + ("MessageId", ydb.PrimitiveType.String), + ] + _create_table(root, session, 'Deduplication', columns, len(queue_keys) + 1, queue_type) + + +def create_groups_table(root, session): + queue_type = QueueType.FIFO + queue_keys = get_table_keys_for_queue() + columns = queue_keys + [ + ("GroupId", ydb.PrimitiveType.String), + ("VisibilityDeadline", ydb.PrimitiveType.Uint64), + ("RandomId", ydb.PrimitiveType.Uint64), + ("Head", ydb.PrimitiveType.Uint64), + ("Tail", ydb.PrimitiveType.Uint64), + ("ReceiveAttemptId", ydb.PrimitiveType.Utf8), + ("LockTimestamp", ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'Groups', columns, len(queue_keys) + 1, queue_type) + + +def create_reads_table(root, session): + queue_type = QueueType.FIFO + queue_keys = get_table_keys_for_queue() + columns = queue_keys + [ + ("ReceiveAttemptId", ydb.PrimitiveType.Utf8), + ("Deadline", ydb.PrimitiveType.Uint64), + ] + _create_table(root, session, 'Reads', columns, len(queue_keys) + 1, queue_type) + + +def create_all_tables(root, driver, session): + create_atomic_counter(root, session) + create_queues_table(root, session) + create_events_table(root, session) + create_settings_table(root, session) + + for queue_type in [QueueType.STD, QueueType.FIFO]: + create_sent_timestamp_idx_table(root, session, queue_type) + create_message_table(root, session, queue_type) + create_state_table(root, session, queue_type) + create_attibutes_table(root, session, queue_type) + + # only STD + create_infly_table(root, session) + create_message_data_table(root, session) + + # only FIFO + create_reads_table(root, session) + create_groups_table(root, session) + create_deduplication_table(root, session) + create_data_table(root, session) diff --git a/ydb/tests/library/sqs/ya.make b/ydb/tests/library/sqs/ya.make index 3a85f2ba34..2e97b73290 100644 --- a/ydb/tests/library/sqs/ya.make +++ b/ydb/tests/library/sqs/ya.make @@ -1,20 +1,20 @@ -PY23_LIBRARY() - -OWNER(g:kikimr) - -PY_SRCS( - __init__.py - tables.py -) - -IF (NOT PYTHON3) - PEERDIR( - contrib/python/enum34 - ) -ENDIF() - -PEERDIR( - ydb/public/sdk/python/ydb -) - -END() +PY23_LIBRARY() + +OWNER(g:kikimr) + +PY_SRCS( + __init__.py + tables.py +) + +IF (NOT PYTHON3) + PEERDIR( + contrib/python/enum34 + ) +ENDIF() + +PEERDIR( + ydb/public/sdk/python/ydb +) + +END() |