diff options
| author | uzhas <uzhas@yandex-team.ru> | 2022-02-10 16:47:06 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:06 +0300 |
| commit | 2002caab389ddffd0e7db4d8c4db835fa4c99f83 (patch) | |
| tree | 9d3448d5ba97edddf88aa69bcb138e88286e8176 | |
| parent | 17d51d51855dc3ed663d0e31fa75d3ff5ab80702 (diff) | |
| download | ydb-2002caab389ddffd0e7db4d8c4db835fa4c99f83.tar.gz | |
Restoring authorship annotation for <uzhas@yandex-team.ru>. Commit 1 of 2.
479 files changed, 10578 insertions, 10578 deletions
diff --git a/contrib/libs/ya.make b/contrib/libs/ya.make index 9c4640fdcf..79334bc261 100644 --- a/contrib/libs/ya.make +++ b/contrib/libs/ya.make @@ -143,7 +143,7 @@ RECURSE( libde265 libdivide libdivsufsort2 - libeatmydata/dynamic + libeatmydata/dynamic libev libevent libexslt diff --git a/library/cpp/actors/core/event_pb_payload_ut.cpp b/library/cpp/actors/core/event_pb_payload_ut.cpp index eab007bc15..8f2f1ef369 100644 --- a/library/cpp/actors/core/event_pb_payload_ut.cpp +++ b/library/cpp/actors/core/event_pb_payload_ut.cpp @@ -9,21 +9,21 @@ using namespace NActors; enum { EvMessageWithPayload = EventSpaceBegin(TEvents::ES_PRIVATE), EvArenaMessage, - EvArenaMessageBig, - EvMessageWithPayloadPreSerialized + EvArenaMessageBig, + EvMessageWithPayloadPreSerialized }; struct TEvMessageWithPayload : TEventPB<TEvMessageWithPayload, TMessageWithPayload, EvMessageWithPayload> { - TEvMessageWithPayload() = default; - explicit TEvMessageWithPayload(const TMessageWithPayload& p) - : TEventPB<TEvMessageWithPayload, TMessageWithPayload, EvMessageWithPayload>(p) - {} + TEvMessageWithPayload() = default; + explicit TEvMessageWithPayload(const TMessageWithPayload& p) + : TEventPB<TEvMessageWithPayload, TMessageWithPayload, EvMessageWithPayload>(p) + {} }; -struct TEvMessageWithPayloadPreSerialized : TEventPreSerializedPB<TEvMessageWithPayloadPreSerialized, TMessageWithPayload, EvMessageWithPayloadPreSerialized> { -}; - - +struct TEvMessageWithPayloadPreSerialized : TEventPreSerializedPB<TEvMessageWithPayloadPreSerialized, TMessageWithPayload, EvMessageWithPayloadPreSerialized> { +}; + + TRope MakeStringRope(const TString& message) { return message ? TRope(message) : TRope(); } @@ -113,42 +113,42 @@ Y_UNIT_TEST_SUITE(TEventProtoWithPayload) { TestSerializeDeserialize<TEvArenaMessage, TEvArenaMessageWithoutArena>(200, 14010); TestSerializeDeserialize<TEvArenaMessageWithoutArena, TEvArenaMessage>(2000, 4010); } - - Y_UNIT_TEST(PreSerializedCompatibility) { - // ensure TEventPreSerializedPB and TEventPB are interchangable with no compatibility issues - TMessageWithPayload msg; - msg.SetMeta("hello, world!"); - msg.AddPayloadId(123); - msg.AddPayloadId(999); - msg.AddSomeData("abc"); - msg.AddSomeData("xyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"); - - TEvMessageWithPayloadPreSerialized e1; + + Y_UNIT_TEST(PreSerializedCompatibility) { + // ensure TEventPreSerializedPB and TEventPB are interchangable with no compatibility issues + TMessageWithPayload msg; + msg.SetMeta("hello, world!"); + msg.AddPayloadId(123); + msg.AddPayloadId(999); + msg.AddSomeData("abc"); + msg.AddSomeData("xyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"); + + TEvMessageWithPayloadPreSerialized e1; Y_PROTOBUF_SUPPRESS_NODISCARD msg.SerializeToString(&e1.PreSerializedData); - - auto serializer1 = MakeHolder<TAllocChunkSerializer>(); + + auto serializer1 = MakeHolder<TAllocChunkSerializer>(); e1.SerializeToArcadiaStream(serializer1.Get()); - auto buffers1 = serializer1->Release(e1.IsExtendedFormat()); - UNIT_ASSERT_VALUES_EQUAL(buffers1->GetSize(), e1.CalculateSerializedSize()); - TString ser1 = buffers1->GetString(); - - TEvMessageWithPayload e2(msg); - auto serializer2 = MakeHolder<TAllocChunkSerializer>(); + auto buffers1 = serializer1->Release(e1.IsExtendedFormat()); + UNIT_ASSERT_VALUES_EQUAL(buffers1->GetSize(), e1.CalculateSerializedSize()); + TString ser1 = buffers1->GetString(); + + TEvMessageWithPayload e2(msg); + auto serializer2 = MakeHolder<TAllocChunkSerializer>(); e2.SerializeToArcadiaStream(serializer2.Get()); - auto buffers2 = serializer2->Release(e2.IsExtendedFormat()); - UNIT_ASSERT_VALUES_EQUAL(buffers2->GetSize(), e2.CalculateSerializedSize()); - TString ser2 = buffers2->GetString(); - UNIT_ASSERT_VALUES_EQUAL(ser1, ser2); - - // deserialize - auto data = MakeIntrusive<TEventSerializedData>(ser1, false); - THolder<TEvMessageWithPayloadPreSerialized> parsedEvent(static_cast<TEvMessageWithPayloadPreSerialized*>(TEvMessageWithPayloadPreSerialized::Load(data))); - UNIT_ASSERT_VALUES_EQUAL(parsedEvent->PreSerializedData, ""); // this field is empty after deserialization - auto& record = parsedEvent->GetRecord(); - UNIT_ASSERT_VALUES_EQUAL(record.GetMeta(), msg.GetMeta()); - UNIT_ASSERT_VALUES_EQUAL(record.PayloadIdSize(), msg.PayloadIdSize()); - UNIT_ASSERT_VALUES_EQUAL(record.PayloadIdSize(), 2); - UNIT_ASSERT_VALUES_EQUAL(record.GetPayloadId(0), msg.GetPayloadId(0)); - UNIT_ASSERT_VALUES_EQUAL(record.GetPayloadId(1), msg.GetPayloadId(1)); - } + auto buffers2 = serializer2->Release(e2.IsExtendedFormat()); + UNIT_ASSERT_VALUES_EQUAL(buffers2->GetSize(), e2.CalculateSerializedSize()); + TString ser2 = buffers2->GetString(); + UNIT_ASSERT_VALUES_EQUAL(ser1, ser2); + + // deserialize + auto data = MakeIntrusive<TEventSerializedData>(ser1, false); + THolder<TEvMessageWithPayloadPreSerialized> parsedEvent(static_cast<TEvMessageWithPayloadPreSerialized*>(TEvMessageWithPayloadPreSerialized::Load(data))); + UNIT_ASSERT_VALUES_EQUAL(parsedEvent->PreSerializedData, ""); // this field is empty after deserialization + auto& record = parsedEvent->GetRecord(); + UNIT_ASSERT_VALUES_EQUAL(record.GetMeta(), msg.GetMeta()); + UNIT_ASSERT_VALUES_EQUAL(record.PayloadIdSize(), msg.PayloadIdSize()); + UNIT_ASSERT_VALUES_EQUAL(record.PayloadIdSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(record.GetPayloadId(0), msg.GetPayloadId(0)); + UNIT_ASSERT_VALUES_EQUAL(record.GetPayloadId(1), msg.GetPayloadId(1)); + } } diff --git a/library/cpp/grpc/client/grpc_client_low.h b/library/cpp/grpc/client/grpc_client_low.h index ab0a0627be..c831b1fa60 100644 --- a/library/cpp/grpc/client/grpc_client_low.h +++ b/library/cpp/grpc/client/grpc_client_low.h @@ -173,7 +173,7 @@ bool inline IsGRpcStatusGood(const TGrpcStatus& status) { return status.Ok(); } -// Response callback type - this callback will be called when request is finished +// Response callback type - this callback will be called when request is finished // (or after getting each chunk in case of streaming mode) template<typename TResponse> using TResponseCallback = std::function<void (TGrpcStatus&&, TResponse&&)>; @@ -236,8 +236,8 @@ public: using TPtr = TIntrusivePtr<TSimpleRequestProcessor>; using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); - explicit TSimpleRequestProcessor(TResponseCallback<TResponse>&& callback) - : Callback_(std::move(callback)) + explicit TSimpleRequestProcessor(TResponseCallback<TResponse>&& callback) + : Callback_(std::move(callback)) { } ~TSimpleRequestProcessor() { @@ -316,8 +316,8 @@ public: using TPtr = TIntrusivePtr<TAdvancedRequestProcessor>; using TAsyncRequest = TAsyncReaderPtr (TStub::*)(grpc::ClientContext*, const TRequest&, grpc::CompletionQueue*); - explicit TAdvancedRequestProcessor(TAdvancedResponseCallback<TResponse>&& callback) - : Callback_(std::move(callback)) + explicit TAdvancedRequestProcessor(TAdvancedResponseCallback<TResponse>&& callback) + : Callback_(std::move(callback)) { } ~TAdvancedRequestProcessor() { @@ -1244,7 +1244,7 @@ public: */ template<typename TRequest, typename TResponse> void DoRequest(const TRequest& request, - TResponseCallback<TResponse> callback, + TResponseCallback<TResponse> callback, typename TSimpleRequestProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, const TCallMeta& metas = { }, IQueueClientContextProvider* provider = nullptr) @@ -1259,7 +1259,7 @@ public: */ template<typename TRequest, typename TResponse> void DoAdvancedRequest(const TRequest& request, - TAdvancedResponseCallback<TResponse> callback, + TAdvancedResponseCallback<TResponse> callback, typename TAdvancedRequestProcessor<TStub, TRequest, TResponse>::TAsyncRequest asyncRequest, const TCallMeta& metas = { }, IQueueClientContextProvider* provider = nullptr) diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 128583bdd7..7cbd4494e8 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -136,12 +136,12 @@ public: }; static void* ListenSocketFunction(void* param) { - try { - ((TImpl*)param)->ListenSocket(); - } catch (...) { - - } + try { + ((TImpl*)param)->ListenSocket(); + } catch (...) { + } + return nullptr; } @@ -798,11 +798,11 @@ bool TRequestReplier::Reply(void* threadSpecificResource) { bool TryToBindAddresses(const THttpServerOptions& options, const std::function<void(TSocket)>* callbackOnBoundAddress) { THttpServerOptions::TBindAddresses addrs; - try { - options.BindAddresses(addrs); + try { + options.BindAddresses(addrs); } catch (const std::exception&) { return false; - } + } for (const auto& na : addrs) { for (TNetworkAddress::TIterator ai = na.Begin(); ai != na.End(); ++ai) { diff --git a/library/cpp/json/writer/json_value.h b/library/cpp/json/writer/json_value.h index 3f0f50bc4c..79d1a4cb3b 100644 --- a/library/cpp/json/writer/json_value.h +++ b/library/cpp/json/writer/json_value.h @@ -6,7 +6,7 @@ #include <util/generic/hash.h> #include <util/generic/vector.h> #include <util/generic/deque.h> -#include <util/generic/utility.h> +#include <util/generic/utility.h> #include <util/generic/yexception.h> namespace NJson { @@ -205,7 +205,7 @@ namespace NJson { TArray* Array; TValueUnion() noexcept { - Zero(*this); + Zero(*this); } ~TValueUnion() noexcept { } diff --git a/library/cpp/lwtrace/mon/mon_lwtrace.cpp b/library/cpp/lwtrace/mon/mon_lwtrace.cpp index a61ee9ce22..ecfa97e4c3 100644 --- a/library/cpp/lwtrace/mon/mon_lwtrace.cpp +++ b/library/cpp/lwtrace/mon/mon_lwtrace.cpp @@ -1798,7 +1798,7 @@ class TLogSources { private: TTraceCleaner& Cleaner; TInstant Now; - using TLogSourcePtr = std::unique_ptr<ILogSource>; + using TLogSourcePtr = std::unique_ptr<ILogSource>; TMap<TString, TLogSourcePtr> LogSources; public: explicit TLogSources(TTraceCleaner& cleaner, TInstant now = TInstant::Now()) @@ -1809,20 +1809,20 @@ public: void Push(const TString& sid, const TAtomicSharedPtr<NLWTrace::TLogPb>& log) { TLogSourcePtr ls(new TSnapshotLogSource(sid, log)); - LogSources.emplace(ls->GetId(), std::move(ls)); + LogSources.emplace(ls->GetId(), std::move(ls)); } void Push(const TString& id, const NLWTrace::TSession* trace) { TLogSourcePtr ls(new TTraceLogSource(id, trace, Cleaner)); - LogSources.emplace(ls->GetId(), std::move(ls)); + LogSources.emplace(ls->GetId(), std::move(ls)); } template <class TFunc> void ForEach(TFunc& func) { for (auto& kv : LogSources) { - func.Push(kv.second.get()); + func.Push(kv.second.get()); } } @@ -1830,7 +1830,7 @@ public: void ForEach(TFunc& func) const { for (const auto& kv : LogSources) { - func.Push(kv.second.get()); + func.Push(kv.second.get()); } } }; diff --git a/library/cpp/messagebus/message.h b/library/cpp/messagebus/message.h index 005ca10c65..0fc1a1b707 100644 --- a/library/cpp/messagebus/message.h +++ b/library/cpp/messagebus/message.h @@ -120,11 +120,11 @@ namespace NBus { struct TBusHeader { friend class TBusMessage; - TBusKey Id = 0; ///< unique message ID - ui32 Size = 0; ///< total size of the message - TBusInstant SendTime = 0; ///< time the message was sent - ui16 FlagsInternal = 0; ///< TRACE is one of the flags - ui16 Type = 0; ///< to be used by TBusProtocol + TBusKey Id = 0; ///< unique message ID + ui32 Size = 0; ///< total size of the message + TBusInstant SendTime = 0; ///< time the message was sent + ui16 FlagsInternal = 0; ///< TRACE is one of the flags + ui16 Type = 0; ///< to be used by TBusProtocol int GetVersionInternal() { return (FlagsInternal & MESSAGE_VERSION_INTERNAL) >> 4; diff --git a/library/cpp/messagebus/message_ptr_and_header.h b/library/cpp/messagebus/message_ptr_and_header.h index 9b4e2fd270..c4364f690f 100644 --- a/library/cpp/messagebus/message_ptr_and_header.h +++ b/library/cpp/messagebus/message_ptr_and_header.h @@ -18,8 +18,8 @@ namespace NBus { { } - explicit TBusMessagePtrAndHeader(TBusMessage* messagePtr) - : MessagePtr(messagePtr) + explicit TBusMessagePtrAndHeader(TBusMessage* messagePtr) + : MessagePtr(messagePtr) , Header(*MessagePtr->GetHeader()) , LocalFlags(MessagePtr->LocalFlags) { diff --git a/library/cpp/messagebus/network.cpp b/library/cpp/messagebus/network.cpp index 304bedae5a..1e2a89d605 100644 --- a/library/cpp/messagebus/network.cpp +++ b/library/cpp/messagebus/network.cpp @@ -14,15 +14,15 @@ namespace { Y_VERIFY(af == AF_INET || af == AF_INET6, "wrong af"); SOCKET fd = ::socket(af, SOCK_STREAM, 0); - if (fd == INVALID_SOCKET) { - ythrow TSystemError() << "failed to create a socket"; - } + if (fd == INVALID_SOCKET) { + ythrow TSystemError() << "failed to create a socket"; + } int one = 1; int r1 = SetSockOpt(fd, SOL_SOCKET, SO_REUSEADDR, one); - if (r1 < 0) { - ythrow TSystemError() << "failed to setsockopt SO_REUSEADDR"; - } + if (r1 < 0) { + ythrow TSystemError() << "failed to setsockopt SO_REUSEADDR"; + } #ifdef SO_REUSEPORT if (reusePort) { @@ -54,72 +54,72 @@ namespace { int r2 = ::bind(fd, sa, len); if (r2 < 0) { - ythrow TSystemError() << "failed to bind on port " << port; + ythrow TSystemError() << "failed to bind on port " << port; } int rsn = ::getsockname(fd, addr->MutableAddr(), addr->LenPtr()); - if (rsn < 0) { - ythrow TSystemError() << "failed to getsockname"; - } + if (rsn < 0) { + ythrow TSystemError() << "failed to getsockname"; + } int r3 = ::listen(fd, 50); - if (r3 < 0) { - ythrow TSystemError() << "listen failed"; - } + if (r3 < 0) { + ythrow TSystemError() << "listen failed"; + } TBindResult r; r.Socket.Reset(new TSocketHolder(fd)); r.Addr = TNetAddr(addr.Release()); return r; } - + TMaybe<TBindResult> TryBindOnPortProto(int port, int af, bool reusePort) { - try { + try { return {BindOnPortProto(port, af, reusePort)}; - } catch (const TSystemError&) { - return {}; - } - } - - std::pair<unsigned, TVector<TBindResult>> AggregateBindResults(TBindResult&& r1, TBindResult&& r2) { - Y_VERIFY(r1.Addr.GetPort() == r2.Addr.GetPort(), "internal"); - std::pair<unsigned, TVector<TBindResult>> r; - r.second.reserve(2); - - r.first = r1.Addr.GetPort(); - r.second.emplace_back(std::move(r1)); - r.second.emplace_back(std::move(r2)); - return r; - } + } catch (const TSystemError&) { + return {}; + } + } + + std::pair<unsigned, TVector<TBindResult>> AggregateBindResults(TBindResult&& r1, TBindResult&& r2) { + Y_VERIFY(r1.Addr.GetPort() == r2.Addr.GetPort(), "internal"); + std::pair<unsigned, TVector<TBindResult>> r; + r.second.reserve(2); + + r.first = r1.Addr.GetPort(); + r.second.emplace_back(std::move(r1)); + r.second.emplace_back(std::move(r2)); + return r; + } } std::pair<unsigned, TVector<TBindResult>> NBus::BindOnPort(int port, bool reusePort) { std::pair<unsigned, TVector<TBindResult>> r; - r.second.reserve(2); + r.second.reserve(2); if (port != 0) { return AggregateBindResults(BindOnPortProto(port, AF_INET, reusePort), BindOnPortProto(port, AF_INET6, reusePort)); - } - - // use nothrow versions in cycle - for (int i = 0; i < 1000; ++i) { + } + + // use nothrow versions in cycle + for (int i = 0; i < 1000; ++i) { TMaybe<TBindResult> in4 = TryBindOnPortProto(0, AF_INET, reusePort); - if (!in4) { - continue; + if (!in4) { + continue; } - + TMaybe<TBindResult> in6 = TryBindOnPortProto(in4->Addr.GetPort(), AF_INET6, reusePort); - if (!in6) { - continue; + if (!in6) { + continue; } - - return AggregateBindResults(std::move(*in4), std::move(*in6)); + + return AggregateBindResults(std::move(*in4), std::move(*in6)); } - + TBindResult in4 = BindOnPortProto(0, AF_INET, reusePort); TBindResult in6 = BindOnPortProto(in4.Addr.GetPort(), AF_INET6, reusePort); - return AggregateBindResults(std::move(in4), std::move(in6)); + return AggregateBindResults(std::move(in4), std::move(in6)); } void NBus::NPrivate::SetSockOptTcpCork(SOCKET s, bool value) { diff --git a/library/cpp/messagebus/network_ut.cpp b/library/cpp/messagebus/network_ut.cpp index f1798419db..f919286e8d 100644 --- a/library/cpp/messagebus/network_ut.cpp +++ b/library/cpp/messagebus/network_ut.cpp @@ -56,10 +56,10 @@ Y_UNIT_TEST_SUITE(Network) { UNIT_ASSERT_VALUES_EQUAL(r.at(0).Addr.GetPort(), r.at(1).Addr.GetPort()); } - + Y_UNIT_TEST(BindOnBusyPort) { auto r = BindOnPort(0, false); - + UNIT_ASSERT_EXCEPTION_CONTAINS(BindOnPort(r.first, false), TSystemError, "failed to bind on port " + ToString(r.first)); - } + } } diff --git a/library/cpp/monlib/service/monservice.cpp b/library/cpp/monlib/service/monservice.cpp index d1b9cda1d2..67b133365e 100644 --- a/library/cpp/monlib/service/monservice.cpp +++ b/library/cpp/monlib/service/monservice.cpp @@ -16,9 +16,9 @@ TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const T : TMonService2(HttpServerOptions(port, host, threads), title, std::move(auth)) { } - + TMonService2::TMonService2(const THttpServerOptions& options, const TString& title, THolder<IAuthProvider> auth) - : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2)) + : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2)) , Title(title) , IndexMonPage(new TIndexMonPage("", Title)) , AuthProvider_{std::move(auth)} diff --git a/library/python/ya.make b/library/python/ya.make index 2e1eb6e0e1..83c3a7fc74 100644 --- a/library/python/ya.make +++ b/library/python/ya.make @@ -21,8 +21,8 @@ RECURSE( bloom boost_test bstr - build_info - build_info/ut + build_info + build_info/ut capabilities celery_dashboard certifi diff --git a/util/generic/algorithm_ut.cpp b/util/generic/algorithm_ut.cpp index 8d732fcc0c..d89116638d 100644 --- a/util/generic/algorithm_ut.cpp +++ b/util/generic/algorithm_ut.cpp @@ -842,9 +842,9 @@ Y_UNIT_TEST_SUITE(TAlgorithm) { UNIT_ASSERT(Find(v, 1) == v.begin()); UNIT_ASSERT(Find(v, 100) == v.end() - 1); } - - Y_UNIT_TEST(AccumulateWithBinOp) { - std::vector<int> v = {1, 2, 777}; + + Y_UNIT_TEST(AccumulateWithBinOp) { + std::vector<int> v = {1, 2, 777}; UNIT_ASSERT_VALUES_EQUAL(TString("begin;1;2;777"), Accumulate(v, TString("begin"), [](auto&& a, auto& b) { return a + ";" + ToString(b); })); - } + } }; diff --git a/util/network/ip.h b/util/network/ip.h index dc7c2d24a0..f172500899 100644 --- a/util/network/ip.h +++ b/util/network/ip.h @@ -61,7 +61,7 @@ struct TIpAddress: public sockaddr_in { inline TIpAddress(const sockaddr_in& addr) noexcept : sockaddr_in(addr) - , tmp(0) + , tmp(0) { } @@ -113,7 +113,7 @@ struct TIpAddress: public sockaddr_in { return InetToHost(sin_port); } -private: - // required for "operator socklen_t*()" +private: + // required for "operator socklen_t*()" mutable socklen_t tmp; }; diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index c666f7468c..f6266621dd 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -180,19 +180,19 @@ struct TAppData { TProgramShouldContinue *kikimrShouldContinue); }; -inline TAppData* AppData(TActorSystem* actorSystem) { - Y_VERIFY_DEBUG(actorSystem); - TAppData * const x = actorSystem->AppData<TAppData>(); +inline TAppData* AppData(TActorSystem* actorSystem) { + Y_VERIFY_DEBUG(actorSystem); + TAppData * const x = actorSystem->AppData<TAppData>(); Y_VERIFY_DEBUG(x && x->Magic == TAppData::MagicTag); return x; } -inline TAppData* AppData() { - return AppData(TlsActivationContext->ExecutorThread.ActorSystem); -} - +inline TAppData* AppData() { + return AppData(TlsActivationContext->ExecutorThread.ActorSystem); +} + inline TAppData* AppData(const TActorContext &ctx) { - return AppData(ctx.ActorSystem()); + return AppData(ctx.ActorSystem()); } } // NKikimr diff --git a/ydb/core/base/tablet.cpp b/ydb/core/base/tablet.cpp index db3a3deb80..f2925b81d8 100644 --- a/ydb/core/base/tablet.cpp +++ b/ydb/core/base/tablet.cpp @@ -2,8 +2,8 @@ namespace NKikimr { -TIntrusivePtr<TTabletStorageInfo> TabletStorageInfoFromProto(const NKikimrTabletBase::TTabletStorageInfo &proto) { - auto info = MakeIntrusive<TTabletStorageInfo>(); +TIntrusivePtr<TTabletStorageInfo> TabletStorageInfoFromProto(const NKikimrTabletBase::TTabletStorageInfo &proto) { + auto info = MakeIntrusive<TTabletStorageInfo>(); info->TabletID = proto.GetTabletID(); diff --git a/ydb/core/base/tablet.h b/ydb/core/base/tablet.h index 602e39c600..90a3322203 100644 --- a/ydb/core/base/tablet.h +++ b/ydb/core/base/tablet.h @@ -20,7 +20,7 @@ namespace NKikimr { -TIntrusivePtr<TTabletStorageInfo> TabletStorageInfoFromProto(const NKikimrTabletBase::TTabletStorageInfo &proto); +TIntrusivePtr<TTabletStorageInfo> TabletStorageInfoFromProto(const NKikimrTabletBase::TTabletStorageInfo &proto); void TabletStorageInfoToProto(const TTabletStorageInfo &info, NKikimrTabletBase::TTabletStorageInfo *proto); inline ui64 MakeGenStepPair(ui32 gen, ui32 step) { diff --git a/ydb/core/base/tablet_resolver.h b/ydb/core/base/tablet_resolver.h index 24744cef47..1843549889 100644 --- a/ydb/core/base/tablet_resolver.h +++ b/ydb/core/base/tablet_resolver.h @@ -181,7 +181,7 @@ struct TEvTabletResolver { TEvForwardResult(NKikimrProto::EReplyStatus status, ui64 tabletId) : Status(status) , TabletID(tabletId) - , CacheEpoch(0) + , CacheEpoch(0) {} TEvForwardResult(ui64 tabletId, const TActorId &tabletActor, const TActorId &tablet, ui64 cacheEpoch) diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp index af2739699d..ef83cedb9a 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp @@ -30,8 +30,8 @@ namespace NKikimr { EstablishingSessionStartTime = TActivationContext::Now(); ConfigureQueryTimeoutEv = nullptr; ClearTimeoutCounters(); - EstablishingSessionsTimeoutEv = new TEvEstablishingSessionTimeout(); - Become(&TThis::StateEstablishingSessions, ProxyEstablishSessionsTimeout, EstablishingSessionsTimeoutEv); + EstablishingSessionsTimeoutEv = new TEvEstablishingSessionTimeout(); + Become(&TThis::StateEstablishingSessions, ProxyEstablishSessionsTimeout, EstablishingSessionsTimeoutEv); SwitchToWorkWhenGoodToGo(); } @@ -56,8 +56,8 @@ namespace NKikimr { ErrorDescription = "StateUnconfigured (DSPE9)."; EstablishingSessionsTimeoutEv = nullptr; ClearTimeoutCounters(); - ConfigureQueryTimeoutEv = new TEvConfigureQueryTimeout(); - Become(&TThis::StateUnconfigured, ProxyConfigurationTimeout, ConfigureQueryTimeoutEv); + ConfigureQueryTimeoutEv = new TEvConfigureQueryTimeout(); + Become(&TThis::StateUnconfigured, ProxyConfigurationTimeout, ConfigureQueryTimeoutEv); } void TBlobStorageGroupProxy::SetStateUnconfiguredTimeout() { diff --git a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_defrag.cpp b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_defrag.cpp index 0cb43beb2a..2ea2790931 100644 --- a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_defrag.cpp +++ b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_defrag.cpp @@ -232,7 +232,7 @@ namespace NKikimr { const TBlobHeader& header = *result.Data.DataPtr<const TBlobHeader>(0); if (!locator.DeleteInProgress && result.Data.IsReadable(0, totalSize)) { - Y_VERIFY(Crc32c(&header.IndexRecord, totalSize - sizeof(ui32)) == header.Checksum); + Y_VERIFY(Crc32c(&header.IndexRecord, totalSize - sizeof(ui32)) == header.Checksum); Y_VERIFY(header.IndexRecord == record); Y_VERIFY(header.ChunkSerNum == ChunkInProgressSerNum); TString data = header.ExtractInplacePayload(); diff --git a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_recovery_scan.cpp b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_recovery_scan.cpp index eb60105861..786380d77f 100644 --- a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_recovery_scan.cpp +++ b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_recovery_scan.cpp @@ -102,7 +102,7 @@ namespace NKikimr { } // validate checksum - const ui32 checksum = Crc32c(&header.ChunkSerNum, sizeof(TBlobIndexHeader) - sizeof(ui32) + dataSize); + const ui32 checksum = Crc32c(&header.ChunkSerNum, sizeof(TBlobIndexHeader) - sizeof(ui32) + dataSize); if (checksum != header.Checksum) { return false; } @@ -129,7 +129,7 @@ namespace NKikimr { break; } - const ui32 checksum = Crc32c(&header.IndexRecord, sizeof(TBlobHeader) - sizeof(ui32) + + const ui32 checksum = Crc32c(&header.IndexRecord, sizeof(TBlobHeader) - sizeof(ui32) + header.IndexRecord.PayloadSize); if (checksum != header.Checksum) { break; diff --git a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_write.cpp b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_write.cpp index ca6cedf5d8..3ec3f7c7c7 100644 --- a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_write.cpp +++ b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_write.cpp @@ -260,7 +260,7 @@ namespace NKikimr { // fill in header; calculate checksum for header (excluding CRC) + data header.ChunkSerNum = chunk.ChunkSerNum; - header.Checksum = Crc32cExtend(Crc32c(&header.IndexRecord, sizeof(TBlobHeader) - sizeof(ui32)), + header.Checksum = Crc32cExtend(Crc32c(&header.IndexRecord, sizeof(TBlobHeader) - sizeof(ui32)), item.Data.data(), item.Data.size()); static_assert(offsetof(TBlobHeader, Checksum) == 0 && sizeof(TBlobHeader::Checksum) == sizeof(ui32), "incorrect displacement of TBlobHeader::Checksum"); @@ -490,7 +490,7 @@ namespace NKikimr { memset(&indexHeader, 0, sizeof(indexHeader)); indexHeader.ChunkSerNum = chunk.ChunkSerNum; indexHeader.NumItems = finIt->second.Index.size(); - indexHeader.Checksum = Crc32cExtend(Crc32c(&indexHeader.ChunkSerNum, + indexHeader.Checksum = Crc32cExtend(Crc32c(&indexHeader.ChunkSerNum, sizeof(indexHeader) - sizeof(TBlobIndexHeader::Checksum)), finIt->second.Index.data(), indexHeader.NumItems * sizeof(TBlobIndexRecord)); static_assert(offsetof(TBlobIndexHeader, Checksum) == 0 && sizeof(TBlobIndexHeader::Checksum) == sizeof(ui32), diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index 47093a108d..9db991b905 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -346,7 +346,7 @@ void TNodeWarden::Handle(TEvPrivate::TEvUpdateNodeDrives::TPtr&) { return [this, drives = std::move(drives)] () { if (drives != WorkingLocalDrives) { SendToController(std::make_unique<TEvBlobStorage::TEvControllerUpdateNodeDrives>(LocalNodeId, drives)); - WorkingLocalDrives = std::move(drives); + WorkingLocalDrives = std::move(drives); } }; }); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index d00fc0f220..cc5b510f31 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -333,7 +333,7 @@ void TPDisk::Stop() { JointLogWrites.clear(); JointCommits.clear(); for (const auto& req : FastOperationsQueue) { - TRequestBase::AbortDelete(req.get(), ActorSystem); + TRequestBase::AbortDelete(req.get(), ActorSystem); } FastOperationsQueue.clear(); for (TRequestBase* req : PausedQueue) { @@ -2023,7 +2023,7 @@ void TPDisk::ProcessFastOperationsQueue() { for (auto& req : FastOperationsQueue) { switch (req->GetType()) { case ERequestType::RequestYardInit: { - std::unique_ptr<TYardInit> init{static_cast<TYardInit*>(req.release())}; + std::unique_ptr<TYardInit> init{static_cast<TYardInit*>(req.release())}; if (YardInitStart(*init)) { PendingYardInits.emplace(std::move(init)); } @@ -2756,7 +2756,7 @@ void TPDisk::PushRequestToForseti(TRequestBase *request) { } } } else { - FastOperationsQueue.push_back(std::unique_ptr<TRequestBase>(request)); + FastOperationsQueue.push_back(std::unique_ptr<TRequestBase>(request)); LOG_DEBUG(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# %" PRIu32 " ReqId# %" PRIu64 " PushRequestToForseti Push to FastOperationsQueue.size# %" PRIu64, (ui32)PDiskId, (ui64)request->ReqId.Id, (ui64)FastOperationsQueue.size()); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h index a9b910cb0c..efc309faaa 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h @@ -75,9 +75,9 @@ public: TVector<TLogWrite*> JointCommits; TVector<TChunkTrim*> JointChunkTrims; - TVector<std::unique_ptr<TRequestBase>> FastOperationsQueue; + TVector<std::unique_ptr<TRequestBase>> FastOperationsQueue; TDeque<TRequestBase*> PausedQueue; - std::set<std::unique_ptr<TYardInit>> PendingYardInits; + std::set<std::unique_ptr<TYardInit>> PendingYardInits; ui64 LastFlushId = 0; bool IsQueuePaused = false; bool IsQueueStep = false; diff --git a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h index 930fce08e6..4a946da217 100644 --- a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h +++ b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h @@ -344,14 +344,14 @@ namespace NKikimr { return location; } - TDiskPart Push(const TRope& buffer) { + TDiskPart Push(const TRope& buffer) { Y_VERIFY_DEBUG(!Finished); - TMaybe<TDiskPart> result = TBase::AppendAligned(buffer); + TMaybe<TDiskPart> result = TBase::AppendAligned(buffer); if (!result) { // there is no space to fit in current chunk -- restart base writer with new chunk TBase::FinishChunk(); - result = TBase::AppendAligned(buffer); + result = TBase::AppendAligned(buffer); Y_VERIFY(result); } diff --git a/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp b/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp index 4e9f1f03c3..9e5e70242f 100644 --- a/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp +++ b/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp @@ -67,10 +67,10 @@ namespace NKikimr { #define SETUP_VECTORS(data, gtype) \ ui32 totalParts = (gtype).TotalPartCount(); \ ui32 start = 2; \ - ui8* const dataPtr = (ui8*)(&(data));\ - TShiftedMainBitVec main(dataPtr, start, (start + totalParts)); \ + ui8* const dataPtr = (ui8*)(&(data));\ + TShiftedMainBitVec main(dataPtr, start, (start + totalParts)); \ start += totalParts; \ - TShiftedMainBitVec local(dataPtr, start, (start + totalParts)); \ + TShiftedMainBitVec local(dataPtr, start, (start + totalParts)); \ start += totalParts; \ ui32 handoffNum = (gtype).Handoff(); \ Y_VERIFY_DEBUG(handoffNum <= MaxHandoffNodes); \ @@ -79,7 +79,7 @@ namespace NKikimr { TShiftedHandoffBitVec handoff[MaxHandoffNodes]; \ { \ for (unsigned i = 0; i < handoffNum; i++) { \ - handoff[i] = TShiftedHandoffBitVec(dataPtr, start, start + handoffVectorBits); \ + handoff[i] = TShiftedHandoffBitVec(dataPtr, start, start + handoffVectorBits); \ start += handoffVectorBits; \ } \ } diff --git a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp index ec2e24ff44..4f5a40b9bd 100644 --- a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp @@ -18,7 +18,7 @@ namespace NKikimr { auto& rItems = context->Result->Items; for (auto& item : msg->Items) { rItems.emplace_back(std::move(item)); - AddBlobQuery(rItems.back().BlobId, rItems.back().Needed, context, &rItems.back()); + AddBlobQuery(rItems.back().BlobId, rItems.back().Needed, context, &rItems.back()); ++context->NumUnrespondedBlobs; } diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdsk.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdsk.cpp index 1a2fb95e1c..b55201e8a7 100644 --- a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdsk.cpp +++ b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdsk.cpp @@ -356,7 +356,7 @@ namespace NKikimr { TIndexedChunkPtr chunkPtr = ManyIdxChunks.front(); if (chunkPtr->GetLastLsn() <= confirmedCutLsn) { ManyIdxChunks.pop_front(); - chunkPtr->SetUpNotifier(notifier); + chunkPtr->SetUpNotifier(notifier); chunks.push_back(chunkPtr->GetChunkIdx()); nchunks++; } else @@ -378,7 +378,7 @@ namespace NKikimr { TIndexedChunkPtr chunkPtr = ManyIdxChunks.front(); ManyIdxChunks.pop_front(); newLogStartLsn = chunkPtr->GetLastLsn() + 1; - chunkPtr->SetUpNotifier(notifier); + chunkPtr->SetUpNotifier(notifier); chunks.push_back(chunkPtr->GetChunkIdx()); } diff --git a/ydb/core/client/server/grpc_server.h b/ydb/core/client/server/grpc_server.h index 2f2ae1da6b..b6700ea139 100644 --- a/ydb/core/client/server/grpc_server.h +++ b/ydb/core/client/server/grpc_server.h @@ -90,7 +90,7 @@ private: std::function<void()> InitCb_; // In flight request management. - NGrpc::TGlobalLimiter* Limiter_ = nullptr; + NGrpc::TGlobalLimiter* Limiter_ = nullptr; }; diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h index 3d40fc9001..8d157e19c7 100644 --- a/ydb/core/client/server/msgbus_server.h +++ b/ydb/core/client/server/msgbus_server.h @@ -139,7 +139,7 @@ class TMessageBusServer : public IMessageBusServer, public NBus::IBusServerHandl std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; protected: TProtocol Protocol; - TActorSystem *ActorSystem = nullptr; + TActorSystem *ActorSystem = nullptr; TIntrusivePtr<IMessageBusHttpServer> HttpServer; public: TMessageBusServer( diff --git a/ydb/core/client/server/msgbus_server_cms.cpp b/ydb/core/client/server/msgbus_server_cms.cpp index 53ada2a3fa..ed69b9e1ed 100644 --- a/ydb/core/client/server/msgbus_server_cms.cpp +++ b/ydb/core/client/server/msgbus_server_cms.cpp @@ -290,7 +290,7 @@ public: private: NKikimrClient::TCmsRequest Request; NKikimrClient::TCmsResponse Response; - ui32 StateStorageGroup = 0; + ui32 StateStorageGroup = 0; TActorId CmsPipe; }; diff --git a/ydb/core/client/server/msgbus_server_console.cpp b/ydb/core/client/server/msgbus_server_console.cpp index 66bc0b02eb..e7885e270a 100644 --- a/ydb/core/client/server/msgbus_server_console.cpp +++ b/ydb/core/client/server/msgbus_server_console.cpp @@ -346,7 +346,7 @@ public: private: NKikimrClient::TConsoleRequest Request; NKikimrClient::TConsoleResponse Response; - ui32 StateStorageGroup = 0; + ui32 StateStorageGroup = 0; TActorId ConsolePipe; }; diff --git a/ydb/core/client/server/msgbus_tabletreq.h b/ydb/core/client/server/msgbus_tabletreq.h index f173ae9097..41b481dc75 100644 --- a/ydb/core/client/server/msgbus_tabletreq.h +++ b/ydb/core/client/server/msgbus_tabletreq.h @@ -28,7 +28,7 @@ protected: private: TActorId PipeClient; - ui64 TabletId = 0; + ui64 TabletId = 0; void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { TEvTabletPipe::TEvClientConnected *msg = ev->Get(); diff --git a/ydb/core/cms/cluster_info.h b/ydb/core/cms/cluster_info.h index af5fe24af8..3e650da171 100644 --- a/ydb/core/cms/cluster_info.h +++ b/ydb/core/cms/cluster_info.h @@ -92,7 +92,7 @@ struct TRequestInfo { TString RequestId; TString Owner; - ui64 Order = 0; + ui64 Order = 0; NKikimrCms::TPermissionRequest Request; }; @@ -403,7 +403,7 @@ public: , VDiskId(id) , PDiskId(pdiskId) , NodeId(nodeId) - , SlotId(0) + , SlotId(0) { } diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 450d4135c6..80d054101c 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -796,10 +796,10 @@ public: TClientCommandSchemaAccess() : TClientCommandTree("access", { "acl" }, "Access operations") { - AddCommand(std::make_unique<TClientCommandSchemaAccessAdd>()); - AddCommand(std::make_unique<TClientCommandSchemaAccessRemove>()); - //AddCommand(std::make_unique<TClientCommandSchemaAccessGrant>()); - //AddCommand(std::make_unique<TClientCommandSchemaAccessRevoke>()); + AddCommand(std::make_unique<TClientCommandSchemaAccessAdd>()); + AddCommand(std::make_unique<TClientCommandSchemaAccessRemove>()); + //AddCommand(std::make_unique<TClientCommandSchemaAccessGrant>()); + //AddCommand(std::make_unique<TClientCommandSchemaAccessRevoke>()); } }; @@ -1014,8 +1014,8 @@ public: TClientCommandSchemaTable() : TClientCommandTree("table", {}, "Table operations") { - AddCommand(std::make_unique<TClientCommandSchemaTableOptions>()); - AddCommand(std::make_unique<TClientCommandSchemaTableCopy>()); + AddCommand(std::make_unique<TClientCommandSchemaTableOptions>()); + AddCommand(std::make_unique<TClientCommandSchemaTableCopy>()); } }; @@ -1250,31 +1250,31 @@ public: TClientCommandSchemaUserAttribute() : TClientCommandTree("user-attribute", { "ua" }, "User attribute operations") { - AddCommand(std::make_unique<TClientCommandSchemaUserAttributeGet>()); - AddCommand(std::make_unique<TClientCommandSchemaUserAttributeSet>()); - AddCommand(std::make_unique<TClientCommandSchemaUserAttributeDel>()); + AddCommand(std::make_unique<TClientCommandSchemaUserAttributeGet>()); + AddCommand(std::make_unique<TClientCommandSchemaUserAttributeSet>()); + AddCommand(std::make_unique<TClientCommandSchemaUserAttributeDel>()); } }; TClientCommandSchemaLite::TClientCommandSchemaLite() : TClientCommandTree("schema", {}, "Schema operations") { - AddCommand(std::make_unique<TClientCommandSchemaExec>()); - AddCommand(std::make_unique<TClientCommandSchemaDescribe>()); - AddCommand(std::make_unique<TClientCommandSchemaLs>()); - AddCommand(std::make_unique<TClientCommandSchemaMkdir>()); - AddCommand(std::make_unique<TClientCommandSchemaDrop>()); - AddCommand(std::make_unique<TClientCommandSchemaChown>()); - AddCommand(std::make_unique<TClientCommandSchemaAccess>()); - AddCommand(std::make_unique<TClientCommandSchemaTable>()); - AddCommand(std::make_unique<TClientCommandSchemaUserAttribute>()); + AddCommand(std::make_unique<TClientCommandSchemaExec>()); + AddCommand(std::make_unique<TClientCommandSchemaDescribe>()); + AddCommand(std::make_unique<TClientCommandSchemaLs>()); + AddCommand(std::make_unique<TClientCommandSchemaMkdir>()); + AddCommand(std::make_unique<TClientCommandSchemaDrop>()); + AddCommand(std::make_unique<TClientCommandSchemaChown>()); + AddCommand(std::make_unique<TClientCommandSchemaAccess>()); + AddCommand(std::make_unique<TClientCommandSchemaTable>()); + AddCommand(std::make_unique<TClientCommandSchemaUserAttribute>()); } class TClientCommandSchema : public TClientCommandSchemaLite { public: TClientCommandSchema() { - AddCommand(std::make_unique<TClientCommandSchemaInit>()); + AddCommand(std::make_unique<TClientCommandSchemaInit>()); } }; @@ -1451,9 +1451,9 @@ public: TClientCommandDb::TClientCommandDb() : TClientCommandTree("db", {}, "KiKiMR DB operations") { - AddCommand(std::make_unique<TClientCommandSchema>()); - AddCommand(std::make_unique<TClientCommandDbExec>()); - AddCommand(std::make_unique<TClientCommandS3Listing>()); + AddCommand(std::make_unique<TClientCommandSchema>()); + AddCommand(std::make_unique<TClientCommandDbExec>()); + AddCommand(std::make_unique<TClientCommandS3Listing>()); } } diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp index 729ae432c3..25a46698af 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp @@ -54,7 +54,7 @@ struct TClientCommandDiscoveryListEndpoints TClientCommandDiscovery::TClientCommandDiscovery() : TClientCommandTree("discovery", {}, "Endpoint discovery") { - AddCommand(std::make_unique<TClientCommandDiscoveryListEndpoints>()); + AddCommand(std::make_unique<TClientCommandDiscoveryListEndpoints>()); } // New YDB behavior: diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp index 197eccfcfd..dc72793b51 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp @@ -78,9 +78,9 @@ public: TClientCommandRootLite() : TClientCommandRootKikimrBase("ydb") { - AddCommand(std::make_unique<TClientCommandSchemaLite>()); - AddCommand(std::make_unique<TClientCommandWhoAmI>()); - AddCommand(std::make_unique<TClientCommandDiscoveryLite>()); + AddCommand(std::make_unique<TClientCommandSchemaLite>()); + AddCommand(std::make_unique<TClientCommandWhoAmI>()); + AddCommand(std::make_unique<TClientCommandDiscoveryLite>()); } void Config(TConfig& config) override { diff --git a/ydb/core/driver_lib/cli_base/cli_grpc.h b/ydb/core/driver_lib/cli_base/cli_grpc.h index 61d09c9f70..9feda09790 100644 --- a/ydb/core/driver_lib/cli_base/cli_grpc.h +++ b/ydb/core/driver_lib/cli_base/cli_grpc.h @@ -45,7 +45,7 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, { NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<TService>(clientConfig); - connection->DoRequest(request, responseCb, function, meta); + connection->DoRequest(request, responseCb, function, meta); } NGrpc::TResponseCallback<Ydb::Operations::GetOperationResponse> operationCb = @@ -64,7 +64,7 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Operation::V1::OperationService>(clientConfig); Ydb::Operations::GetOperationRequest request; request.set_id(response.id()); - connection->DoRequest(request, operationCb, &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, meta); + connection->DoRequest(request, operationCb, &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, meta); } return res; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_admin.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_admin.cpp index 49935a5e98..f02bd2b762 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_admin.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_admin.cpp @@ -7,12 +7,12 @@ namespace NDriverClient { TClientCommandAdmin::TClientCommandAdmin() : TClientCommandTree("admin", {}, "KiKiMR management and administration") { - AddCommand(std::make_unique<TClientCommandTablet>()); - AddCommand(std::make_unique<TClientCommandNode>()); - AddCommand(std::make_unique<TClientCommandDebug>()); - AddCommand(std::make_unique<TClientCommandBlobStorage>()); - AddCommand(std::make_unique<TClientCommandTenant>()); - AddCommand(std::make_unique<TClientCommandConsole>()); + AddCommand(std::make_unique<TClientCommandTablet>()); + AddCommand(std::make_unique<TClientCommandNode>()); + AddCommand(std::make_unique<TClientCommandDebug>()); + AddCommand(std::make_unique<TClientCommandBlobStorage>()); + AddCommand(std::make_unique<TClientCommandTenant>()); + AddCommand(std::make_unique<TClientCommandConsole>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_bs.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_bs.cpp index ef731708bf..3e9c933b16 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_bs.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_bs.cpp @@ -4,11 +4,11 @@ namespace NKikimr { namespace NDriverClient { -std::unique_ptr<TClientCommand> CreateClientCommandDisk(); -std::unique_ptr<TClientCommand> CreateClientCommandGroup(); -std::unique_ptr<TClientCommand> CreateClientCommandGenConfig(); -std::unique_ptr<TClientCommand> CreateClientCommandGet(); -std::unique_ptr<TClientCommand> CreateClientCommandBsConfig(); +std::unique_ptr<TClientCommand> CreateClientCommandDisk(); +std::unique_ptr<TClientCommand> CreateClientCommandGroup(); +std::unique_ptr<TClientCommand> CreateClientCommandGenConfig(); +std::unique_ptr<TClientCommand> CreateClientCommandGet(); +std::unique_ptr<TClientCommand> CreateClientCommandBsConfig(); TClientCommandBlobStorage::TClientCommandBlobStorage() : TClientCommandTree("blobstorage", { "bs" }, "Blob Storage management") diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp index 8d8e54af65..def9e6ec01 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp @@ -500,9 +500,9 @@ public: NKikimrCms::TAction::EType type, TClientCommandWithAction::EFreeField freeArgField, bool hasDuration) - : TClientCommandTree(name, {}, description) + : TClientCommandTree(name, {}, description) { - AddCommand(std::make_unique<TClientCommandMakeRequest>(requestDescription, type, freeArgField, hasDuration)); + AddCommand(std::make_unique<TClientCommandMakeRequest>(requestDescription, type, freeArgField, hasDuration)); } }; @@ -511,11 +511,11 @@ public: TClientCommandAddRequest() : TClientCommandTree("add", {}, "") { - AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to add host", + AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to add host", NKikimrCms::TAction::ADD_HOST, TClientCommandWithAction::FF_HOST, false)); - AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to add device", + AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to add device", NKikimrCms::TAction::ADD_DEVICES, TClientCommandWithAction::FF_DEVICE, false)); @@ -527,15 +527,15 @@ public: TClientCommandRestartRequest() : TClientCommandTree("restart", {}, "") { - AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart service", + AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart service", NKikimrCms::TAction::RESTART_SERVICES, TClientCommandWithAction::FF_SERVICE, true)); - AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart host", + AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart host", NKikimrCms::TAction::SHUTDOWN_HOST, TClientCommandWithAction::FF_HOST, true)); - AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart tenant's hosts", + AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart tenant's hosts", NKikimrCms::TAction::SHUTDOWN_HOST, TClientCommandWithAction::FF_TENANT, true)); @@ -548,32 +548,32 @@ public: TClientCommandRequest() : TClientCommandTree("request", {}, "Make and manage permission requests") { - //AddCommand(std::make_unique<TClientCommandSingleRequest>("start", "", "Ask for permission to start service", + //AddCommand(std::make_unique<TClientCommandSingleRequest>("start", "", "Ask for permission to start service", // NKikimrCms::TAction::START_SERVICES, // TClientCommandWithAction::FF_SERVICE, // false)); - //AddCommand(std::make_unique<TClientCommandSingleRequest>("stop", "", "Ask for permission to stop service", + //AddCommand(std::make_unique<TClientCommandSingleRequest>("stop", "", "Ask for permission to stop service", // NKikimrCms::TAction::STOP_SERVICES, // TClientCommandWithAction::FF_SERVICE, // false)); - //AddCommand(std::make_unique<TClientCommandSingleRequest>("decommission", "", "Ask for permission to decommission host", + //AddCommand(std::make_unique<TClientCommandSingleRequest>("decommission", "", "Ask for permission to decommission host", // NKikimrCms::TAction::DECOMMISSION_HOST, // TClientCommandWithAction::FF_HOST, // false)); - AddCommand(std::make_unique<TClientCommandSingleRequest>("replace", "", "Ask for permission to replace device", + AddCommand(std::make_unique<TClientCommandSingleRequest>("replace", "", "Ask for permission to replace device", NKikimrCms::TAction::REPLACE_DEVICES, TClientCommandWithAction::FF_DEVICE, true)); - //AddCommand(std::make_unique<TClientCommandSingleRequest>("remove", "", "Ask for permission to remove device", + //AddCommand(std::make_unique<TClientCommandSingleRequest>("remove", "", "Ask for permission to remove device", // NKikimrCms::TAction::REMOVE_DEVICES, // TClientCommandWithAction::FF_DEVICE, // false)); - //AddCommand(std::make_unique<TClientCommandAddRequest>()); - AddCommand(std::make_unique<TClientCommandRestartRequest>()); - AddCommand(std::make_unique<TClientCommandGetRequest>()); - AddCommand(std::make_unique<TClientCommandListRequest>()); - AddCommand(std::make_unique<TClientCommandRejectRequest>()); - AddCommand(std::make_unique<TClientCommandCheckRequest>()); + //AddCommand(std::make_unique<TClientCommandAddRequest>()); + AddCommand(std::make_unique<TClientCommandRestartRequest>()); + AddCommand(std::make_unique<TClientCommandGetRequest>()); + AddCommand(std::make_unique<TClientCommandListRequest>()); + AddCommand(std::make_unique<TClientCommandRejectRequest>()); + AddCommand(std::make_unique<TClientCommandCheckRequest>()); } }; @@ -630,9 +630,9 @@ public: NKikimrCms::TAction::EType type, TClientCommandWithAction::EFreeField freeArgField, bool hasDuration) - : TClientCommandTree(name, {}, description) + : TClientCommandTree(name, {}, description) { - AddCommand(std::make_unique<TClientCommandSendNotification>(requestDescription, type, freeArgField, hasDuration)); + AddCommand(std::make_unique<TClientCommandSendNotification>(requestDescription, type, freeArgField, hasDuration)); } }; @@ -641,11 +641,11 @@ public: TClientCommandNotifyAdd() : TClientCommandTree("add", {}, "") { - AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about new host", + AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about new host", NKikimrCms::TAction::ADD_HOST, TClientCommandWithAction::FF_HOST, false)); - AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about new device", + AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about new device", NKikimrCms::TAction::ADD_DEVICES, TClientCommandWithAction::FF_DEVICE, false)); @@ -657,11 +657,11 @@ public: TClientCommandNotifyRestart() : TClientCommandTree("restart", {}, "") { - AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about service restart", + AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about service restart", NKikimrCms::TAction::RESTART_SERVICES, TClientCommandWithAction::FF_SERVICE, true)); - AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about host restart", + AddCommand(std::make_unique<TClientCommandSendNotification>("Notify about host restart", NKikimrCms::TAction::SHUTDOWN_HOST, TClientCommandWithAction::FF_HOST, true)); @@ -755,31 +755,31 @@ public: TClientCommandNotify() : TClientCommandTree("notify", {}, "Send and manage notifications") { - //AddCommand(std::make_unique<TClientCommandSingleNotify>("start", "", "Notify about service start", + //AddCommand(std::make_unique<TClientCommandSingleNotify>("start", "", "Notify about service start", // NKikimrCms::TAction::START_SERVICES, // TClientCommandWithAction::FF_SERVICE, // false)); - //AddCommand(std::make_unique<TClientCommandSingleNotify>("stop", "", "Notify about service stop", + //AddCommand(std::make_unique<TClientCommandSingleNotify>("stop", "", "Notify about service stop", // NKikimrCms::TAction::STOP_SERVICES, // TClientCommandWithAction::FF_SERVICE, // false)); - //AddCommand(std::make_unique<TClientCommandSingleNotify>("decommission", "", "Notify about host decomission", + //AddCommand(std::make_unique<TClientCommandSingleNotify>("decommission", "", "Notify about host decomission", // NKikimrCms::TAction::DECOMMISSION_HOST, // TClientCommandWithAction::FF_HOST, // false)); - AddCommand(std::make_unique<TClientCommandSingleNotify>("replace", "", "Notify about device replacement", + AddCommand(std::make_unique<TClientCommandSingleNotify>("replace", "", "Notify about device replacement", NKikimrCms::TAction::REPLACE_DEVICES, TClientCommandWithAction::FF_DEVICE, true)); - //AddCommand(std::make_unique<TClientCommandSingleNotify>("remove", "", "Notify about device removal", + //AddCommand(std::make_unique<TClientCommandSingleNotify>("remove", "", "Notify about device removal", // NKikimrCms::TAction::REMOVE_DEVICES, // TClientCommandWithAction::FF_DEVICE, // false)); - //AddCommand(std::make_unique<TClientCommandNotifyAdd>()); - AddCommand(std::make_unique<TClientCommandNotifyRestart>()); - AddCommand(std::make_unique<TClientCommandGetNotification>()); - AddCommand(std::make_unique<TClientCommandListNotification>()); - AddCommand(std::make_unique<TClientCommandRejectNotification>()); + //AddCommand(std::make_unique<TClientCommandNotifyAdd>()); + AddCommand(std::make_unique<TClientCommandNotifyRestart>()); + AddCommand(std::make_unique<TClientCommandGetNotification>()); + AddCommand(std::make_unique<TClientCommandListNotification>()); + AddCommand(std::make_unique<TClientCommandRejectNotification>()); } }; @@ -912,11 +912,11 @@ public: TClientCommandPermission() : TClientCommandTree("permission", {}, "Manage issued permissions") { - AddCommand(std::make_unique<TClientCommandGetPermission>()); - AddCommand(std::make_unique<TClientCommandListPermission>()); - AddCommand(std::make_unique<TClientCommandDonePermission>()); - //AddCommand(std::make_unique<TClientCommandExtendPermission>()); - AddCommand(std::make_unique<TClientCommandRejectPermission>()); + AddCommand(std::make_unique<TClientCommandGetPermission>()); + AddCommand(std::make_unique<TClientCommandListPermission>()); + AddCommand(std::make_unique<TClientCommandDonePermission>()); + //AddCommand(std::make_unique<TClientCommandExtendPermission>()); + AddCommand(std::make_unique<TClientCommandRejectPermission>()); } }; @@ -965,8 +965,8 @@ public: TClientCommandManageConfig() : TClientCommandTree("config", {}, "Manage CMS config") { - AddCommand(std::make_unique<TClientCommandGetConfig>()); - AddCommand(std::make_unique<TClientCommandSetConfig>()); + AddCommand(std::make_unique<TClientCommandGetConfig>()); + AddCommand(std::make_unique<TClientCommandSetConfig>()); } }; @@ -1074,8 +1074,8 @@ public: TClientCommandMarker() : TClientCommandTree("marker", {}, "Manage markers") { - AddCommand(std::make_unique<TClientCommandSetMarker>()); - AddCommand(std::make_unique<TClientCommandResetMarker>()); + AddCommand(std::make_unique<TClientCommandSetMarker>()); + AddCommand(std::make_unique<TClientCommandResetMarker>()); } }; @@ -1158,20 +1158,20 @@ public: TClientCommandLog() : TClientCommandTree("log", {}, "Inspect log records") { - AddCommand(std::make_unique<TClientCommandLogTail>()); + AddCommand(std::make_unique<TClientCommandLogTail>()); } }; TClientCommandCms::TClientCommandCms() : TClientCommandTree("cms", {}, "CMS requests") { - AddCommand(std::make_unique<TClientCommandState>()); - AddCommand(std::make_unique<TClientCommandRequest>()); - AddCommand(std::make_unique<TClientCommandNotify>()); - AddCommand(std::make_unique<TClientCommandPermission>()); - AddCommand(std::make_unique<TClientCommandManageConfig>()); - AddCommand(std::make_unique<TClientCommandMarker>()); - AddCommand(std::make_unique<TClientCommandLog>()); + AddCommand(std::make_unique<TClientCommandState>()); + AddCommand(std::make_unique<TClientCommandRequest>()); + AddCommand(std::make_unique<TClientCommandNotify>()); + AddCommand(std::make_unique<TClientCommandPermission>()); + AddCommand(std::make_unique<TClientCommandManageConfig>()); + AddCommand(std::make_unique<TClientCommandMarker>()); + AddCommand(std::make_unique<TClientCommandLog>()); } } // namespace NDriverClient diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_config.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_config.cpp index aae54255e9..6e93c8aac9 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_config.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_config.cpp @@ -242,7 +242,7 @@ public: TPropose() : TClientCommandTree("propose", {}, "Configuration proposition for migration and initial configuring") { - AddCommand(std::make_unique<TProposeStoragePools>()); + AddCommand(std::make_unique<TProposeStoragePools>()); } }; @@ -251,14 +251,14 @@ public: TClientCommandBsConfig() : TClientCommandTree("config", {}, "Configuration management") { - AddCommand(std::make_unique<TPropose>()); - AddCommand(std::make_unique<TInvoke>()); + AddCommand(std::make_unique<TPropose>()); + AddCommand(std::make_unique<TInvoke>()); AddCommand(std::make_unique<TInit>()); } }; -std::unique_ptr<TClientCommand> CreateClientCommandBsConfig() { - return std::make_unique<TClientCommandBsConfig>(); +std::unique_ptr<TClientCommand> CreateClientCommandBsConfig() { + return std::make_unique<TClientCommandBsConfig>(); } } // NDriverClient diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp index 0497d4ea8f..bf90f0bfbe 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp @@ -316,8 +316,8 @@ public: TClientCommandConsoleConfigs() : TClientCommandTree("configs", {}, "") { - AddCommand(std::make_unique<TClientCommandConsoleConfigsLoad>()); - AddCommand(std::make_unique<TClientCommandConsoleConfigsUpdate>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigsLoad>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigsUpdate>()); } }; @@ -417,8 +417,8 @@ public: TClientCommandConsoleConfig() : TClientCommandTree("config", {}, "") { - AddCommand(std::make_unique<TClientCommandConsoleConfigGet>()); - AddCommand(std::make_unique<TClientCommandConsoleConfigSet>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigGet>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigSet>()); } }; @@ -522,19 +522,19 @@ public: TClientCommandConsoleValidator() : TClientCommandTree("validator", {}, "") { - AddCommand(std::make_unique<TClientCommandConsoleValidatorDisable>()); - AddCommand(std::make_unique<TClientCommandConsoleValidatorEnable>()); - AddCommand(std::make_unique<TClientCommandConsoleValidatorList>()); + AddCommand(std::make_unique<TClientCommandConsoleValidatorDisable>()); + AddCommand(std::make_unique<TClientCommandConsoleValidatorEnable>()); + AddCommand(std::make_unique<TClientCommandConsoleValidatorList>()); } }; TClientCommandConsole::TClientCommandConsole() : TClientCommandTree("console", {}, "Console commands") { - AddCommand(std::make_unique<TClientCommandConsoleConfig>()); - AddCommand(std::make_unique<TClientCommandConsoleConfigs>()); - AddCommand(std::make_unique<TClientCommandConsoleExecute>()); - AddCommand(std::make_unique<TClientCommandConsoleValidator>()); + AddCommand(std::make_unique<TClientCommandConsoleConfig>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigs>()); + AddCommand(std::make_unique<TClientCommandConsoleExecute>()); + AddCommand(std::make_unique<TClientCommandConsoleValidator>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_debug.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_debug.cpp index 9cd9e582fa..d81fa484ab 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_debug.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_debug.cpp @@ -281,18 +281,18 @@ public: TInterconnect() : TClientCommandTree("interconnect", {"ic"}, "Interconnect debugging facilities") { - AddCommand(std::make_unique<TInterconnectLoad>()); - AddCommand(std::make_unique<TInterconnectClosePeerSocket>()); - AddCommand(std::make_unique<TInterconnectCloseInputSession>()); - AddCommand(std::make_unique<TInterconnectPoisonSession>()); - AddCommand(std::make_unique<TInterconnectSlowpoke>()); + AddCommand(std::make_unique<TInterconnectLoad>()); + AddCommand(std::make_unique<TInterconnectClosePeerSocket>()); + AddCommand(std::make_unique<TInterconnectCloseInputSession>()); + AddCommand(std::make_unique<TInterconnectPoisonSession>()); + AddCommand(std::make_unique<TInterconnectSlowpoke>()); } }; TClientCommandDebug::TClientCommandDebug() : TClientCommandTree("debug") { - AddCommand(std::make_unique<TInterconnect>()); + AddCommand(std::make_unique<TInterconnect>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp index a9be370d37..9d221f69fd 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp @@ -209,14 +209,14 @@ public: TClientCommandDisk() : TClientCommandTree("disk", {}, "Disk management") { - AddCommand(std::make_unique<TClientCommandDiskInfo>()); - AddCommand(std::make_unique<TClientCommandDiskFormat>()); - AddCommand(std::make_unique<TClientCommandDiskObliterate>()); + AddCommand(std::make_unique<TClientCommandDiskInfo>()); + AddCommand(std::make_unique<TClientCommandDiskFormat>()); + AddCommand(std::make_unique<TClientCommandDiskObliterate>()); } }; -std::unique_ptr<TClientCommand> CreateClientCommandDisk() { - return std::make_unique<TClientCommandDisk>(); +std::unique_ptr<TClientCommand> CreateClientCommandDisk() { + return std::make_unique<TClientCommandDisk>(); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp index af2801de36..0e5cce574b 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp @@ -346,12 +346,12 @@ public: TClientCommandGenConfig() : TClientCommandTree("genconfig", {}, "Config generation") { - AddCommand(std::make_unique<TClientCommandGenConfigStatic>()); + AddCommand(std::make_unique<TClientCommandGenConfigStatic>()); } }; -std::unique_ptr<TClientCommand> CreateClientCommandGenConfig() { - return std::make_unique<TClientCommandGenConfig>(); +std::unique_ptr<TClientCommand> CreateClientCommandGenConfig() { + return std::make_unique<TClientCommandGenConfig>(); } } // NDriverClient diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_get.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_get.cpp index f659360087..fe1d8fc0ec 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_get.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_get.cpp @@ -82,12 +82,12 @@ public: TClientCommandGet() : TClientCommandTree("get", {}, "Various storage low-level queries") { - AddCommand(std::make_unique<TClientCommandGetBlob>()); + AddCommand(std::make_unique<TClientCommandGetBlob>()); } }; -std::unique_ptr<TClientCommand> CreateClientCommandGet() { - return std::make_unique<TClientCommandGet>(); +std::unique_ptr<TClientCommand> CreateClientCommandGet() { + return std::make_unique<TClientCommandGet>(); } } // NDriverClient diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_group.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_group.cpp index 0b4804c1c1..0c20ce1c94 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_group.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_group.cpp @@ -70,7 +70,7 @@ public: TClientCommandGroupReconfigure() : TClientCommandTree("reconfigure", {}, "Change the configuration of a group") { - AddCommand(std::make_unique<TClientCommandGroupReconfigureWipe>()); + AddCommand(std::make_unique<TClientCommandGroupReconfigureWipe>()); } }; @@ -79,12 +79,12 @@ public: TClientCommandGroup() : TClientCommandTree("group", {}, "Group management") { - AddCommand(std::make_unique<TClientCommandGroupReconfigure>()); + AddCommand(std::make_unique<TClientCommandGroupReconfigure>()); } }; -std::unique_ptr<TClientCommand> CreateClientCommandGroup() { - return std::make_unique<TClientCommandGroup>(); +std::unique_ptr<TClientCommand> CreateClientCommandGroup() { + return std::make_unique<TClientCommandGroup>(); } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp index 11e89d1960..6f60b056d9 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp @@ -37,8 +37,8 @@ public: TClientCommandNodeN() : TClientCommandTree("*", {}, "<node id or hostname>") { - AddCommand(std::make_unique<TClientCommandDrain>()); - AddCommand(std::make_unique<TClientCommandFill>()); + AddCommand(std::make_unique<TClientCommandDrain>()); + AddCommand(std::make_unique<TClientCommandFill>()); } virtual void Config(TConfig& config) override { @@ -94,7 +94,7 @@ public: TClientCommandNode::TClientCommandNode() : TClientCommandTree("node", {}, "Nodes infrastructure administration") { - AddCommand(std::make_unique<TClientCommandNodeN>()); + AddCommand(std::make_unique<TClientCommandNodeN>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp index ec049fab11..47340310d2 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp @@ -18,11 +18,11 @@ public: TClientCommandRoot(std::shared_ptr<TModuleFactories> factories) : TClientCommandRootKikimrBase("kikimr") { - AddCommand(std::make_unique<TClientCommandAdmin>()); - AddCommand(std::make_unique<TClientCommandDb>()); - AddCommand(std::make_unique<TClientCommandCms>()); - AddCommand(std::make_unique<TClientCommandWhoAmI>()); - AddCommand(std::make_unique<TClientCommandDiscovery>()); + AddCommand(std::make_unique<TClientCommandAdmin>()); + AddCommand(std::make_unique<TClientCommandDb>()); + AddCommand(std::make_unique<TClientCommandCms>()); + AddCommand(std::make_unique<TClientCommandWhoAmI>()); + AddCommand(std::make_unique<TClientCommandDiscovery>()); AddClientCommandServer(*this, std::move(factories)); } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 6766dd3171..88ed304256 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -1153,8 +1153,8 @@ private: }; void AddClientCommandServer(TClientCommandTree& parent, std::shared_ptr<TModuleFactories> factories) { - parent.AddCommand(std::make_unique<TClientCommandServer>(factories)); - parent.AddCommand(std::make_unique<TClientCommandServerConfig>()); + parent.AddCommand(std::make_unique<TClientCommandServer>(factories)); + parent.AddCommand(std::make_unique<TClientCommandServerConfig>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp index 361e757ca1..eb40faa374 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp @@ -77,7 +77,7 @@ public: TClientCommandKeyValue() : TClientCommandTree("keyvalue", { "kv" }) { - AddCommand(std::make_unique<TClientCommandKeyValueRequest>()); + AddCommand(std::make_unique<TClientCommandKeyValueRequest>()); } }; @@ -258,10 +258,10 @@ public: TClientCommandTabletN() : TClientCommandTree("#", {}, "<tablet id>") { - AddCommand(std::make_unique<TClientCommandKeyValue>()); - AddCommand(std::make_unique<TClientCommandTabletExec>()); - AddCommand(std::make_unique<TClientCommandTabletKill>()); - AddCommand(std::make_unique<TClientCommandTabletSchemeTx>()); + AddCommand(std::make_unique<TClientCommandKeyValue>()); + AddCommand(std::make_unique<TClientCommandTabletExec>()); + AddCommand(std::make_unique<TClientCommandTabletKill>()); + AddCommand(std::make_unique<TClientCommandTabletSchemeTx>()); } virtual void Parse(TConfig& config) override { @@ -273,9 +273,9 @@ public: TClientCommandTablet::TClientCommandTablet() : TClientCommandTree("tablet", {}, "Tablet infrastructure administration") { - AddCommand(std::make_unique<TClientCommandTabletN>()); - AddCommand(std::make_unique<TClientCommandDrainNode>()); - AddCommand(std::make_unique<TClientCommandFillNode>()); + AddCommand(std::make_unique<TClientCommandTabletN>()); + AddCommand(std::make_unique<TClientCommandDrainNode>()); + AddCommand(std::make_unique<TClientCommandFillNode>()); } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp index db44a3a516..696361b379 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp @@ -600,10 +600,10 @@ public: TClientCommandTenantUnits() : TClientCommandTree("units", {}, "") { - AddCommand(std::make_unique<TClientCommandTenantAddUnits>()); - AddCommand(std::make_unique<TClientCommandTenantRemoveUnits>()); - AddCommand(std::make_unique<TClientCommandTenantRegisterUnits>()); - AddCommand(std::make_unique<TClientCommandTenantDeregisterUnits>()); + AddCommand(std::make_unique<TClientCommandTenantAddUnits>()); + AddCommand(std::make_unique<TClientCommandTenantRemoveUnits>()); + AddCommand(std::make_unique<TClientCommandTenantRegisterUnits>()); + AddCommand(std::make_unique<TClientCommandTenantDeregisterUnits>()); } }; @@ -612,7 +612,7 @@ public: TClientCommandTenantPools() : TClientCommandTree("pools", {}, "") { - AddCommand(std::make_unique<TClientCommandTenantAddPools>()); + AddCommand(std::make_unique<TClientCommandTenantAddPools>()); } }; @@ -621,11 +621,11 @@ public: TClientCommandTenantName() : TClientCommandTree("*", {}, "<database name>") { - AddCommand(std::make_unique<TClientCommandTenantCreate>()); - AddCommand(std::make_unique<TClientCommandTenantPools>()); - AddCommand(std::make_unique<TClientCommandTenantRemove>()); - AddCommand(std::make_unique<TClientCommandTenantStatus>()); - AddCommand(std::make_unique<TClientCommandTenantUnits>()); + AddCommand(std::make_unique<TClientCommandTenantCreate>()); + AddCommand(std::make_unique<TClientCommandTenantPools>()); + AddCommand(std::make_unique<TClientCommandTenantRemove>()); + AddCommand(std::make_unique<TClientCommandTenantStatus>()); + AddCommand(std::make_unique<TClientCommandTenantUnits>()); } virtual void Parse(TConfig& config) override { @@ -638,9 +638,9 @@ public: TClientCommandTenant::TClientCommandTenant() : TClientCommandTree("database", {"db", "tenant"}, "Database administration") { - AddCommand(std::make_unique<TClientCommandTenantName>()); - AddCommand(std::make_unique<TClientCommandTenantList>()); - AddCommand(std::make_unique<TClientCommandTenantOptions>()); + AddCommand(std::make_unique<TClientCommandTenantName>()); + AddCommand(std::make_unique<TClientCommandTenantList>()); + AddCommand(std::make_unique<TClientCommandTenantOptions>()); } } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 819c1478d1..acd7c7afb5 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -1300,7 +1300,7 @@ void TBootstrapperInitializer::InitializeServices( for (const ui32 bootstrapperNode : boot.GetNode()) { if (bootstrapperNode == NodeId) { - TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(boot.GetInfo())); + TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(boot.GetInfo())); auto tabletType = BootstrapperTypeToTabletType(boot.GetType()); @@ -1351,7 +1351,7 @@ void TTabletsInitializer::InitializeServices( for (const auto& tabletConfig: Config.GetTabletsConfig().GetTablet()) { for (ui32 bootstrapperNode: tabletConfig.GetNode()) { if (bootstrapperNode == setup->NodeId) { - auto tabletInfo = TabletStorageInfoFromProto(tabletConfig.GetInfo()); + auto tabletInfo = TabletStorageInfoFromProto(tabletConfig.GetInfo()); auto tabletType = tabletConfig.GetType(); auto tabletSetup = CreateTablet(tabletType, tabletInfo, appData, CustomTablets); diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp index e2b12e2d7a..e5dc327568 100644 --- a/ydb/core/engine/mkql_engine_flat_ut.cpp +++ b/ydb/core/engine/mkql_engine_flat_ut.cpp @@ -4776,10 +4776,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["1"];["11"];"mapped_11";"Ok"]; - [["5"];["15"];"mapped_15";"Ok"]; - [["4"];["14"];"mapped_14";"Ok"]; - [["6"];["16"];"mapped_16";"Ok"] + [["1"];["11"];"mapped_11";"Ok"]; + [["5"];["15"];"mapped_15";"Ok"]; + [["4"];["14"];"mapped_14";"Ok"]; + [["6"];["16"];"mapped_16";"Ok"] ]]])___", res); } @@ -4839,14 +4839,14 @@ Value { NKqp::CompareYson(R"___([ [[ - [["1"];["11"];["Ok"]]; - [["3"];["13"];["Bad"]]; - [["5"];["15"];["Ok"]] + [["1"];["11"];["Ok"]]; + [["3"];["13"];["Bad"]]; + [["5"];["15"];["Ok"]] ]]; [[ - [["1"];["Ok"]]; - [["3"];["Bad"]]; - [["5"];["Ok"]] + [["1"];["Ok"]]; + [["3"];["Bad"]]; + [["5"];["Ok"]] ]] ])___", res); } @@ -4906,9 +4906,9 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [[[["3"];["Bad"]]];["1"];["Ok"]]; - [[[["3"];["Bad"]]];["3"];["Bad"]]; - [[[["3"];["Bad"]]];["5"];["Ok"]] + [[[["3"];["Bad"]]];["1"];["Ok"]]; + [[[["3"];["Bad"]]];["3"];["Bad"]]; + [[[["3"];["Bad"]]];["5"];["Ok"]] ]]])___", res); } @@ -4983,8 +4983,8 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["1"];["11"];"mapped_11";["Ok"]]; - [["5"];["15"];"mapped_15";["Ok"]] + [["1"];["11"];"mapped_11";["Ok"]]; + [["5"];["15"];"mapped_15";["Ok"]] ]]])___", res); } @@ -5083,10 +5083,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(checkpgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[[ - [["0"];["0"]]; - [["1"];["1"]]; - [["3"];["3"]]; - [["5"];["5"]]]; + [["0"];["0"]]; + [["1"];["1"]]; + [["3"];["3"]]; + [["5"];["5"]]]; %false]]])___", res); } } @@ -5174,10 +5174,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [[["1"];["Ok"]];[["2"];["20"]]]; - [[["1"];["Ok"]];[["4"];["40"]]]; - [[["3"];["Bad"]];[["2"];["20"]]]; - [[["3"];["Bad"]];[["4"];["40"]]] + [[["1"];["Ok"]];[["2"];["20"]]]; + [[["1"];["Ok"]];[["4"];["40"]]]; + [[["3"];["Bad"]];[["2"];["20"]]]; + [[["3"];["Bad"]];[["4"];["40"]]] ]]])___", res); } @@ -5272,10 +5272,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["1"];[["20"];["40"]]]; - [["Ok"];[["20"];["40"]]]; - [["3"];[["20"];["40"]]]; - [["Bad"];[["20"];["40"]]] + [["1"];[["20"];["40"]]]; + [["Ok"];[["20"];["40"]]]; + [["3"];[["20"];["40"]]]; + [["Bad"];[["20"];["40"]]] ]]])___", res); } @@ -5386,10 +5386,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["Value1"];["6"]]; - [["Value2"];["3"]]; - [["Value2"];["8"]]; - [["Value3"];["4"]] + [["Value1"];["6"]]; + [["Value2"];["3"]]; + [["Value2"];["8"]]; + [["Value3"];["4"]] ]]])___", res); } @@ -5496,9 +5496,9 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["Value1"];["6"]]; - [["Value2"];["11"]]; - [["Value3"];["4"]] + [["Value1"];["6"]]; + [["Value2"];["11"]]; + [["Value3"];["4"]] ]]])___", res); } @@ -5579,8 +5579,8 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["3"];["Value2"]]; - [["2"];["Value2"]] + [["3"];["Value2"]]; + [["2"];["Value2"]] ]]])___", res); } @@ -5653,10 +5653,10 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([[[ - [["1"];["Value1"]]; - [["2"];["Value2"]]; - [["3"];["Value1"]]; - [["4"];["Value2"]] + [["1"];["Value1"]]; + [["2"];["Value2"]]; + [["3"];["Value1"]]; + [["4"];["Value2"]] ]]])___", res); } @@ -6347,8 +6347,8 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); NKqp::CompareYson(R"___([ - [[[[["1"];["Value1"]];[["3"];["Value3"]]]; - [[["2"];["12"];["20"]];[["4"];["14"];["40"]]]]] + [[[[["1"];["Value1"]];[["3"];["Value3"]]]; + [[["2"];["12"];["20"]];[["4"];["14"];["40"]]]]] ])___", res); } diff --git a/ydb/core/erasure/erasure.cpp b/ydb/core/erasure/erasure.cpp index a41b027932..2b80c49d5e 100644 --- a/ydb/core/erasure/erasure.cpp +++ b/ydb/core/erasure/erasure.cpp @@ -201,8 +201,8 @@ public: Prime = type.Prime(); CrcMode = crcMode; - - Data = nullptr; + + Data = nullptr; } template <bool isStripe> @@ -277,7 +277,7 @@ public: // Use the remaining parts to fill in the last block // Write the tail of the data if (TailSize) { - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockData<isStripe>(lastBlockSource, bufferDataPart); @@ -534,7 +534,7 @@ public: // Use the remaining parts to fill in the last block // Write the tail of the data if (TailSize) { - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; if (!isFromDataParts) { PrepareLastBlockData<isStripe>(lastBlockSource, bufferDataPart); @@ -569,7 +569,7 @@ public: // Use the remaining parts to fill in the last block // Write the tail of the data if (hasTail && outPartSet.IsSplitDone()) { - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; if (!isFromDataParts) { PrepareLastBlockData<isStripe>(lastBlockSource, bufferDataPart); @@ -753,7 +753,7 @@ public: // Read the tail of the data if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { TRACE("EoDiagonalRestorePart tail" << Endl); - char lastBlock[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlock[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockPointers<isStripe>(lastBlock, bufferDataPart); @@ -1119,7 +1119,7 @@ public: partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB); if (TailSize) { - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockPointers<isStripe>(lastBlockSource, bufferDataPart); @@ -1151,7 +1151,7 @@ public: partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC); if (TailSize) { - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockPointers<isStripe>(lastBlockSource, bufferDataPart); @@ -1197,7 +1197,7 @@ public: if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { TRACE("EoMainRestoreParts restore tail" << Endl); - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockPointers<isStripe>(lastBlockSource, bufferDataPart); @@ -1316,8 +1316,8 @@ public: beginBlockIdx, beginBlockIdx + wholeBlocks, missingDataPartIdx); if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { - TRACE("Restore tail, restoreFullData# " << restoreFullData << " restoreParts# " << restoreParts << Endl); - char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; + TRACE("Restore tail, restoreFullData# " << restoreFullData << " restoreParts# " << restoreParts << Endl); + char lastBlockSource[MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)] = {}; TBufferDataPart bufferDataPart; PrepareLastBlockPointers<isStripe>(lastBlockSource, bufferDataPart); diff --git a/ydb/core/grpc_services/rpc_alter_table.cpp b/ydb/core/grpc_services/rpc_alter_table.cpp index d1f2755b66..1014ac6219 100644 --- a/ydb/core/grpc_services/rpc_alter_table.cpp +++ b/ydb/core/grpc_services/rpc_alter_table.cpp @@ -526,8 +526,8 @@ private: Request_->ReplyWithYdbStatus(status); Die(ctx); } - ui64 TxId = 0; - ui64 SchemeshardId = 0; + ui64 TxId = 0; + ui64 SchemeshardId = 0; TActorId SchemeCache; TString DatabaseName; TIntrusivePtr<NTxProxy::TTxProxyMon> TxProxyMon; diff --git a/ydb/core/grpc_services/rpc_cancel_operation.cpp b/ydb/core/grpc_services/rpc_cancel_operation.cpp index 579f927c80..ac844e3ee1 100644 --- a/ydb/core/grpc_services/rpc_cancel_operation.cpp +++ b/ydb/core/grpc_services/rpc_cancel_operation.cpp @@ -115,7 +115,7 @@ public: private: TOperationId OperationId; - ui64 RawOperationId = 0; + ui64 RawOperationId = 0; }; // TCancelOperationRPC diff --git a/ydb/core/grpc_services/rpc_forget_operation.cpp b/ydb/core/grpc_services/rpc_forget_operation.cpp index 8b250125ca..e17c7c20f1 100644 --- a/ydb/core/grpc_services/rpc_forget_operation.cpp +++ b/ydb/core/grpc_services/rpc_forget_operation.cpp @@ -115,7 +115,7 @@ public: private: TOperationId OperationId; - ui64 RawOperationId = 0; + ui64 RawOperationId = 0; }; // TForgetOperationRPC diff --git a/ydb/core/grpc_services/rpc_get_operation.cpp b/ydb/core/grpc_services/rpc_get_operation.cpp index 6f7a140686..2e206e74eb 100644 --- a/ydb/core/grpc_services/rpc_get_operation.cpp +++ b/ydb/core/grpc_services/rpc_get_operation.cpp @@ -274,7 +274,7 @@ private: } TOperationId OperationId_; - ui64 RawOperationId_ = 0; + ui64 RawOperationId_ = 0; TActorId PipeActorId_; }; diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index 4baae606a9..3088f4ce73 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -81,8 +81,8 @@ public: , Timeout(TDuration::Seconds(DEFAULT_TIMEOUT_SEC)) , WaitingResolveReply(false) , Finished(false) - , MinKeyInclusive(0) - , MaxKeyInclusive(0) + , MinKeyInclusive(0) + , MaxKeyInclusive(0) , ShardRequestCount(0) , ShardReplyCount(0) , SysViewMaxRows(100000) diff --git a/ydb/core/grpc_services/rpc_yq.cpp b/ydb/core/grpc_services/rpc_yq.cpp index 6fe5f868d6..ef5504e98d 100644 --- a/ydb/core/grpc_services/rpc_yq.cpp +++ b/ydb/core/grpc_services/rpc_yq.cpp @@ -81,12 +81,12 @@ public: FolderId = path.back(); if (!FolderId) { ReplyWithStatus("Folder id is empty", StatusIds::BAD_REQUEST); - return; + return; } if (FolderId.length() > 1024) { ReplyWithStatus("Folder id length greater than 1024 characters: " + FolderId, StatusIds::BAD_REQUEST); - return; + return; } const TString& internalToken = proxyCtx->GetInternalToken(); diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h index 1e06beca39..3510fc6968 100644 --- a/ydb/core/kqp/common/kqp_gateway.h +++ b/ydb/core/kqp/common/kqp_gateway.h @@ -101,7 +101,7 @@ public: struct TKqpSnapshotHandle : public IKqpGateway::TGenericResult { TKqpSnapshot Snapshot; NActors::TActorId ManagingActor; - NKikimrIssues::TStatusIds::EStatusCode Status = NKikimrIssues::TStatusIds::UNKNOWN; + NKikimrIssues::TStatusIds::EStatusCode Status = NKikimrIssues::TStatusIds::UNKNOWN; }; struct TExecPhysicalRequest : private TMoveOnly { diff --git a/ydb/core/kqp/executer/kqp_partition_helper.cpp b/ydb/core/kqp/executer/kqp_partition_helper.cpp index ac786f6c24..4ea433861a 100644 --- a/ydb/core/kqp/executer/kqp_partition_helper.cpp +++ b/ydb/core/kqp/executer/kqp_partition_helper.cpp @@ -146,7 +146,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq:: for (TPartitionWithRange& partitionWithRange : rangePartitions) { ui64 shardId = partitionWithRange.PartitionInfo->ShardId; - shardParamValues[shardId].emplace_back(paramValue); + shardParamValues[shardId].emplace_back(paramValue); auto& shardData = ret[shardId]; if (partitionWithRange.FullRange) { diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 44af6d1168..9d84bb4b6c 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -1051,7 +1051,7 @@ public: TypesCtx->TimeProvider = TAppData::TimeProvider; TypesCtx->RandomProvider = TAppData::RandomProvider; TypesCtx->Modules = ModuleResolver; - TypesCtx->UserDataStorage = MakeIntrusive<TUserDataStorage>(nullptr, TUserDataTable(), nullptr, nullptr); + TypesCtx->UserDataStorage = MakeIntrusive<TUserDataStorage>(nullptr, TUserDataTable(), nullptr, nullptr); TypesCtx->JsonQueryReturnsJsonDocument = true; // Result provider diff --git a/ydb/core/kqp/host/kqp_run_scan.cpp b/ydb/core/kqp/host/kqp_run_scan.cpp index 7c91ae8200..15e2577534 100644 --- a/ydb/core/kqp/host/kqp_run_scan.cpp +++ b/ydb/core/kqp/host/kqp_run_scan.cpp @@ -116,7 +116,7 @@ public: if (!handle.Snapshot.IsValid()) { YQL_CLOG(NOTICE, ProviderKqp) << "Failed to create persistent snapshot. " << "Status: " << NKikimrIssues::TStatusIds_EStatusCode_Name(handle.Status) - << ", issues: " << handle.Issues().ToString(); + << ", issues: " << handle.Issues().ToString(); TIssue issue("Failed to create persistent snapshot"); switch (handle.Status) { @@ -135,7 +135,7 @@ public: break; } - for (const auto& subIssue: handle.Issues()) { + for (const auto& subIssue: handle.Issues()) { issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); } ctx.AddError(issue); diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index 42b4eca7c6..ca7855c185 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -705,7 +705,7 @@ public: break; } - for (const auto& subIssue: handle.Issues()) { + for (const auto& subIssue: handle.Issues()) { issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); } ctx.AddError(issue); diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 47489a9bd5..ff641ae270 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -37,10 +37,10 @@ namespace NKikimr { namespace NKqp { -using NYql::TIssue; -using TIssuesIds = NYql::TIssuesIds; +using NYql::TIssue; +using TIssuesIds = NYql::TIssuesIds; using namespace NThreading; -using namespace NYql::NCommon; +using namespace NYql::NCommon; using namespace NSchemeShard; using namespace NKikimrSchemeOp; @@ -65,7 +65,7 @@ namespace { template <class TResult> static NThreading::TFuture<TResult> NotImplemented() { TResult result; - result.AddIssue(TIssue({}, "Not implemented in interconnect gateway.")); + result.AddIssue(TIssue({}, "Not implemented in interconnect gateway.")); return NThreading::MakeFuture(result); } @@ -783,7 +783,7 @@ public: message << ", reason: " << response.GetSchemeShardReason(); } - Promise.SetValue(ResultFromError<TResult>(TIssue({}, message))); + Promise.SetValue(ResultFromError<TResult>(TIssue({}, message))); this->Die(ctx); } @@ -1006,7 +1006,7 @@ public: return Cluster; } - TMaybe<NYql::TKikimrClusterConfig> GetClusterConfig(const TString& cluster) override { + TMaybe<NYql::TKikimrClusterConfig> GetClusterConfig(const TString& cluster) override { Y_UNUSED(cluster); return {}; } @@ -1089,7 +1089,7 @@ public: } } - TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir) override { + TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir) override { using TRequest = TEvTxUserProxy::TEvProposeTransaction; try { @@ -1870,7 +1870,7 @@ public: handle.Snapshot = response.Snapshot; handle.ManagingActor = snapMgrActorId; handle.Status = response.Status; - handle.AddIssues(response.Issues); + handle.AddIssues(response.Issues); promise.SetValue(handle); } ); @@ -1895,7 +1895,7 @@ public: IKqpGateway::TKqpSnapshotHandle handle; handle.Snapshot = response.Snapshot; handle.Status = response.Status; - handle.AddIssues(response.Issues); + handle.AddIssues(response.Issues); promise.SetValue(handle); } ); diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index 65ef7eb932..7d674b281b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -320,7 +320,7 @@ public: for (auto& cluster : Gateway->GetClusters()) { auto token = defaultToken; - if (auto credential = Types.FindCredential(TString("default_") + cluster)) { + if (auto credential = Types.FindCredential(TString("default_") + cluster)) { if (credential->Category != KikimrProviderName) { ctx.AddError(TIssue({}, TStringBuilder() << "Mismatch credential category, for cluster " << cluster diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index e49f84b5e1..25e893f83c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -48,7 +48,7 @@ TKikimrClusterMapping::TKikimrClusterMapping(const TKikimrGatewayConfig& config) for (size_t i = 0; i < config.ClusterMappingSize(); ++i) { const TKikimrClusterConfig& cluster = config.GetClusterMapping(i); - auto name = cluster.GetName(); + auto name = cluster.GetName(); if (Clusters.contains(name)) { ythrow yexception() << "TKikimrGatewayConfig: Duplicate cluster name: " << name; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 635d164827..a1cee242ee 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -422,7 +422,7 @@ NNodes::TCoAtomList BuildColumnsList( .Done(); } -NNodes::TCoAtomList BuildKeyColumnsList(const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx) { +NNodes::TCoAtomList BuildKeyColumnsList(const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx) { TVector<TExprBase> columnsToSelect; columnsToSelect.reserve(table.Metadata->KeyColumnNames.size()); for (auto key : table.Metadata->KeyColumnNames) { @@ -741,9 +741,9 @@ bool AddDmlIssue(const TIssue& issue, bool strictDml, TExprContext& ctx) { ctx.AddError(newIssue); return false; } else { - if (!ctx.AddWarning(issue)) { - return false; - } + if (!ctx.AddWarning(issue)) { + return false; + } return true; } } diff --git a/ydb/core/kqp/provider/yql_kikimr_results.cpp b/ydb/core/kqp/provider/yql_kikimr_results.cpp index 8ec93f416d..f6d002a137 100644 --- a/ydb/core/kqp/provider/yql_kikimr_results.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_results.cpp @@ -26,13 +26,13 @@ bool ResultsOverflow(ui64 rows, ui64 bytes, const IDataProvider::TFillSettings& return false; } -void WriteValueToYson(const TStringStream& stream, NCommon::TYsonResultWriter& writer, const NKikimrMiniKQL::TType& type, +void WriteValueToYson(const TStringStream& stream, NCommon::TYsonResultWriter& writer, const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, const TVector<TString>* fieldsOrder, const IDataProvider::TFillSettings& fillSettings, bool& truncated, bool firstLevel = false) { switch (type.GetKind()) { case NKikimrMiniKQL::ETypeKind::Void: - writer.OnVoid(); + writer.OnVoid(); return; case NKikimrMiniKQL::ETypeKind::Data: @@ -98,7 +98,7 @@ void WriteValueToYson(const TStringStream& stream, NCommon::TYsonResultWriter& w } if (value.HasFloat()) { - writer.OnFloatScalar(value.GetFloat()); + writer.OnFloatScalar(value.GetFloat()); } if (value.HasDouble()) { @@ -351,8 +351,8 @@ void KikimrResultToYson(const TStringStream& stream, NYson::TYsonWriter& writer, const TVector<TString>& columnHints, const IDataProvider::TFillSettings& fillSettings, bool& truncated) { truncated = false; - NCommon::TYsonResultWriter resultWriter(writer); - WriteValueToYson(stream, resultWriter, result.GetType(), result.GetValue(), columnHints.empty() ? nullptr : &columnHints, + NCommon::TYsonResultWriter resultWriter(writer); + WriteValueToYson(stream, resultWriter, result.GetType(), result.GetValue(), columnHints.empty() ? nullptr : &columnHints, fillSettings, truncated, true); } diff --git a/ydb/core/mind/bscontroller/group_geometry_info.h b/ydb/core/mind/bscontroller/group_geometry_info.h index 10e5daedba..46b76f590b 100644 --- a/ydb/core/mind/bscontroller/group_geometry_info.h +++ b/ydb/core/mind/bscontroller/group_geometry_info.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "defs.h" @@ -73,7 +73,7 @@ namespace NKikimr::NBsController { i64 requiredSpace) const { TString error; for (const bool requireOperational : {true, false}) { - if (mapper.AllocateGroup(groupId, group, replacedDiskIds, numReplacedDisks, forbid, + if (mapper.AllocateGroup(groupId, group, replacedDiskIds, numReplacedDisks, forbid, requiredSpace, requireOperational, error)) { return; } diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index ba311a254c..d32d8925c7 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -905,8 +905,8 @@ public: using Table = Schema::BoxStoragePool; struct TPDiskFilter { - Schema::BoxStoragePoolPDiskFilter::BoxId::Type BoxId{}; - Schema::BoxStoragePoolPDiskFilter::StoragePoolId::Type StoragePoolId{}; + Schema::BoxStoragePoolPDiskFilter::BoxId::Type BoxId{}; + Schema::BoxStoragePoolPDiskFilter::StoragePoolId::Type StoragePoolId{}; TMaybe<Schema::BoxStoragePoolPDiskFilter::TypeCol::Type> Type; TMaybe<Schema::BoxStoragePoolPDiskFilter::SharedWithOs::Type> SharedWithOs; TMaybe<Schema::BoxStoragePoolPDiskFilter::ReadCentric::Type> ReadCentric; diff --git a/ydb/core/mind/configured_tablet_bootstrapper.cpp b/ydb/core/mind/configured_tablet_bootstrapper.cpp index 008acd2b40..ff67157425 100644 --- a/ydb/core/mind/configured_tablet_bootstrapper.cpp +++ b/ydb/core/mind/configured_tablet_bootstrapper.cpp @@ -67,7 +67,7 @@ class TConfiguredTabletBootstrapper : public TActorBootstrapped<TConfiguredTable // not apply config const ui32 selfNode = SelfId().NodeId(); if (Find(config.GetNode(), selfNode) != config.GetNode().end()) { - TIntrusivePtr<TTabletStorageInfo> storageInfo = TabletStorageInfoFromProto(config.GetInfo()); + TIntrusivePtr<TTabletStorageInfo> storageInfo = TabletStorageInfoFromProto(config.GetInfo()); const auto *appData = AppData(); // extract from kikimr_services_initializer diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index 58ca2ac3e4..ff5eb14a4b 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -271,7 +271,7 @@ void THive::ProcessBootQueue() { } void THive::PostponeProcessBootQueue(TDuration after) { - if (!ProcessBootQueuePostponed) { + if (!ProcessBootQueuePostponed) { BLOG_D("PostponeProcessBootQueue (" << after << ")"); ProcessBootQueuePostponed = true; Schedule(after, new TEvPrivate::TEvPostponeProcessBootQueue()); @@ -1086,7 +1086,7 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) { for (size_t i = 0; i < dcs.size(); ++i) { dataCentersGroupsHolder[i].AddDataCenter(dcs[i]); dataCentersGroupsHolder[i].AddDataCenterNum(DataCenterFromString(dcs[i])); - dataCentersGroupsPointers[i] = dataCentersGroupsHolder.data() + i; + dataCentersGroupsPointers[i] = dataCentersGroupsHolder.data() + i; } dataCentersGroups = TArrayRef<const NKikimrHive::TDataCentersGroup*>(dataCentersGroupsPointers.data(), dcTablets.size()); } @@ -1104,11 +1104,11 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) { const NKikimrHive::TDataCentersGroup* dcGroup = dataCentersGroups[numGroup]; if (dcGroup->DataCenterSize()) { for (TDataCenterId dc : dcGroup->GetDataCenter()) { - indexDC2Group[dc] = candidateGroups.data() + numGroup; + indexDC2Group[dc] = candidateGroups.data() + numGroup; } } else { for (const ui64 dcId : dcGroup->GetDataCenterNum()) { - indexDC2Group[DataCenterToString(dcId)] = candidateGroups.data() + numGroup; + indexDC2Group[DataCenterToString(dcId)] = candidateGroups.data() + numGroup; } } } @@ -1137,7 +1137,7 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) { << " checking candidates group " << (itCandidateNodes - candidateGroups.begin() + 1) << " of " << candidateGroups.size()); } - + selectedNodes.clear(); selectedNodes.reserve(candidateNodes.size()); @@ -1602,7 +1602,7 @@ void THive::FillTabletInfo(NKikimrHive::TEvResponseHiveInfo& response, ui64 tabl if (!follower.IsRunning()) { tabletInfo.SetLastAliveTimestamp(follower.Statistics.GetLastAliveTimestamp()); } - tabletInfo.SetRestartsPerPeriod(follower.Statistics.RestartTimestampSize()); + tabletInfo.SetRestartsPerPeriod(follower.Statistics.RestartTimestampSize()); if (req.GetReturnMetrics()) { tabletInfo.MutableMetrics()->CopyFrom(follower.GetResourceValues()); } @@ -1820,7 +1820,7 @@ void THive::Handle(TEvHive::TEvCutTabletHistory::TPtr& ev) { void THive::Handle(TEvHive::TEvDrainNode::TPtr& ev) { Execute(CreateSwitchDrainOn(ev->Get()->Record.GetNodeID(), { - .Persist = ev->Get()->Record.GetPersist(), + .Persist = ev->Get()->Record.GetPersist(), .KeepDown = ev->Get()->Record.GetKeepDown(), .DrainInFlight = ev->Get()->Record.GetDrainInFlight(), }, ev->Sender)); @@ -2189,7 +2189,7 @@ THive::THive(TTabletStorageInfo *info, const TActorId &tablet) , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) , HiveUid(Max<ui32>()) , HiveDomain(Max<ui32>()) - , RootHiveId() + , RootHiveId() , HiveId(Max<ui64>()) , HiveGeneration(0) , PipeClientCacheConfig(new NTabletPipe::TBoundedClientCacheConfig()) diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index 7e9a83b354..a18761a55f 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -2010,7 +2010,7 @@ public: TNodeId NodeId = 0; bool Wait = true; TActorId WaitActorId; - + TTxMonEvent_DrainNode(const TActorId& source, NMon::TEvRemoteHttpInfo::TPtr& ev, TSelf* hive) : TBase(hive) , Event(ev->Release()) @@ -2035,9 +2035,9 @@ public: void Complete(const TActorContext& ctx) override { if (Wait) { - Self->Execute(Self->CreateSwitchDrainOn(NodeId, {.Persist = true, .KeepDown = true}, WaitActorId)); + Self->Execute(Self->CreateSwitchDrainOn(NodeId, {.Persist = true, .KeepDown = true}, WaitActorId)); } else { - Self->Execute(Self->CreateSwitchDrainOn(NodeId, {.Persist = true, .KeepDown = true}, {})); + Self->Execute(Self->CreateSwitchDrainOn(NodeId, {.Persist = true, .KeepDown = true}, {})); ctx.Send(Source, new NMon::TEvRemoteJsonInfoRes("{\"status\":\"SCHEDULED\"}")); } } diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index 09fc15f76d..7d205133c8 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -378,7 +378,7 @@ class TLocalNodeRegistrar : public TActorBootstrapped<TLocalNodeRegistrar> { void Handle(TEvLocal::TEvBootTablet::TPtr &ev, const TActorContext &ctx) { NKikimrLocal::TEvBootTablet &record = ev->Get()->Record; - TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(record.GetInfo())); + TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(record.GetInfo())); info->HiveId = HiveId; TTabletId tabletId(info->TabletID, record.GetFollowerId()); LOG_DEBUG_S(ctx, NKikimrServices::LOCAL, "TLocalNodeRegistrar::Handle TEvLocal::TEvBootTablet tabletId:" diff --git a/ydb/core/mind/table_adapter.h b/ydb/core/mind/table_adapter.h index f0ec8db1e2..80bf5596e7 100644 --- a/ydb/core/mind/table_adapter.h +++ b/ydb/core/mind/table_adapter.h @@ -279,7 +279,7 @@ namespace NKikimr { template<typename TCallback> void ForEachInlineTable(TCallback &&callback) const { - Value.ForEachInlineTable(std::forward<TCallback>(callback)); + Value.ForEachInlineTable(std::forward<TCallback>(callback)); } }; @@ -318,8 +318,8 @@ namespace NKikimr { template<typename TCallback> void ForEachInlineTable(TCallback &&callback) const { - Value.ForEachInlineTable(callback); - Rest.ForEachInlineTable(callback); + Value.ForEachInlineTable(callback); + Rest.ForEachInlineTable(callback); } }; @@ -483,7 +483,7 @@ namespace NKikimr { } }; try { - TBase::Apply(param, [&](auto *adapter) { adapter->ForEachInlineTable(processInlineTable); }); + TBase::Apply(param, [&](auto *adapter) { adapter->ForEachInlineTable(processInlineTable); }); } catch (const TNotReady&) { return false; } @@ -554,7 +554,7 @@ namespace NKikimr { template<typename TCallback> void ForEachInlineTable(TCallback &&callback) const { - Cells.ForEachInlineTable(std::forward<TCallback>(callback)); + Cells.ForEachInlineTable(std::forward<TCallback>(callback)); } }; diff --git a/ydb/core/mon/mon.h b/ydb/core/mon/mon.h index 1cb5dbecd0..657ea21e02 100644 --- a/ydb/core/mon/mon.h +++ b/ydb/core/mon/mon.h @@ -12,7 +12,7 @@ namespace NActors { class TActorSystem; struct TActorId; - class TMon : public NMonitoring::TMonService2 { + class TMon : public NMonitoring::TMonService2 { public: using TRequestAuthorizer = std::function<IEventHandle*(const NActors::TActorId& owner, NMonitoring::IMonHttpRequest& request)>; diff --git a/ydb/core/mon/ya.make b/ydb/core/mon/ya.make index c717b0f5b9..830918e64e 100644 --- a/ydb/core/mon/ya.make +++ b/ydb/core/mon/ya.make @@ -2,7 +2,7 @@ LIBRARY() OWNER( ddoarn - xenoxeno + xenoxeno g:kikimr ) diff --git a/ydb/core/mon_alloc/profiler.cpp b/ydb/core/mon_alloc/profiler.cpp index bfff45c180..5abbb32048 100644 --- a/ydb/core/mon_alloc/profiler.cpp +++ b/ydb/core/mon_alloc/profiler.cpp @@ -43,7 +43,7 @@ namespace NActors { } }; - class TLfAllocProfiler: public IProfilerLogic { + class TLfAllocProfiler: public IProfilerLogic { public: void Start() override { NAllocProfiler::StartAllocationSampling(true); @@ -56,7 +56,7 @@ namespace NActors { } }; - class TYtAllocProfiler: public IProfilerLogic { + class TYtAllocProfiler: public IProfilerLogic { public: TYtAllocProfiler() { Init(); @@ -158,7 +158,7 @@ namespace NActors { #endif // PROFILE_MEMORY_ALLOCATIONS #if defined(EXEC_PROFILER_ENABLED) - class TExecProfiler: public IProfilerLogic { + class TExecProfiler: public IProfilerLogic { public: void Start() override { ResetProfile(); @@ -184,7 +184,7 @@ namespace NActors { }; #endif // EXEC_PROFILER_ENABLED - struct TFakeProfiler: public IProfilerLogic { + struct TFakeProfiler: public IProfilerLogic { void Start() override { } @@ -195,7 +195,7 @@ namespace NActors { } }; - std::unique_ptr<IProfilerLogic> CreateProfiler() { + std::unique_ptr<IProfilerLogic> CreateProfiler() { const auto& info = NMalloc::MallocInfo(); TStringBuf name(info.Name); @@ -232,7 +232,7 @@ namespace NActors { private: const TDynamicCountersPtr DynamicCounters; const TString Dir; - const std::unique_ptr<IProfilerLogic> Profiler; + const std::unique_ptr<IProfilerLogic> Profiler; TCounters Counters; bool IsProfiling = false; @@ -245,7 +245,7 @@ namespace NActors { return ACTORLIB_STATS; } - TProfilerActor(TDynamicCountersPtr counters, TString dir, std::unique_ptr<IProfilerLogic> profiler) + TProfilerActor(TDynamicCountersPtr counters, TString dir, std::unique_ptr<IProfilerLogic> profiler) : TActor(&TThis::StateWork) , DynamicCounters(std::move(counters)) , Dir(std::move(dir)) @@ -466,10 +466,10 @@ namespace NActors { } } - IActor* CreateProfilerActor(TDynamicCountersPtr counters, TString dir, std::unique_ptr<IProfilerLogic> profiler) { + IActor* CreateProfilerActor(TDynamicCountersPtr counters, TString dir, std::unique_ptr<IProfilerLogic> profiler) { return new TProfilerActor( std::move(counters), std::move(dir), - profiler ? std::move(profiler) : CreateProfiler()); + profiler ? std::move(profiler) : CreateProfiler()); } } diff --git a/ydb/core/mon_alloc/profiler.h b/ydb/core/mon_alloc/profiler.h index 4fba8941da..bacc5ee7bd 100644 --- a/ydb/core/mon_alloc/profiler.h +++ b/ydb/core/mon_alloc/profiler.h @@ -104,12 +104,12 @@ namespace NActors { }; }; - struct IProfilerLogic { - virtual ~IProfilerLogic() = default; - virtual void Start() = 0; + struct IProfilerLogic { + virtual ~IProfilerLogic() = default; + virtual void Start() = 0; virtual void Stop(IOutputStream& out, size_t limit, bool forLog) = 0; - }; - + }; + inline TActorId MakeProfilerID(ui32 nodeId) { char x[12] = {'p', 'r', 'o', 'f', 'i', 'l', 'e', 'r', 's', 'e', 'r', 'v'}; return TActorId(nodeId, TStringBuf(x, 12)); @@ -117,6 +117,6 @@ namespace NActors { IActor* CreateProfilerActor( TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - TString dir, - std::unique_ptr<IProfilerLogic> profiler = nullptr); + TString dir, + std::unique_ptr<IProfilerLogic> profiler = nullptr); } diff --git a/ydb/core/persqueue/key.h b/ydb/core/persqueue/key.h index 098c67cf7d..77278e9c92 100644 --- a/ydb/core/persqueue/key.h +++ b/ydb/core/persqueue/key.h @@ -134,7 +134,7 @@ public: } TKey() - : TKey(TypeNone, 0, 0, 0, 0, 0) + : TKey(TypeNone, 0, 0, 0, 0, 0) {} virtual ~TKey() diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index cc8e2419cf..0086d96355 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -4613,11 +4613,11 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u return; } - const TString user = info.User; + const TString user = info.User; bool res = ReadInfo.insert({cookie, std::move(info)}).second; Y_VERIFY(res); - THolder<TEvPQ::TEvBlobRequest> request(new TEvPQ::TEvBlobRequest(user, cookie, Partition, + THolder<TEvPQ::TEvBlobRequest> request(new TEvPQ::TEvBlobRequest(user, cookie, Partition, lastOffset, std::move(blobs))); ctx.Send(BlobCache, request.Release()); diff --git a/ydb/core/persqueue/type_coders.h b/ydb/core/persqueue/type_coders.h index 26c2566d32..5079e42623 100644 --- a/ydb/core/persqueue/type_coders.h +++ b/ydb/core/persqueue/type_coders.h @@ -166,7 +166,7 @@ public: inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { const auto zigZagged = static_cast<i64>(ZigZagEncode(value)); - char varIntOut[sizeof(zigZagged) + 1]; + char varIntOut[sizeof(zigZagged) + 1]; auto bytes = out_long(zigZagged, varIntOut); output->Write(varIntOut, bytes); return bytes; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index c17c8a7dc3..98e24ff749 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -707,40 +707,40 @@ message TActivity { EXPORT_SCAN_ACTOR = 392; EXPORT_YT_UPLOADER_ACTOR = 393; EXPORT_S3_UPLOADER_ACTOR = 394; - YF_SYNC_SERVICE_ACTOR = 395; - YF_SYNC_SERVICE_WORKER_ACTOR = 396; - YF_DESCRIPTIONS_SERVICE_ACTOR = 397; - YF_GW_SERIALIZER_ACTOR = 398; - YF_SERVICE_ACTOR = 399; - YF_WORKER_ACTOR_SERVICE_ACTOR = 400; - YF_RTMR_WORKER_ACTOR_SERVICE_ACTOR = 401; - YF_NAMESERVER_WRAPPER_ACTOR = 402; - YF_DIRECT_CLIENT_BALANCER_ACTOR = 403; - YF_AGENT_ACTOR = 404; - YF_ARCHIVE_CACHE_ACTOR = 405; - YF_ARTIFACT_CACHE_ACTOR = 406; - YF_SKY_GET_COMMAND_ACTOR = 407; - YF_UNPACKER_ACTOR = 408; - YF_CALL_BALANCER_ACTOR = 409; - YF_GATEWAY_ACTOR = 410; - YF_GATEWAY_SERVICE_ACTOR = 411; - YF_GRPC_SERVICE_COUNTER_UPDATER_ACTOR = 412; - YF_INFLIGHT_REQUEST_ACTOR = 413; - YF_MON_SERVICE_ACTOR = 414; + YF_SYNC_SERVICE_ACTOR = 395; + YF_SYNC_SERVICE_WORKER_ACTOR = 396; + YF_DESCRIPTIONS_SERVICE_ACTOR = 397; + YF_GW_SERIALIZER_ACTOR = 398; + YF_SERVICE_ACTOR = 399; + YF_WORKER_ACTOR_SERVICE_ACTOR = 400; + YF_RTMR_WORKER_ACTOR_SERVICE_ACTOR = 401; + YF_NAMESERVER_WRAPPER_ACTOR = 402; + YF_DIRECT_CLIENT_BALANCER_ACTOR = 403; + YF_AGENT_ACTOR = 404; + YF_ARCHIVE_CACHE_ACTOR = 405; + YF_ARTIFACT_CACHE_ACTOR = 406; + YF_SKY_GET_COMMAND_ACTOR = 407; + YF_UNPACKER_ACTOR = 408; + YF_CALL_BALANCER_ACTOR = 409; + YF_GATEWAY_ACTOR = 410; + YF_GATEWAY_SERVICE_ACTOR = 411; + YF_GRPC_SERVICE_COUNTER_UPDATER_ACTOR = 412; + YF_INFLIGHT_REQUEST_ACTOR = 413; + YF_MON_SERVICE_ACTOR = 414; YF_SKY_FETCHER_ACTOR = 415; - YF_LIST_INSTANCES_JOB_CACHE_ACTOR = 416; - YF_IC_CLIENT_ACTOR = 417; - YF_DIRECT_CLIENT_SENDER_ACTOR = 418; - YF_QUEUED_FECTER_ACTOR = 419; - YF_GRPC_JOB_SERVICE_ACTOR = 420; - YF_HTTP_FETCHER_ACTOR = 421; + YF_LIST_INSTANCES_JOB_CACHE_ACTOR = 416; + YF_IC_CLIENT_ACTOR = 417; + YF_DIRECT_CLIENT_SENDER_ACTOR = 418; + YF_QUEUED_FECTER_ACTOR = 419; + YF_GRPC_JOB_SERVICE_ACTOR = 420; + YF_HTTP_FETCHER_ACTOR = 421; YF_JOB_SERVICE_LEADER_ACTOR = 422; - YF_JOB_SERVICE_PROXY_ACTOR = 423; - YF_JOB_SERVICE_SESSION_MONITOR_ACTOR = 424; - YF_JOB_SERVICE_WORKER_ACTOR = 425; - YF_LOCAL_FS_FETCHER_ACTOR = 426; - YF_RUNNER_WORKER_PRIVATE_ACTOR = 427; - YF_RUNNER_PRIVATE_ACTOR = 428; + YF_JOB_SERVICE_PROXY_ACTOR = 423; + YF_JOB_SERVICE_SESSION_MONITOR_ACTOR = 424; + YF_JOB_SERVICE_WORKER_ACTOR = 425; + YF_LOCAL_FS_FETCHER_ACTOR = 426; + YF_RUNNER_WORKER_PRIVATE_ACTOR = 427; + YF_RUNNER_PRIVATE_ACTOR = 428; YF_REQUEST_REGISTER_FUNCTION_ACTOR = 429; SERVICE_ACCOUNT_SERVICE_ACTOR = 430; BLOCKSTORE_DISK_REGISTRY_PROXY = 431; diff --git a/ydb/core/tablet/tablet_counters.h b/ydb/core/tablet/tablet_counters.h index a8cc27e9ed..3760e54b6f 100644 --- a/ydb/core/tablet/tablet_counters.h +++ b/ydb/core/tablet/tablet_counters.h @@ -297,7 +297,7 @@ class TCountersArray : TNonCopyable { friend class TTabletCountersBase; friend class TTabletLabeledCountersBase; public: - typedef std::shared_ptr<T> TCountersHolder; + typedef std::shared_ptr<T> TCountersHolder; // TCountersArray(ui32 countersQnt) : CountersQnt(countersQnt) @@ -305,8 +305,8 @@ public: , Counters(nullptr) { if (CountersQnt) { - CountersHolder.reset(new T[CountersQnt](), &CheckedArrayDelete<T>); - Counters = CountersHolder.get(); + CountersHolder.reset(new T[CountersQnt](), &CheckedArrayDelete<T>); + Counters = CountersHolder.get(); } } @@ -348,13 +348,13 @@ private: // void Reset(const TCountersArray<T>& rp) { Y_VERIFY(!CountersQnt); - CountersHolder.reset(); + CountersHolder.reset(); Counters = nullptr; CountersQnt = rp.CountersQnt; if (CountersQnt) { - CountersHolder.reset(new T[CountersQnt](), &CheckedArrayDelete<T>); - Counters = CountersHolder.get(); + CountersHolder.reset(new T[CountersQnt](), &CheckedArrayDelete<T>); + Counters = CountersHolder.get(); } for (ui32 i = 0, e = CountersQnt; i < e; ++i) { diff --git a/ydb/core/tablet/tablet_req_blockbs.cpp b/ydb/core/tablet/tablet_req_blockbs.cpp index 7e2e94ec95..3307303af6 100644 --- a/ydb/core/tablet/tablet_req_blockbs.cpp +++ b/ydb/core/tablet/tablet_req_blockbs.cpp @@ -110,7 +110,7 @@ class TTabletReqBlockBlobStorage : public TActorBootstrapped<TTabletReqBlockBlob case NKikimrProto::OK: if (++Replied == ReqActors.size()) return ReplyAndDie(NKikimrProto::OK); - break; + break; default: return ReplyAndDie(msg->Status, msg->ErrorReason); } diff --git a/ydb/core/tablet/tablet_sys.cpp b/ydb/core/tablet/tablet_sys.cpp index 44bfac7b2d..3bf40f620d 100644 --- a/ydb/core/tablet/tablet_sys.cpp +++ b/ydb/core/tablet/tablet_sys.cpp @@ -473,7 +473,7 @@ void TTablet::HandleByFollower(TEvTablet::TEvFollowerUpdate::TPtr &ev) { // update storage info for case of channel history upgrade if (record.HasTabletStorageInfo()) { - Info = TabletStorageInfoFromProto(record.GetTabletStorageInfo()); + Info = TabletStorageInfoFromProto(record.GetTabletStorageInfo()); } // Drop currently running graph rebuild request @@ -1024,7 +1024,7 @@ TTablet::TLogEntry* TTablet::MakeLogEntry(TEvTablet::TCommitInfo &commitInfo, NK const ui32 step = Graph.NextEntry++; TLogEntry *entry = new TLogEntry(step, Graph.Confirmed, 0); - Graph.Queue.push_back(std::unique_ptr<TLogEntry>(entry)); + Graph.Queue.push_back(std::unique_ptr<TLogEntry>(entry)); Graph.Index[step] = entry; entry->IsSnapshot = commitInfo.IsSnapshot || commitInfo.IsTotalSnapshot; entry->IsTotalSnapshot = commitInfo.IsTotalSnapshot; @@ -1093,9 +1093,9 @@ void TTablet::Handle(TEvTablet::TEvCommit::TPtr &ev) { bool TTablet::HandleNext(TEvTablet::TEvCommit::TPtr &ev) { TEvTablet::TEvCommit *msg = ev->Get(); - std::unique_ptr<NKikimrTabletBase::TTabletLogEntry> x(new NKikimrTabletBase::TTabletLogEntry()); + std::unique_ptr<NKikimrTabletBase::TTabletLogEntry> x(new NKikimrTabletBase::TTabletLogEntry()); - TLogEntry *entry = msg->PreCommited ? FindLogEntry(*msg, *x) : MakeLogEntry(*msg, x.get()); + TLogEntry *entry = msg->PreCommited ? FindLogEntry(*msg, *x) : MakeLogEntry(*msg, x.get()); if (entry == nullptr) { CancelTablet(TEvTablet::TEvTabletDead::ReasonInconsistentCommit); @@ -1178,7 +1178,7 @@ bool TTablet::HandleNext(TEvTablet::TEvCommit::TPtr &ev) { entry->StateStorageConfirmed = true; // todo: do real query against state-storage (optionally?) entry->Task = Register( - CreateTabletReqWriteLog(SelfId(), logid, x.release(), msg->References, msg->CommitTactic, Info.Get()) + CreateTabletReqWriteLog(SelfId(), logid, x.release(), msg->References, msg->CommitTactic, Info.Get()) ); Graph.StepsInFlight += 1; @@ -1345,7 +1345,7 @@ void TTablet::SendFollowerAuxUpdate(TLeaderInfo& info, const TActorId& follower, bool TTablet::ProgressCommitQueue() { const ui64 tabletId = TabletID(); while (!Graph.Queue.empty()) { - TLogEntry *entry = Graph.Queue.front().get(); + TLogEntry *entry = Graph.Queue.front().get(); if (!entry->Commited) break; @@ -1357,7 +1357,7 @@ bool TTablet::ProgressCommitQueue() { } if (entry->FollowerUpdate && LeaderInfo && step >= Graph.MinFollowerUpdate) { - Graph.PostponedFollowerUpdates.emplace_back(std::move(Graph.Queue.front())); + Graph.PostponedFollowerUpdates.emplace_back(std::move(Graph.Queue.front())); } else if (entry->WaitFollowerGcAck) { Send(UserTablet, new TEvTablet::TEvFollowerGcApplied(tabletId, StateStorageInfo.KnownGeneration, step, TDuration::Max())); } @@ -1379,7 +1379,7 @@ void TTablet::ProgressFollowerQueue() { const ui32 goodUntil = LeaderInfo ? Graph.ConfirmedCommited : Max<ui32>(); while (!Graph.PostponedFollowerUpdates.empty()) { - TLogEntry *entry = Graph.PostponedFollowerUpdates.front().get(); + TLogEntry *entry = Graph.PostponedFollowerUpdates.front().get(); const ui32 step = entry->Step; if (step > goodUntil) break; @@ -1635,7 +1635,7 @@ bool TTablet::CheckBlobStorageError() { if (!Graph.Queue.empty()) { // Check if the head entry is still waiting to be committed - TLogEntry *entry = Graph.Queue.front().get(); + TLogEntry *entry = Graph.Queue.front().get(); if (entry->Step < BlobStorageErrorStep && entry->Task) { // Commit still inflight, wait for result return false; diff --git a/ydb/core/tablet/tablet_sys.h b/ydb/core/tablet/tablet_sys.h index 661d75a4e0..5707848b71 100644 --- a/ydb/core/tablet/tablet_sys.h +++ b/ydb/core/tablet/tablet_sys.h @@ -112,7 +112,7 @@ class TTablet : public TActor<TTablet> { }; struct TGraph { - typedef TDeque<std::unique_ptr<TLogEntry>> TQueueType; + typedef TDeque<std::unique_ptr<TLogEntry>> TQueueType; typedef THashMap<ui32, TLogEntry *> TIndex; TQueueType Queue; diff --git a/ydb/core/tablet_flat/flat_executor_borrowlogic.cpp b/ydb/core/tablet_flat/flat_executor_borrowlogic.cpp index bd52264b10..3ad9fc8fad 100644 --- a/ydb/core/tablet_flat/flat_executor_borrowlogic.cpp +++ b/ydb/core/tablet_flat/flat_executor_borrowlogic.cpp @@ -454,7 +454,7 @@ void TExecutorBorrowLogic::RestoreFollowerBorrowedInfo(const TLogoBlobID &blobId if (proto.StorageInfoSize() > 0) { Y_VERIFY(proto.StorageInfoSize() == 1); - storedInfo.LoanInfo.StorageInfo = TabletStorageInfoFromProto(proto.GetStorageInfo(0)); + storedInfo.LoanInfo.StorageInfo = TabletStorageInfoFromProto(proto.GetStorageInfo(0)); UpdateStorageInfo(storedInfo.LoanInfo.StorageInfo.Get()); } } @@ -499,7 +499,7 @@ void TExecutorBorrowLogic::RestoreBorrowedInfo(const TLogoBlobID &blobId, const if (proto.StorageInfoSize() > 0) { Y_VERIFY(proto.StorageInfoSize() == 1); - storedInfo.LoanInfo.StorageInfo = TabletStorageInfoFromProto(proto.GetStorageInfo(0)); + storedInfo.LoanInfo.StorageInfo = TabletStorageInfoFromProto(proto.GetStorageInfo(0)); UpdateStorageInfo(storedInfo.LoanInfo.StorageInfo.Get()); } } diff --git a/ydb/core/tablet_flat/flat_page_frames.h b/ydb/core/tablet_flat/flat_page_frames.h index 0edcc1eacc..d4c78bd898 100644 --- a/ydb/core/tablet_flat/flat_page_frames.h +++ b/ydb/core/tablet_flat/flat_page_frames.h @@ -72,7 +72,7 @@ namespace NPage { : Raw(std::move(raw)) , End({ Max<TRowId>(), Max<ui16>(), 0, 0 }) { - Y_VERIFY(uintptr_t(Raw.data()) % alignof(TEntry) == 0); + Y_VERIFY(uintptr_t(Raw.data()) % alignof(TEntry) == 0); auto got = NPage::THello().Read(Raw, EPage::Frames); diff --git a/ydb/core/tablet_flat/flat_range_cache.cpp b/ydb/core/tablet_flat/flat_range_cache.cpp index dbac2c7981..77ffa06dd9 100644 --- a/ydb/core/tablet_flat/flat_range_cache.cpp +++ b/ydb/core/tablet_flat/flat_range_cache.cpp @@ -19,13 +19,13 @@ TArrayRef<TCell> TKeyRangeCache::AllocateArrayCopy(TSpecialMemoryPool* pool, TAr return { }; } - TCell* rawPtr = static_cast<TCell*>(pool->Allocate(sizeof(TCell) * key.size())); - TCell* nextCell = rawPtr; + TCell* rawPtr = static_cast<TCell*>(pool->Allocate(sizeof(TCell) * key.size())); + TCell* nextCell = rawPtr; for (const TCell& cell : key) { - new(nextCell++) TCell(cell); + new(nextCell++) TCell(cell); } - return TArrayRef<TCell>(rawPtr, key.size()); + return TArrayRef<TCell>(rawPtr, key.size()); } TCell TKeyRangeCache::AllocateCellCopy(TSpecialMemoryPool* pool, const TCell& cell) { diff --git a/ydb/core/tablet_flat/flat_redo_writer.h b/ydb/core/tablet_flat/flat_redo_writer.h index dafeeb38e2..f2f3d2744d 100644 --- a/ydb/core/tablet_flat/flat_redo_writer.h +++ b/ydb/core/tablet_flat/flat_redo_writer.h @@ -91,8 +91,8 @@ namespace NRedo { TEvBegin_v1 ev{ { ERedo::Begin, 0, 0x8000, size }, tail, head, serial, stamp }; - void* evBegin = &ev; - return Push(TString(NUtil::NBin::ToByte(evBegin), size), size); + void* evBegin = &ev; + return Push(TString(NUtil::NBin::ToByte(evBegin), size), size); } TWriter& EvFlush(ui32 table, ui64 stamp, TEpoch epoch) @@ -101,8 +101,8 @@ namespace NRedo { TEvFlush ev{ { ERedo::Flush, 0, 0x8000, size }, table, 0, stamp, epoch.ToRedoLog() }; - void* evBegin = &ev; - return Push(TString(NUtil::NBin::ToByte(evBegin), size), size); + void* evBegin = &ev; + return Push(TString(NUtil::NBin::ToByte(evBegin), size), size); } TWriter& EvAnnex(TArrayRef<const NPageCollection::TGlobId> blobs) diff --git a/ydb/core/tablet_flat/flat_scan_actor.h b/ydb/core/tablet_flat/flat_scan_actor.h index 5cbf2797a0..a9eeed963b 100644 --- a/ydb/core/tablet_flat/flat_scan_actor.h +++ b/ydb/core/tablet_flat/flat_scan_actor.h @@ -247,7 +247,7 @@ namespace NOps { conf.AheadLo = Args.AheadLo; conf.AheadHi = Args.AheadHi; - if (Conf.ReadAheadLo != Max<ui64>() && Conf.ReadAheadLo <= conf.AheadLo) { + if (Conf.ReadAheadLo != Max<ui64>() && Conf.ReadAheadLo <= conf.AheadLo) { conf.AheadLo = Conf.ReadAheadLo; } diff --git a/ydb/core/tablet_flat/flat_util_binary.h b/ydb/core/tablet_flat/flat_util_binary.h index 7a6ee78c13..748fcc408a 100644 --- a/ydb/core/tablet_flat/flat_util_binary.h +++ b/ydb/core/tablet_flat/flat_util_binary.h @@ -24,13 +24,13 @@ namespace NBin { return SizeOfOne_<T>(left) + SizeOf(std::forward<Tail>(tail)...); } - template<typename T, typename = TStdLayoutOrVoid<T>> + template<typename T, typename = TStdLayoutOrVoid<T>> static inline char* ToByte(T *ptr) { return static_cast<char*>(static_cast<void*>(ptr)); } - template<typename T, typename = TStdLayoutOrVoid<T>> + template<typename T, typename = TStdLayoutOrVoid<T>> static inline const char* ToByte(const T *ptr) { return static_cast<char*>(static_cast<void*>(ptr)); diff --git a/ydb/core/tablet_flat/util_deref.h b/ydb/core/tablet_flat/util_deref.h index 49d02d690a..599dbc79e6 100644 --- a/ydb/core/tablet_flat/util_deref.h +++ b/ydb/core/tablet_flat/util_deref.h @@ -8,9 +8,9 @@ namespace NKikimr { template<typename T> using TStdLayout = std::enable_if_t<std::is_standard_layout<T>::value, T>; - template<typename T> - using TStdLayoutOrVoid = std::enable_if_t<std::is_standard_layout<T>::value || std::is_void<T>::value, T>; - + template<typename T> + using TStdLayoutOrVoid = std::enable_if_t<std::is_standard_layout<T>::value || std::is_void<T>::value, T>; + template<typename T, typename = TStdLayout<T>> struct TDeref { static constexpr const T* At(const void *ptr, size_t off = 0) noexcept { diff --git a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp index 026b14e461..09ce701797 100644 --- a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp @@ -545,7 +545,7 @@ void TDataShard::Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, return; } - ui64 localTxId = 0; + ui64 localTxId = 0; THolder<IScan> scan; switch (condition) { diff --git a/ydb/core/tx/datashard/datashard__init.cpp b/ydb/core/tx/datashard/datashard__init.cpp index dc12c49959..a1f646f903 100644 --- a/ydb/core/tx/datashard/datashard__init.cpp +++ b/ydb/core/tx/datashard/datashard__init.cpp @@ -288,7 +288,7 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { TString splitDescr; LOAD_SYS_BYTES(db, Schema::Sys_DstSplitDescription, splitDescr); if (!splitDescr.empty()) { - Self->DstSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); + Self->DstSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); bool parseOk = ParseFromStringNoSizeLimit(*Self->DstSplitDescription, splitDescr); Y_VERIFY(parseOk); } @@ -342,7 +342,7 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { TString splitDescr; LOAD_SYS_BYTES(db, Schema::Sys_SrcSplitDescription, splitDescr); if (!splitDescr.empty()) { - Self->SrcSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); + Self->SrcSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); bool parseOk = ParseFromStringNoSizeLimit(*Self->SrcSplitDescription, splitDescr); Y_VERIFY(parseOk); diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index 43abd525e1..84c9fddb0d 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -30,7 +30,7 @@ class TActiveTransaction; struct TScanState { TString LastKey; ui64 Bytes = 0; - Ydb::StatusIds::StatusCode StatusCode = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; + Ydb::StatusIds::StatusCode StatusCode = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; NYql::TIssues Issues; }; diff --git a/ydb/core/tx/datashard/datashard_distributed_erase.cpp b/ydb/core/tx/datashard/datashard_distributed_erase.cpp index fae7d2edeb..73b352cf67 100644 --- a/ydb/core/tx/datashard/datashard_distributed_erase.cpp +++ b/ydb/core/tx/datashard/datashard_distributed_erase.cpp @@ -953,7 +953,7 @@ class TDistEraser: public TActorBootstrapped<TDistEraser> { case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclinedNoSpace: case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusRestarting: // TODO: retry CancelProposal(); - [[fallthrough]]; + [[fallthrough]]; default: Mon->ClientTxStatusCoordinatorDeclined->Inc(); return CoordinatorDeclined(TStringBuilder() << "Tx failed to plan" diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 11313474f8..6ce61100e1 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -1924,8 +1924,8 @@ private: ui64 DstSplitOpId; ui64 SrcSplitOpId; bool DstSplitSchemaInitialized = false; - std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> DstSplitDescription; - std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SrcSplitDescription; + std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> DstSplitDescription; + std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SrcSplitDescription; THashSet<TActorId> SrcAckSplitTo; THashMap<TActorId, THashSet<ui64>> SrcAckPartitioningChangedTo; const ui32 SysTablesToTransferAtSplit[4] = { diff --git a/ydb/core/tx/datashard/datashard_split_dst.cpp b/ydb/core/tx/datashard/datashard_split_dst.cpp index 6575081001..f2e1d158a6 100644 --- a/ydb/core/tx/datashard/datashard_split_dst.cpp +++ b/ydb/core/tx/datashard/datashard_split_dst.cpp @@ -53,7 +53,7 @@ public: } } - Self->DstSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(Ev->Get()->Record.GetSplitDescription()); + Self->DstSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(Ev->Get()->Record.GetSplitDescription()); for (ui32 i = 0; i < Self->DstSplitDescription->SourceRangesSize(); ++i) { ui64 srcTabletId = Self->DstSplitDescription->GetSourceRanges(i).GetTabletID(); diff --git a/ydb/core/tx/datashard/datashard_split_src.cpp b/ydb/core/tx/datashard/datashard_split_src.cpp index a7349fc9cf..3f772dfecc 100644 --- a/ydb/core/tx/datashard/datashard_split_src.cpp +++ b/ydb/core/tx/datashard/datashard_split_src.cpp @@ -31,7 +31,7 @@ public: if (Self->State == TShardState::Ready) { Self->SrcAckSplitTo.insert(Ev->Sender); Self->SrcSplitOpId = opId; - Self->SrcSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(Ev->Get()->Record.GetSplitDescription()); + Self->SrcSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(Ev->Get()->Record.GetSplitDescription()); // Persist split description TString splitDescr; diff --git a/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp b/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp index 04c09cf74b..9bb382e58c 100644 --- a/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp +++ b/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp @@ -51,7 +51,7 @@ void TDataShard::CheckMvccStateChangeCanStart(const TActorContext& ctx) { return; } - [[fallthrough]]; + [[fallthrough]]; case TSwitchState::SWITCHING: { ui64 txInFly = TxInFly(); @@ -60,7 +60,7 @@ void TDataShard::CheckMvccStateChangeCanStart(const TActorContext& ctx) { SetCounter(COUNTER_MVCC_STATE_CHANGE_WAIT_IMMEDIATE_TX_IN_FLY, immediateTxInFly); if (txInFly == 0 && immediateTxInFly == 0 && !Pipeline.HasWaitingSchemeOps()) Execute(CreateTxExecuteMvccStateChange(), ctx); - break; + break; } case TSwitchState::DONE: return; diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 0dab30cb84..611e3f3f3a 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -337,9 +337,9 @@ struct TUserTable : public TThrRefBase { ui32 ColIdEpoch = Max<ui32>(); ui32 ColIdUpdateNo = Max<ui32>(); - ui64 Tablet = 0; - ui64 Epoch = 0; - ui64 UpdateNo = 0; + ui64 Tablet = 0; + ui64 Epoch = 0; + ui64 UpdateNo = 0; }; ui32 LocalTid = Max<ui32>(); diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 650506549e..8974fea3b8 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -2702,10 +2702,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Y_VERIFY(dataColId == alterData->IndexDataColumns.size()); alterData->IndexDataColumns.emplace_back(dataColName); } else { - leakedDataColumnsAlterData.emplace_back(pathId, dataColId); + leakedDataColumnsAlterData.emplace_back(pathId, dataColId); } } else { - leakedDataColumnsAlterData.emplace_back(pathId, dataColId); + leakedDataColumnsAlterData.emplace_back(pathId, dataColId); } if (!rowset.Next()) @@ -3162,7 +3162,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } else if (txState.TxType == TTxState::TxSplitTablePartition || txState.TxType == TTxState::TxMergeTablePartition) { Y_VERIFY(!extraData.empty(), "Split Tx must have non-empty split description"); - txState.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); + txState.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); bool deserializeRes = ParseFromStringNoSizeLimit(*txState.SplitDescription, extraData); Y_VERIFY(deserializeRes); splitOpIds.push_back(operationId); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp index 645d7f37e1..73a0be0428 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp @@ -577,7 +577,7 @@ public: firstRangeBegin = TSerializedCellVec::Serialize(firstKey); } - op.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); + op.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); // Fill src shards TString prevRangeEnd = firstRangeBegin; for (ui64 pi : srcPartitionIdxs) { @@ -656,7 +656,7 @@ public: // Last dst shard ends where src shard used to end rangeEnds.push_back(tableInfo->GetPartitions()[srcPartitionIdx].EndOfRange); - op.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); + op.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); auto* srcRange = op.SplitDescription->AddSourceRanges(); auto srcShardIdx = tableInfo->GetPartitions()[srcPartitionIdx].ShardIdx; srcRange->SetShardIdx(ui64(srcShardIdx.GetLocalId())); diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index 41a8045155..02240b899b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -237,7 +237,7 @@ struct TTxState { THashSet<TShardIdx> ShardsInProgress; // indexes of datashards or pqs that operation waits for THashMap<TShardIdx, std::pair<TActorId, ui32>> SchemeChangeNotificationReceived; bool ReadyForNotifications = false; - std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SplitDescription; + std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SplitDescription; bool TxShardsListFinalized = false; TTxId BuildIndexId; // fields below used for backup/restore diff --git a/ydb/core/tx/tx_proxy/read_table_impl.cpp b/ydb/core/tx/tx_proxy/read_table_impl.cpp index 6c30975fd9..cd3cff5aba 100644 --- a/ydb/core/tx/tx_proxy/read_table_impl.cpp +++ b/ydb/core/tx/tx_proxy/read_table_impl.cpp @@ -2404,8 +2404,8 @@ private: // WARNING: we must work correctly even when ResolveKeySet flip-flops // between resolved partitions in some very unfortunate edge cases. - TShardList oldShardList; - ShardList.swap(oldShardList); + TShardList oldShardList; + ShardList.swap(oldShardList); auto oldShardListIt = oldShardList.begin(); TShardState* oldShard = *oldShardListIt; diff --git a/ydb/core/tx/tx_proxy/storage_tenant_ut.cpp b/ydb/core/tx/tx_proxy/storage_tenant_ut.cpp index c3b8e9921a..e2c1aacc19 100644 --- a/ydb/core/tx/tx_proxy/storage_tenant_ut.cpp +++ b/ydb/core/tx/tx_proxy/storage_tenant_ut.cpp @@ -330,7 +330,7 @@ Y_UNIT_TEST_SUITE(TStorageTenantTest) { void CheckThatDSProxyReturnNoGroupIfTryBlock(TTestActorRuntime* runtime, const NKikimrHive::TEvGetTabletStorageInfoResult& storageInfo, ui32 nodeIdx = 0) { - TIntrusivePtr<TTabletStorageInfo> info = TabletStorageInfoFromProto(storageInfo.GetInfo()); + TIntrusivePtr<TTabletStorageInfo> info = TabletStorageInfoFromProto(storageInfo.GetInfo()); TActorId edge = runtime->AllocateEdgeActor(nodeIdx); IActor* x = CreateTabletReqBlockBlobStorage(edge, info.Get(), Max<ui32>(), false); diff --git a/ydb/core/util/hazard.cpp b/ydb/core/util/hazard.cpp index 6052c5145a..40031eac3b 100644 --- a/ydb/core/util/hazard.cpp +++ b/ydb/core/util/hazard.cpp @@ -62,7 +62,7 @@ THazardPointer* THazardDomain::Acquire() { } while (index < head->Offset) { - // coverity[overwrite_var : FALSE]: false positive, reported to coverity + // coverity[overwrite_var : FALSE]: false positive, reported to coverity head = head->Next; Y_VERIFY(head, "Unexpected failure to find index %" PRISZT " in chunk list", index); } diff --git a/ydb/core/viewer/json_browse.h b/ydb/core/viewer/json_browse.h index cba672826e..89e084a94a 100644 --- a/ydb/core/viewer/json_browse.h +++ b/ydb/core/viewer/json_browse.h @@ -35,8 +35,8 @@ class TJsonBrowse : public TActorBootstrapped<TJsonBrowse> { IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; TJsonSettings JsonSettings; - ui32 Timeout = 0; - bool Recursive = false; + ui32 Timeout = 0; + bool Recursive = false; struct TPathStateInfo { TString Name; diff --git a/ydb/core/viewer/json_bscontrollerinfo.h b/ydb/core/viewer/json_bscontrollerinfo.h index 5df47ed553..ca290e30c7 100644 --- a/ydb/core/viewer/json_bscontrollerinfo.h +++ b/ydb/core/viewer/json_bscontrollerinfo.h @@ -20,7 +20,7 @@ class TJsonBSControllerInfo : public TViewerPipeClient<TJsonBSControllerInfo> { NMon::TEvHttpInfo::TPtr Event; TAutoPtr<TEvBlobStorage::TEvResponseControllerInfo> ControllerInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_compute.h b/ydb/core/viewer/json_compute.h index 980b8bfc1e..c5d7728f74 100644 --- a/ydb/core/viewer/json_compute.h +++ b/ydb/core/viewer/json_compute.h @@ -37,7 +37,7 @@ class TJsonCompute : public TViewerPipeClient<TJsonCompute> { TMap<TNodeId, THolder<TEvWhiteboard::TEvTabletStateResponse>> NodeTabletInfo; THolder<TEvInterconnect::TEvNodesInfo> NodesInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TString User; TString Path; TPathId FilterSubDomain; diff --git a/ydb/core/viewer/json_describe.h b/ydb/core/viewer/json_describe.h index 05ef441f8f..13fd97bfb7 100644 --- a/ydb/core/viewer/json_describe.h +++ b/ydb/core/viewer/json_describe.h @@ -20,7 +20,7 @@ class TJsonDescribe : public TViewerPipeClient<TJsonDescribe> { NMon::TEvHttpInfo::TPtr Event; TAutoPtr<TEvSchemeShard::TEvDescribeSchemeResult> DescribeResult; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_healthcheck.h b/ydb/core/viewer/json_healthcheck.h index 2419c5b8b1..c98ac5f8b7 100644 --- a/ydb/core/viewer/json_healthcheck.h +++ b/ydb/core/viewer/json_healthcheck.h @@ -21,7 +21,7 @@ class TJsonHealthCheck : public TActorBootstrapped<TJsonHealthCheck> { IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_hiveinfo.h b/ydb/core/viewer/json_hiveinfo.h index f4d04ddfaf..50caf0cf45 100644 --- a/ydb/core/viewer/json_hiveinfo.h +++ b/ydb/core/viewer/json_hiveinfo.h @@ -20,7 +20,7 @@ class TJsonHiveInfo : public TViewerPipeClient<TJsonHiveInfo> { NMon::TEvHttpInfo::TPtr Event; TAutoPtr<TEvHive::TEvResponseHiveInfo> HiveInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TNodeId NodeId = 0; public: diff --git a/ydb/core/viewer/json_hivestats.h b/ydb/core/viewer/json_hivestats.h index 7f24693d63..fdc5fd2103 100644 --- a/ydb/core/viewer/json_hivestats.h +++ b/ydb/core/viewer/json_hivestats.h @@ -20,7 +20,7 @@ class TJsonHiveStats : public TViewerPipeClient<TJsonHiveStats> { NMon::TEvHttpInfo::TPtr Event; TAutoPtr<TEvHive::TEvResponseHiveDomainStats> HiveStats; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_hotkeys.h b/ydb/core/viewer/json_hotkeys.h index 290e385a9b..f031d8f186 100644 --- a/ydb/core/viewer/json_hotkeys.h +++ b/ydb/core/viewer/json_hotkeys.h @@ -23,10 +23,10 @@ class TJsonHotkeys : public TViewerPipeClient<TJsonHotkeys> { IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; TAutoPtr<TEvSchemeShard::TEvDescribeSchemeResult> DescribeResult; - ui32 Timeout = 0; - ui32 Limit = 0; - float PollingFactor = 0.0; - bool EnableSampling = false; + ui32 Timeout = 0; + ui32 Limit = 0; + float PollingFactor = 0.0; + bool EnableSampling = false; struct KeysComparator { bool operator ()(const std::pair<ui64, TVector<TString>>& a, const std::pair<ui64, TVector<TString>>& b) const { diff --git a/ydb/core/viewer/json_labeledcounters.h b/ydb/core/viewer/json_labeledcounters.h index 6fce18b32b..72ea4f8b96 100644 --- a/ydb/core/viewer/json_labeledcounters.h +++ b/ydb/core/viewer/json_labeledcounters.h @@ -26,7 +26,7 @@ class TJsonLabeledCounters : public TActorBootstrapped<TJsonLabeledCounters> { TString DC; TVector<TString> Counters; ui32 Version = 1; - ui32 Timeout = 0; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_metainfo.h b/ydb/core/viewer/json_metainfo.h index 54064d7403..69b5c0aa79 100644 --- a/ydb/core/viewer/json_metainfo.h +++ b/ydb/core/viewer/json_metainfo.h @@ -29,8 +29,8 @@ class TJsonMetaInfo : public TActorBootstrapped<TJsonMetaInfo> { IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; TJsonSettings JsonSettings; - ui32 Timeout = 0; - bool Counters = false; + ui32 Timeout = 0; + bool Counters = false; NKikimrViewer::TMetaInfo MetaInfo; TActorId BrowseActorID; using TBrowseRequestKey = std::tuple<TActorId, TTabletId, ui32>; diff --git a/ydb/core/viewer/json_netinfo.h b/ydb/core/viewer/json_netinfo.h index 3d7c3835a0..3133013892 100644 --- a/ydb/core/viewer/json_netinfo.h +++ b/ydb/core/viewer/json_netinfo.h @@ -35,7 +35,7 @@ class TJsonNetInfo : public TViewerPipeClient<TJsonNetInfo> { std::unordered_map<TNodeId, std::unique_ptr<TEvWhiteboard::TEvNodeStateResponse>> NodeNetInfo; std::unique_ptr<TEvInterconnect::TEvNodesInfo> NodesInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TString User; TString Path; diff --git a/ydb/core/viewer/json_nodes.h b/ydb/core/viewer/json_nodes.h index bfc0bae0e2..2aaf9aa417 100644 --- a/ydb/core/viewer/json_nodes.h +++ b/ydb/core/viewer/json_nodes.h @@ -33,7 +33,7 @@ class TJsonNodes : public TViewerPipeClient<TJsonNodes> { std::unordered_map<TString, THolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>> DescribeResult; std::unique_ptr<TEvBlobStorage::TEvControllerConfigResponse> BaseConfig; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TString FilterTenant; std::vector<TNodeId> FilterNodeIds; std::unordered_set<TNodeId> NodeIds; @@ -262,7 +262,7 @@ public: pDiskIndex.emplace(pDiskId, &pDiskInfo); return pDiskInfo; }; - + if (BaseConfig) { const NKikimrBlobStorage::TEvControllerConfigResponse& pbRecord(BaseConfig->Record); const NKikimrBlobStorage::TConfigResponse::TStatus& pbStatus(pbRecord.GetResponse().GetStatus(0)); @@ -318,7 +318,7 @@ public: } } } - + TStringStream json; TProtoToJson::ProtoToJson(json, nodesInfo, JsonSettings); Send(Initiator, new NMon::TEvHttpInfoRes(Viewer->GetHTTPOKJSON() + json.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h index b98dfc2632..4d2c468a9c 100644 --- a/ydb/core/viewer/json_pqconsumerinfo.h +++ b/ydb/core/viewer/json_pqconsumerinfo.h @@ -23,8 +23,8 @@ class TJsonPQConsumerInfo : public TActorBootstrapped<TJsonPQConsumerInfo> { TString Topic; TString Client; TString DC; - ui32 Version = 0; - ui32 Timeout = 0; + ui32 Version = 0; + ui32 Timeout = 0; ui32 Requests = 0; ui32 Responses = 0; diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h index 600d8088e1..a7d88c1776 100644 --- a/ydb/core/viewer/json_query.h +++ b/ydb/core/viewer/json_query.h @@ -26,7 +26,7 @@ class TJsonQuery : public TActorBootstrapped<TJsonQuery> { TJsonSettings JsonSettings; TActorId Initiator; NMon::TEvHttpInfo::TPtr Event; - ui32 Timeout = 0; + ui32 Timeout = 0; TVector<Ydb::ResultSet> ResultSets; TString Action; TString Stats; diff --git a/ydb/core/viewer/json_storage.h b/ydb/core/viewer/json_storage.h index 76e5d2f5e8..05590e337d 100644 --- a/ydb/core/viewer/json_storage.h +++ b/ydb/core/viewer/json_storage.h @@ -43,7 +43,7 @@ class TJsonStorage : public TViewerPipeClient<TJsonStorage> { THashMap<TString, TStoragePoolInfo> StoragePoolInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TString FilterTenant; THashSet<TString> FilterStoragePools; TVector<TString> FilterGroupIds; diff --git a/ydb/core/viewer/json_tabletcounters.h b/ydb/core/viewer/json_tabletcounters.h index 7f39910441..056c33bb0b 100644 --- a/ydb/core/viewer/json_tabletcounters.h +++ b/ydb/core/viewer/json_tabletcounters.h @@ -25,8 +25,8 @@ class TJsonTabletCounters : public TActorBootstrapped<TJsonTabletCounters> { TMap<TTabletId, THolder<TEvTablet::TEvGetCountersResponse>> Results; THolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> DescribeResult; TJsonSettings JsonSettings; - ui32 Timeout = 0; - bool Aggregate = false; + ui32 Timeout = 0; + bool Aggregate = false; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h index 749422ba39..08bdab32c9 100644 --- a/ydb/core/viewer/json_tenantinfo.h +++ b/ydb/core/viewer/json_tenantinfo.h @@ -26,7 +26,7 @@ class TJsonTenantInfo : public TViewerPipeClient<TJsonTenantInfo> { IViewer* Viewer; THashMap<TString, NKikimrViewer::TTenant> TenantByPath; THashMap<TPathId, NKikimrViewer::TTenant> TenantBySubDomainKey; - THashMap<TString, THolder<NSchemeCache::TSchemeCacheNavigate>> NavigateResult; + THashMap<TString, THolder<NSchemeCache::TSchemeCacheNavigate>> NavigateResult; THashMap<TTabletId, THolder<TEvHive::TEvResponseHiveDomainStats>> HiveDomainStats; THashMap<TTabletId, THolder<TEvHive::TEvResponseHiveStorageStats>> HiveStorageStats; NMon::TEvHttpInfo::TPtr Event; @@ -34,7 +34,7 @@ class TJsonTenantInfo : public TViewerPipeClient<TJsonTenantInfo> { TMap<TNodeId, THolder<TEvWhiteboard::TEvSystemStateResponse>> NodeSysInfo; TMap<TNodeId, THolder<TEvWhiteboard::TEvTabletStateResponse>> NodeTabletInfo; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; TString User; TString Path; bool Tablets = false; @@ -325,7 +325,7 @@ public: tenant.MergeFrom(tenantBySubDomainKey); if (!tenant.GetId()) { tenant.SetId(GetDomainId(subDomainKey)); - } + } if (tenant.GetType() == NKikimrViewer::UnknownTenantType) { tenant.SetType(NKikimrViewer::Dedicated); } @@ -402,7 +402,7 @@ public: overall = Max(overall, flag); } } - + THashSet<TNodeId> tenantNodes; for (TNodeId nodeId : tenant.GetNodeIds()) { diff --git a/ydb/core/viewer/json_tenants.h b/ydb/core/viewer/json_tenants.h index 762d5e3b43..548232e4a6 100644 --- a/ydb/core/viewer/json_tenants.h +++ b/ydb/core/viewer/json_tenants.h @@ -21,7 +21,7 @@ class TJsonTenants : public TViewerPipeClient<TJsonTenants> { NKikimrViewer::TTenants Result; NMon::TEvHttpInfo::TPtr Event; TJsonSettings JsonSettings; - ui32 Timeout = 0; + ui32 Timeout = 0; bool State = true; THashMap<TString, NKikimrViewer::TTenant*> TenantIndex; diff --git a/ydb/core/viewer/json_topicinfo.h b/ydb/core/viewer/json_topicinfo.h index cbf42ee727..c7a9b82edb 100644 --- a/ydb/core/viewer/json_topicinfo.h +++ b/ydb/core/viewer/json_topicinfo.h @@ -21,8 +21,8 @@ class TJsonTopicInfo : public TActorBootstrapped<TJsonTopicInfo> { TString Topic; TString Client; TString GroupNames; - bool ShowAll = false; - ui32 Timeout = 0; + bool ShowAll = false; + ui32 Timeout = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { diff --git a/ydb/core/viewer/json_wb_req.h b/ydb/core/viewer/json_wb_req.h index f04c85688f..678ac3e23c 100644 --- a/ydb/core/viewer/json_wb_req.h +++ b/ydb/core/viewer/json_wb_req.h @@ -53,9 +53,9 @@ protected: TString FilterFields; TString MergeFields; TJsonSettings JsonSettings; - bool AllEnums = false; - ui32 Timeout = 0; - ui32 Retries = 0; + bool AllEnums = false; + ui32 Timeout = 0; + ui32 Retries = 0; std::unordered_map<TNodeId, ui32> NodeRetries; bool StaticNodesOnly = TWhiteboardInfo<ResponseType>::StaticNodesOnly; TDuration RetryPeriod = TDuration::MilliSeconds(500); diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 9f6c822a89..c138c59f82 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -1,5 +1,5 @@ syntax = "proto3"; - + import "ydb/core/protos/tablet_counters.proto"; import "ydb/core/protos/node_whiteboard.proto"; import "ydb/core/protos/flat_scheme_op.proto"; diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.cpp b/ydb/core/yq/libs/actors/clusters_from_connections.cpp index 93e0471892..c9dfd08387 100644 --- a/ydb/core/yq/libs/actors/clusters_from_connections.cpp +++ b/ydb/core/yq/libs/actors/clusters_from_connections.cpp @@ -1,138 +1,138 @@ -#include "clusters_from_connections.h" - +#include "clusters_from_connections.h" + #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - -#include <util/generic/hash.h> -#include <util/string/builder.h> -#include <util/system/env.h> - + +#include <util/generic/hash.h> +#include <util/string/builder.h> +#include <util/system/env.h> + namespace NYq { - + using namespace NYql; -namespace { - -template <typename TClusterConfig> -void FillClusterAuth(TClusterConfig& clusterCfg, const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) { - switch (auth.identity_case()) { - case YandexQuery::IamAuth::kNone: - break; - case YandexQuery::IamAuth::kCurrentIam: - clusterCfg.SetToken(authToken); - break; - case YandexQuery::IamAuth::kServiceAccount: - clusterCfg.SetServiceAccountId(auth.service_account().id()); - clusterCfg.SetServiceAccountIdSignature(accountIdSignatures.at(auth.service_account().id())); - break; - // Do not replace with default. Adding a new auth item should cause a compilation error - case YandexQuery::IamAuth::IDENTITY_NOT_SET: - break; - } -} - +namespace { + +template <typename TClusterConfig> +void FillClusterAuth(TClusterConfig& clusterCfg, const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) { + switch (auth.identity_case()) { + case YandexQuery::IamAuth::kNone: + break; + case YandexQuery::IamAuth::kCurrentIam: + clusterCfg.SetToken(authToken); + break; + case YandexQuery::IamAuth::kServiceAccount: + clusterCfg.SetServiceAccountId(auth.service_account().id()); + clusterCfg.SetServiceAccountIdSignature(accountIdSignatures.at(auth.service_account().id())); + break; + // Do not replace with default. Adding a new auth item should cause a compilation error + case YandexQuery::IamAuth::IDENTITY_NOT_SET: + break; + } +} + } //namespace -void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, - bool useBearerForYdb, - const TString& objectStorageEndpoint, - const TString& authToken, - const THashMap<TString, TString>& accountIdSignatures, - TGatewaysConfig& gatewaysConfig, - THashMap<TString, TString>& clusters) { - for (const auto&[_, conn] : connections) { - auto connectionName = conn.content().name(); - switch (conn.content().setting().connection_case()) { - case YandexQuery::ConnectionSetting::kYdbDatabase: { - const auto& db = conn.content().setting().ydb_database(); - auto* clusterCfg = gatewaysConfig.MutableYdb()->AddClusterMapping(); - clusterCfg->SetName(connectionName); - clusterCfg->SetId(db.database_id()); +void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, + bool useBearerForYdb, + const TString& objectStorageEndpoint, + const TString& authToken, + const THashMap<TString, TString>& accountIdSignatures, + TGatewaysConfig& gatewaysConfig, + THashMap<TString, TString>& clusters) { + for (const auto&[_, conn] : connections) { + auto connectionName = conn.content().name(); + switch (conn.content().setting().connection_case()) { + case YandexQuery::ConnectionSetting::kYdbDatabase: { + const auto& db = conn.content().setting().ydb_database(); + auto* clusterCfg = gatewaysConfig.MutableYdb()->AddClusterMapping(); + clusterCfg->SetName(connectionName); + clusterCfg->SetId(db.database_id()); if (db.database()) clusterCfg->SetDatabase(db.database()); if (db.endpoint()) clusterCfg->SetEndpoint(db.endpoint()); - clusterCfg->SetSecure(db.secure()); - clusterCfg->SetAddBearerToToken(useBearerForYdb); - FillClusterAuth(*clusterCfg, db.auth(), authToken, accountIdSignatures); - clusters.emplace(connectionName, YdbProviderName); - break; - } - case YandexQuery::ConnectionSetting::kClickhouseCluster: { - const auto& ch = conn.content().setting().clickhouse_cluster(); - auto* clusterCfg = gatewaysConfig.MutableClickHouse()->AddClusterMapping(); - clusterCfg->SetName(connectionName); + clusterCfg->SetSecure(db.secure()); + clusterCfg->SetAddBearerToToken(useBearerForYdb); + FillClusterAuth(*clusterCfg, db.auth(), authToken, accountIdSignatures); + clusters.emplace(connectionName, YdbProviderName); + break; + } + case YandexQuery::ConnectionSetting::kClickhouseCluster: { + const auto& ch = conn.content().setting().clickhouse_cluster(); + auto* clusterCfg = gatewaysConfig.MutableClickHouse()->AddClusterMapping(); + clusterCfg->SetName(connectionName); clusterCfg->SetId(ch.database_id()); if (ch.host()) clusterCfg->SetCluster(ch.host()); clusterCfg->SetNativeHostPort(9440); clusterCfg->SetNativeSecure(true); - clusterCfg->SetCHToken(TStringBuilder() << "basic#" << ch.login() << "#" << ch.password()); - clusters.emplace(connectionName, ClickHouseProviderName); - break; - } - case YandexQuery::ConnectionSetting::kObjectStorage: { - const auto& s3 = conn.content().setting().object_storage(); - auto* clusterCfg = gatewaysConfig.MutableS3()->AddClusterMapping(); - clusterCfg->SetName(connectionName); - TString objectStorageUrl; - if (objectStorageEndpoint == "https://s3.mds.yandex.net") { - objectStorageUrl = TStringBuilder() << "https://" << s3.bucket() << ".s3.mds.yandex.net/"; - } else { - objectStorageUrl = TStringBuilder() << objectStorageEndpoint << '/' << s3.bucket() << '/'; - } - clusterCfg->SetUrl(objectStorageUrl); - FillClusterAuth(*clusterCfg, s3.auth(), authToken, accountIdSignatures); - clusters.emplace(connectionName, S3ProviderName); - break; - } - case YandexQuery::ConnectionSetting::kDataStreams: { - const auto& ds = conn.content().setting().data_streams(); - auto* clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName(connectionName); + clusterCfg->SetCHToken(TStringBuilder() << "basic#" << ch.login() << "#" << ch.password()); + clusters.emplace(connectionName, ClickHouseProviderName); + break; + } + case YandexQuery::ConnectionSetting::kObjectStorage: { + const auto& s3 = conn.content().setting().object_storage(); + auto* clusterCfg = gatewaysConfig.MutableS3()->AddClusterMapping(); + clusterCfg->SetName(connectionName); + TString objectStorageUrl; + if (objectStorageEndpoint == "https://s3.mds.yandex.net") { + objectStorageUrl = TStringBuilder() << "https://" << s3.bucket() << ".s3.mds.yandex.net/"; + } else { + objectStorageUrl = TStringBuilder() << objectStorageEndpoint << '/' << s3.bucket() << '/'; + } + clusterCfg->SetUrl(objectStorageUrl); + FillClusterAuth(*clusterCfg, s3.auth(), authToken, accountIdSignatures); + clusters.emplace(connectionName, S3ProviderName); + break; + } + case YandexQuery::ConnectionSetting::kDataStreams: { + const auto& ds = conn.content().setting().data_streams(); + auto* clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName(connectionName); if (ds.endpoint()) clusterCfg->SetEndpoint(ds.endpoint()); - clusterCfg->SetDatabase(ds.database()); + clusterCfg->SetDatabase(ds.database()); clusterCfg->SetDatabaseId(ds.database_id()); - clusterCfg->SetUseSsl(ds.secure()); - clusterCfg->SetAddBearerToToken(useBearerForYdb); - clusterCfg->SetClusterType(TPqClusterConfig::CT_DATA_STREAMS); - FillClusterAuth(*clusterCfg, ds.auth(), authToken, accountIdSignatures); - clusters.emplace(connectionName, PqProviderName); - break; - } - case YandexQuery::ConnectionSetting::kMonitoring: { - const auto& monitoring = conn.content().setting().monitoring(); - auto* clusterCfg = gatewaysConfig.MutableSolomon()->AddClusterMapping(); - clusterCfg->SetName(connectionName); - - // TODO: move Endpoint to yq config - auto solomonEndpointForTest = GetEnv("SOLOMON_ENDPOINT"); - auto solomonEndpoint = solomonEndpointForTest ? TString(solomonEndpointForTest) : TString(); - if (solomonEndpoint.empty()) { - if (connectionName.StartsWith("pre")) { - solomonEndpoint = "monitoring.api.cloud-preprod.yandex.net"; + clusterCfg->SetUseSsl(ds.secure()); + clusterCfg->SetAddBearerToToken(useBearerForYdb); + clusterCfg->SetClusterType(TPqClusterConfig::CT_DATA_STREAMS); + FillClusterAuth(*clusterCfg, ds.auth(), authToken, accountIdSignatures); + clusters.emplace(connectionName, PqProviderName); + break; + } + case YandexQuery::ConnectionSetting::kMonitoring: { + const auto& monitoring = conn.content().setting().monitoring(); + auto* clusterCfg = gatewaysConfig.MutableSolomon()->AddClusterMapping(); + clusterCfg->SetName(connectionName); + + // TODO: move Endpoint to yq config + auto solomonEndpointForTest = GetEnv("SOLOMON_ENDPOINT"); + auto solomonEndpoint = solomonEndpointForTest ? TString(solomonEndpointForTest) : TString(); + if (solomonEndpoint.empty()) { + if (connectionName.StartsWith("pre")) { + solomonEndpoint = "monitoring.api.cloud-preprod.yandex.net"; clusterCfg->SetUseSsl(true); - } else if (connectionName.StartsWith("so")) { - solomonEndpoint = "solomon.yandex.net"; - } else { - solomonEndpoint = "monitoring.api.cloud.yandex.net"; + } else if (connectionName.StartsWith("so")) { + solomonEndpoint = "solomon.yandex.net"; + } else { + solomonEndpoint = "monitoring.api.cloud.yandex.net"; clusterCfg->SetUseSsl(true); - } - } - - clusterCfg->SetCluster(solomonEndpoint); - clusterCfg->SetClusterType(TSolomonClusterConfig::SCT_MONITORING); - clusterCfg->MutablePath()->SetProject(monitoring.project()); - clusterCfg->MutablePath()->SetCluster(monitoring.cluster()); - FillClusterAuth(*clusterCfg, monitoring.auth(), authToken, accountIdSignatures); - clusters.emplace(connectionName, SolomonProviderName); - break; - } - - // Do not replace with default. Adding a new connection should cause a compilation error - case YandexQuery::ConnectionSetting::CONNECTION_NOT_SET: - break; - } - } -} + } + } + + clusterCfg->SetCluster(solomonEndpoint); + clusterCfg->SetClusterType(TSolomonClusterConfig::SCT_MONITORING); + clusterCfg->MutablePath()->SetProject(monitoring.project()); + clusterCfg->MutablePath()->SetCluster(monitoring.cluster()); + FillClusterAuth(*clusterCfg, monitoring.auth(), authToken, accountIdSignatures); + clusters.emplace(connectionName, SolomonProviderName); + break; + } + + // Do not replace with default. Adding a new connection should cause a compilation error + case YandexQuery::ConnectionSetting::CONNECTION_NOT_SET: + break; + } + } +} } //NYq diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.h b/ydb/core/yq/libs/actors/clusters_from_connections.h index 08db9704ea..fafbd35e03 100644 --- a/ydb/core/yq/libs/actors/clusters_from_connections.h +++ b/ydb/core/yq/libs/actors/clusters_from_connections.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/public/api/protos/yq.pb.h> - + namespace NYq { -void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, - bool useBearerForYdb, - const TString& objectStorageEndpoint, - const TString& authToken, - const THashMap<TString, TString>& accountIdSignatures, +void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, + bool useBearerForYdb, + const TString& objectStorageEndpoint, + const TString& authToken, + const THashMap<TString, TString>& accountIdSignatures, NYql::TGatewaysConfig& gatewaysConfig, - THashMap<TString, TString>& clusters); + THashMap<TString, TString>& clusters); } //NYq diff --git a/ydb/core/yq/libs/actors/database_resolver.cpp b/ydb/core/yq/libs/actors/database_resolver.cpp index c784d80c00..34887a78fb 100644 --- a/ydb/core/yq/libs/actors/database_resolver.cpp +++ b/ydb/core/yq/libs/actors/database_resolver.cpp @@ -39,8 +39,8 @@ public: TResponseProcessor( const TActorId sender, TCache& cache, - const THashMap<std::pair<TString, DatabaseType>, TEndpoint>& ready, - const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>>& requests, + const THashMap<std::pair<TString, DatabaseType>, TEndpoint>& ready, + const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>>& requests, const TString& traceId, bool mdbTransformHost, const TParsers& parsers) @@ -106,8 +106,8 @@ private: TString status; TString errorMessage; TString databaseId; - DatabaseType databaseType = DatabaseType::Ydb; - TEvents::TDatabaseAuth info; + DatabaseType databaseType = DatabaseType::Ydb; + TEvents::TDatabaseAuth info; TMaybe<TEndpoint> result; HandledIds++; if (ev->Get()->Error.empty() && (ev->Get()->Response && ((status = ev->Get()->Response->Status) == "200"))) { @@ -119,26 +119,26 @@ private: if (it == Requests.end()) { errorMessage = "Unknown databaseId"; } else { - std::tie(databaseId, databaseType, info) = it->second; - const bool parseJsonOk = NJson::ReadJsonTree(ev->Get()->Response->Body, &jsonConfig, &databaseInfo); - TParsers::const_iterator parserIt; - if (parseJsonOk && (parserIt = Parsers.find(databaseType)) != Parsers.end()) { + std::tie(databaseId, databaseType, info) = it->second; + const bool parseJsonOk = NJson::ReadJsonTree(ev->Get()->Response->Body, &jsonConfig, &databaseInfo); + TParsers::const_iterator parserIt; + if (parseJsonOk && (parserIt = Parsers.find(databaseType)) != Parsers.end()) { try { - auto res = parserIt->second(databaseInfo, MdbTransformHost); + auto res = parserIt->second(databaseInfo, MdbTransformHost); LOG_D("Got " << databaseId << " " << databaseType << " endpoint " << res.Endpoint << " " << res.Database); - DatabaseId2Endpoint[std::make_pair(databaseId, databaseType)] = res; + DatabaseId2Endpoint[std::make_pair(databaseId, databaseType)] = res; result.ConstructInPlace(res); } catch (...) { errorMessage = TStringBuilder() << " Couldn't resolve " - << databaseType << " Id: " + << databaseType << " Id: " << databaseId << "\n" << CurrentExceptionMessage(); } } else { errorMessage = TStringBuilder() << "Unable to parse database information." << "Database Id: " << databaseId - << ", Database Type: " << databaseType; + << ", Database Type: " << databaseType; } } } else { @@ -152,17 +152,17 @@ private: if (errorMessage) { LOG_E("Error on response parsing: " << errorMessage); - Success = false; + Success = false; } - if (databaseId) { - auto key = std::make_tuple(databaseId, databaseType, info); + if (databaseId) { + auto key = std::make_tuple(databaseId, databaseType, info); if (errorMessage) { Cache.Put(key, errorMessage); } else { Cache.Put(key, result); } - } + } if (HandledIds == Requests.size()) { SendResolvedEndpointsAndDie(); @@ -171,15 +171,15 @@ private: LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done"); } -private: +private: const TActorId Sender; TCache& Cache; - const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> Requests; + const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> Requests; const TString TraceId; const bool MdbTransformHost; - THashMap<std::pair<TString, DatabaseType>, TEvents::TEvEndpointResponse::TEndpoint> DatabaseId2Endpoint; + THashMap<std::pair<TString, DatabaseType>, TEvents::TEvEndpointResponse::TEndpoint> DatabaseId2Endpoint; size_t HandledIds = 0; - bool Success = true; + bool Success = true; const TParsers& Parsers; TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg }; @@ -187,10 +187,10 @@ private: class TDatabaseResolver: public TActor<TDatabaseResolver> { public: - TDatabaseResolver(TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) + TDatabaseResolver(TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) : TActor<TDatabaseResolver>(&TDatabaseResolver::State) , HttpProxy(httpProxy) - , CredentialsFactory(credentialsFactory) + , CredentialsFactory(credentialsFactory) , Cache( TTtlCacheSettings() .SetTtl(TDuration::Minutes(10)) @@ -199,7 +199,7 @@ public: { auto ydbParser = [](NJson::TJsonValue& databaseInfo, bool) { bool secure = false; - TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); + TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); TString prefix("/?database="); TString database; auto pos = endpoint.rfind(prefix); @@ -215,7 +215,7 @@ public: endpoint = endpoint.substr(pos+3); } - Y_ENSURE(endpoint); + Y_ENSURE(endpoint); return TEvents::TEvEndpointResponse::TEndpoint{endpoint, database, secure}; }; Parsers[DatabaseType::Ydb] = ydbParser; @@ -232,7 +232,7 @@ public: Parsers[DatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, bool mdbTransformHost) { TString endpoint; TVector<TString> aliveHosts; - for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { + for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { if (host["health"].GetString() == "ALIVE" && host["type"].GetString() == "CLICKHOUSE") { aliveHosts.push_back(host["name"].GetString()); } @@ -260,12 +260,12 @@ private: { TraceId = ev->Get()->TraceId; LOG_D("Start databaseId resolver for " << ev->Get()->DatabaseIds.size() << " ids"); - THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> requests; // request, (dbId, type, info) - THashMap<std::pair<TString, DatabaseType>, TEndpoint> ready; - for (const auto& [p, info] : ev->Get()->DatabaseIds) { - const auto& [databaseId, type] = p; + THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> requests; // request, (dbId, type, info) + THashMap<std::pair<TString, DatabaseType>, TEndpoint> ready; + for (const auto& [p, info] : ev->Get()->DatabaseIds) { + const auto& [databaseId, type] = p; TMaybe<std::variant<TEndpoint, TString>> endpoint; - auto key = std::make_tuple(databaseId, type, info); + auto key = std::make_tuple(databaseId, type, info); if (Cache.Get(key, &endpoint)) { if (endpoint) { ready.insert(std::make_pair(p, @@ -275,25 +275,25 @@ private: continue; } - try { - TString url = IsIn({ DatabaseType::Ydb, DatabaseType::DataStreams }, type) ? - ev->Get()->YdbMvpEndpoint + "/database?databaseId=" + databaseId : - ev->Get()->MdbGateway + "/managed-clickhouse/v1/clusters/" + databaseId + "/hosts"; - LOG_D("Get '" << url << "'"); - - NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestGet(url); - - auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, info.StructuredToken, info.AddBearerToToken); - auto token = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); - if (token) { - httpRequest->Set("Authorization", token); - } - - requests[httpRequest] = key; - } catch (const std::exception& e) { + try { + TString url = IsIn({ DatabaseType::Ydb, DatabaseType::DataStreams }, type) ? + ev->Get()->YdbMvpEndpoint + "/database?databaseId=" + databaseId : + ev->Get()->MdbGateway + "/managed-clickhouse/v1/clusters/" + databaseId + "/hosts"; + LOG_D("Get '" << url << "'"); + + NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestGet(url); + + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, info.StructuredToken, info.AddBearerToToken); + auto token = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); + if (token) { + httpRequest->Set("Authorization", token); + } + + requests[httpRequest] = key; + } catch (const std::exception& e) { const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what(); - LOG_E(msg); - Cache.Put(key, endpoint); + LOG_E(msg); + Cache.Put(key, endpoint); } } @@ -305,20 +305,20 @@ private: ctx.Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request))); } } else { - Send(ev->Sender, new TEvents::TEvEndpointResponse(std::move(ready), /*success=*/true)); + Send(ev->Sender, new TEvents::TEvEndpointResponse(std::move(ready), /*success=*/true)); } } -private: +private: TParsers Parsers; const TActorId HttpProxy; - const ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + const ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; TString TraceId; TCache Cache; }; -NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { - return new TDatabaseResolver(httpProxy, credentialsFactory); +NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { + return new TDatabaseResolver(httpProxy, credentialsFactory); } } /* namespace NYq */ diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp index 0095963c8f..f1394e4152 100644 --- a/ydb/core/yq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp @@ -122,7 +122,7 @@ public: , RandomProvider(randomProvider) , DqCompFactory(dqCompFactory) , ServiceCounters(serviceCounters, "pending_fetcher") - , CredentialsFactory(credentialsFactory) + , CredentialsFactory(credentialsFactory) , S3Gateway(s3Gateway) , PqCmConnections(std::move(pqCmConnections)) , Guid(CreateGuidAsString()) @@ -150,7 +150,7 @@ public: Y_UNUSED(ctx); - DatabaseResolver = Register(CreateDatabaseResolver(MakeYqlAnalyticsHttpProxyId(), CredentialsFactory)); + DatabaseResolver = Register(CreateDatabaseResolver(MakeYqlAnalyticsHttpProxyId(), CredentialsFactory)); Send(SelfId(), new NActors::TEvents::TEvWakeup()); LOG_I("STARTED"); @@ -182,10 +182,10 @@ private: } const auto& res = ev->Get()->Result; - + LOG_D("Tasks count: " << res.tasks().size()); if (!res.tasks().empty()) { - ProcessTask(res); + ProcessTask(res); HasRunningRequest = true; GetPendingTask(); } @@ -245,8 +245,8 @@ private: task.user_id(), Guid, task.generation(), VectorFromProto(task.connection()), VectorFromProto(task.binding()), - CredentialsFactory, - serviceAccounts, + CredentialsFactory, + serviceAccounts, task.query_type(), task.execute_mode(), GetEntityIdAsString(CommonConfig.GetIdsPrefix(), EEntityType::RESULT), @@ -263,7 +263,7 @@ private: NProtoInterop::CastFromProto(task.deadline()), ClientCounters); - NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE); + NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE); Register(CreateRunActor(ServiceCounters, std::move(params))); } @@ -296,7 +296,7 @@ private: TActorId DatabaseResolver; - ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; const IHTTPGateway::TPtr S3Gateway; const ::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections; diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h index 442d29ac44..46c34f66e8 100644 --- a/ydb/core/yq/libs/actors/proxy.h +++ b/ydb/core/yq/libs/actors/proxy.h @@ -53,7 +53,7 @@ NActors::IActor* CreatePendingFetcher( NActors::IActor* CreateRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters, - TRunActorParams&& params + TRunActorParams&& params ); struct TResultId { diff --git a/ydb/core/yq/libs/actors/result_writer.cpp b/ydb/core/yq/libs/actors/result_writer.cpp index d51b8aae9f..0cf70c0028 100644 --- a/ydb/core/yq/libs/actors/result_writer.cpp +++ b/ydb/core/yq/libs/actors/result_writer.cpp @@ -46,7 +46,7 @@ public: const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters) : ExecuterId(executerId) - , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) + , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) , ResultId({resultId}) , TraceId(traceId) , Deadline(deadline) @@ -108,7 +108,7 @@ private: Finished = true; NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record); - *queryResult.MutableYson() = ResultBuilder->BuildYson(Head); + *queryResult.MutableYson() = ResultBuilder->BuildYson(Head); if (!Issues.Empty()) { IssuesToMessage(Issues, queryResult.MutableIssues()); } @@ -240,7 +240,7 @@ private: void ProcessData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { auto& data = ev->Get()->Record.GetChannelData().GetData(); - auto resultSet = ResultBuilder->BuildResultSet({data}); + auto resultSet = ResultBuilder->BuildResultSet({data}); FreeSpace -= data.GetRaw().size(); OccupiedSpace += data.GetRaw().size(); diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp index 5549f8f254..18d6555bbd 100644 --- a/ydb/core/yq/libs/actors/run_actor.cpp +++ b/ydb/core/yq/libs/actors/run_actor.cpp @@ -1,7 +1,7 @@ #include "proxy.h" -#include "clusters_from_connections.h" -#include "system_clusters.h" -#include "table_bindings_from_bindings.h" +#include "clusters_from_connections.h" +#include "system_clusters.h" +#include "table_bindings_from_bindings.h" #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/dq/actors/compute/dq_checkpoints.h> @@ -53,8 +53,8 @@ #include <ydb/core/yq/libs/common/entity_id.h> #include <ydb/core/yq/libs/actors/nodes_manager.h> #include <ydb/core/yq/libs/gateway/empty_gateway.h> -#include <ydb/core/yq/libs/read_rule/read_rule_creator.h> -#include <ydb/core/yq/libs/read_rule/read_rule_deleter.h> +#include <ydb/core/yq/libs/read_rule/read_rule_creator.h> +#include <ydb/core/yq/libs/read_rule/read_rule_deleter.h> #include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> #include <util/system/hostname.h> @@ -82,7 +82,7 @@ namespace NYq { using namespace NActors; using namespace NYql; -using namespace NDqs; +using namespace NDqs; class TDeferredCountersCleanupActor : public NActors::TActorBootstrapped<TDeferredCountersCleanupActor> { public: @@ -126,8 +126,8 @@ class TRunActor : public NActors::TActorBootstrapped<TRunActor> { public: explicit TRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters - , TRunActorParams&& params) - : Params(std::move(params)) + , TRunActorParams&& params) + : Params(std::move(params)) , CreatedAt(TInstant::Now()) , ServiceCounters(serviceCounters) , QueryCounters(serviceCounters) @@ -294,11 +294,11 @@ private: } } - void FailOnException() { - Fail(CurrentExceptionMessage()); - } - - void Fail(const TString& errorMessage) { + void FailOnException() { + Fail(CurrentExceptionMessage()); + } + + void Fail(const TString& errorMessage) { LOG_E("Fail for query " << Params.QueryId << ", finishing: " << Finishing << ", details: " << errorMessage); if (YqConnections.empty()) { @@ -426,8 +426,8 @@ private: } else { CredentialsForConsumersCreation.emplace_back(NYdb::CreateInsecureCredentialsProviderFactory()); } - - TopicsForConsumersCreation.emplace_back(std::move(srcDesc)); + + TopicsForConsumersCreation.emplace_back(std::move(srcDesc)); } } } @@ -532,10 +532,10 @@ private: QueryStateUpdateRequest.set_result_set_count(UpdateResultIndices()); for (const auto& graphParams : DqGraphParams) { if (graphParams.GetResultType()) { - TProtoBuilder builder(graphParams.GetResultType(), {graphParams.GetColumns().begin(), graphParams.GetColumns().end()}); - const auto emptyResultSet = builder.BuildResultSet({}); + TProtoBuilder builder(graphParams.GetResultType(), {graphParams.GetColumns().begin(), graphParams.GetColumns().end()}); + const auto emptyResultSet = builder.BuildResultSet({}); auto* header = QueryStateUpdateRequest.add_result_set_meta(); - (*header->mutable_column()) = emptyResultSet.columns(); + (*header->mutable_column()) = emptyResultSet.columns(); } } *request.mutable_result_set_meta() = QueryStateUpdateRequest.result_set_meta(); @@ -778,7 +778,7 @@ private: void RunReadRulesDeletionActor() { TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials; - credentials.reserve(Params.CreatedTopicConsumers.size()); + credentials.reserve(Params.CreatedTopicConsumers.size()); for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) { if (const TString& tokenName = c.token_name()) { credentials.emplace_back( @@ -800,14 +800,14 @@ private: } void RunDqGraphs() { - if (DqGraphParams.empty()) { - *QueryStateUpdateRequest.mutable_started_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(CreatedAt.MilliSeconds()); - QueryStateUpdateRequest.set_resign_query(false); - const bool isOk = Issues.Size() == 0; + if (DqGraphParams.empty()) { + *QueryStateUpdateRequest.mutable_started_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(CreatedAt.MilliSeconds()); + QueryStateUpdateRequest.set_resign_query(false); + const bool isOk = Issues.Size() == 0; Finish(GetFinishStatus(isOk)); - return; - } - + return; + } + { Params.Status = YandexQuery::QueryMeta::RUNNING; Yq::Private::PingTaskRequest request; @@ -852,9 +852,9 @@ private: } void RunNextDqGraph() { - auto& dqGraphParams = DqGraphParams.at(DqGraphIndex); + auto& dqGraphParams = DqGraphParams.at(DqGraphIndex); TDqConfiguration::TPtr dqConfiguration = MakeIntrusive<TDqConfiguration>(); - dqConfiguration->Dispatch(dqGraphParams.GetSettings()); + dqConfiguration->Dispatch(dqGraphParams.GetSettings()); dqConfiguration->FreezeDefaults(); dqConfiguration->FallbackPolicy = "never"; @@ -867,7 +867,7 @@ private: writerResultId.HistoryId = Params.QueryId; writerResultId.Id = Params.ResultId; writerResultId.Owner = Params.Owner; - writerResultId.SetId = DqGrapResultIndices.at(DqGraphIndex); + writerResultId.SetId = DqGrapResultIndices.at(DqGraphIndex); } TVector<TString> columns; for (const auto& column : dqGraphParams.GetColumns()) { @@ -908,43 +908,43 @@ private: LOG_D("Executer: " << ExecuterId << ", Controller: " << ControlId << ", ResultIdActor: " << resultId << ", CheckPointCoordinatior " << CheckpointCoordinatorId); } - void SetupDqSettings(::google::protobuf::RepeatedPtrField< ::NYql::TAttr>& dqSettings) const { - auto attr = dqSettings.Add(); - attr->SetName("MaxTasksPerStage"); - attr->SetValue("500"); - - attr = dqSettings.Add(); - attr->SetName("MaxTasksPerOperation"); + void SetupDqSettings(::google::protobuf::RepeatedPtrField< ::NYql::TAttr>& dqSettings) const { + auto attr = dqSettings.Add(); + attr->SetName("MaxTasksPerStage"); + attr->SetValue("500"); + + attr = dqSettings.Add(); + attr->SetName("MaxTasksPerOperation"); attr->SetValue(ToString(MaxTasksPerOperation)); - - attr = dqSettings.Add(); - attr->SetName("EnableComputeActor"); - attr->SetValue("1"); - - // Streaming queries: - // - turn off timeout; - // - turn on check that query has one graph. + + attr = dqSettings.Add(); + attr->SetName("EnableComputeActor"); + attr->SetValue("1"); + + // Streaming queries: + // - turn off timeout; + // - turn on check that query has one graph. if (Params.QueryType == YandexQuery::QueryContent::STREAMING) { - attr = dqSettings.Add(); - attr->SetName("_TableTimeout"); - attr->SetValue("0"); - - attr = dqSettings.Add(); - attr->SetName("_LiteralTimeout"); - attr->SetValue("0"); - - attr = dqSettings.Add(); - attr->SetName("_OneGraphPerQuery"); - attr->SetValue("1"); - } + attr = dqSettings.Add(); + attr->SetName("_TableTimeout"); + attr->SetValue("0"); + + attr = dqSettings.Add(); + attr->SetName("_LiteralTimeout"); + attr->SetValue("0"); + + attr = dqSettings.Add(); + attr->SetName("_OneGraphPerQuery"); + attr->SetValue("1"); + } if (Params.QueryType == YandexQuery::QueryContent_QueryType_ANALYTICS) { attr = dqSettings.Add(); attr->SetName("AnalyticsHopping"); attr->SetValue("1"); } - } - + } + void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const { for (const auto& pq : Params.GatewaysConfig.GetPq().GetClusterMapping()) { auto& clusterCfg = *gatewaysConfig.MutablePq()->AddClusterMapping(); @@ -957,8 +957,8 @@ private: clusterCfg = solomon; clusters.emplace(clusterCfg.GetName(), SolomonProviderName); } - } - + } + YandexQuery::QueryMeta::ComputeStatus GetFinishStatus(bool isOk) const { if (isOk) { return YandexQuery::QueryMeta::COMPLETED; @@ -1091,7 +1091,7 @@ private: void FillDqGraphParams() { for (const auto& s : Params.DqGraphs) { NYq::NProto::TGraphParams dqGraphParams; - Y_VERIFY(dqGraphParams.ParseFromString(s)); + Y_VERIFY(dqGraphParams.ParseFromString(s)); DqGraphParams.emplace_back(std::move(dqGraphParams)); } } @@ -1112,21 +1112,21 @@ private: bool CompileQuery() { LOG_D("Compiling query ..."); NYql::TGatewaysConfig gatewaysConfig; - SetupDqSettings(*gatewaysConfig.MutableDq()->MutableDefaultSettings()); + SetupDqSettings(*gatewaysConfig.MutableDq()->MutableDefaultSettings()); - THashMap<TString, TString> clusters; + THashMap<TString, TString> clusters; - //todo: consider cluster name clashes - AddClustersFromConfig(gatewaysConfig, clusters); - AddSystemClusters(gatewaysConfig, clusters, Params.AuthToken); - AddClustersFromConnections(YqConnections, + //todo: consider cluster name clashes + AddClustersFromConfig(gatewaysConfig, clusters); + AddSystemClusters(gatewaysConfig, clusters, Params.AuthToken); + AddClustersFromConnections(YqConnections, Params.CommonConfig.GetUseBearerForYdb(), Params.CommonConfig.GetObjectStorageEndpoint(), - Params.AuthToken, - Params.AccountIdSignatures, - // out params: - gatewaysConfig, - clusters); + Params.AuthToken, + Params.AccountIdSignatures, + // out params: + gatewaysConfig, + clusters); TVector<TDataProviderInitializer> dataProvidersInit; const auto dbResolver = std::make_shared<TDatabaseAsyncResolverWithMeta>(TDatabaseAsyncResolverWithMeta(NActors::TActivationContext::ActorSystem(), Params.DatabaseResolver, @@ -1169,7 +1169,7 @@ private: dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false)); } - TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq"); + TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq"); progFactory.SetModules(Params.ModuleResolver); progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr)); progFactory.SetGatewaysConfig(&gatewaysConfig); @@ -1188,53 +1188,53 @@ private: sqlSettings.ClusterMapping = clusters; sqlSettings.SyntaxVersion = 1; sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; - sqlSettings.Flags.insert({ "DqEngineEnable", "DqEngineForce" }); - try { - AddTableBindingsFromBindings(Params.Bindings, YqConnections, sqlSettings); - } catch (const std::exception& e) { - Issues.AddIssue(ExceptionToIssue(e)); - return false; - } - - // parse phase - { - if (!Program->ParseSql(sqlSettings)) { - Issues.AddIssues(Program->Issues()); - return false; - - } - - if (Params.ExecuteMode == YandexQuery::ExecuteMode::PARSE) { - return true; - } - } - - // compile phase - { - if (!Program->Compile("")) { - Issues.AddIssues(Program->Issues()); - return false; - } - - if (Params.ExecuteMode == YandexQuery::ExecuteMode::COMPILE) { - return true; - } - } - - // next phases can be async: optimize, validate, run - TProgram::TFutureStatus futureStatus; - switch (Params.ExecuteMode) { - case YandexQuery::ExecuteMode::EXPLAIN: - futureStatus = Program->OptimizeAsync(""); - break; - case YandexQuery::ExecuteMode::VALIDATE: - futureStatus = Program->ValidateAsync(""); - break; - case YandexQuery::ExecuteMode::RUN: - futureStatus = Program->RunAsync(""); - break; - default: - Issues.AddIssue(TStringBuilder() << "Unexpected execute mode " << static_cast<int>(Params.ExecuteMode)); + sqlSettings.Flags.insert({ "DqEngineEnable", "DqEngineForce" }); + try { + AddTableBindingsFromBindings(Params.Bindings, YqConnections, sqlSettings); + } catch (const std::exception& e) { + Issues.AddIssue(ExceptionToIssue(e)); + return false; + } + + // parse phase + { + if (!Program->ParseSql(sqlSettings)) { + Issues.AddIssues(Program->Issues()); + return false; + + } + + if (Params.ExecuteMode == YandexQuery::ExecuteMode::PARSE) { + return true; + } + } + + // compile phase + { + if (!Program->Compile("")) { + Issues.AddIssues(Program->Issues()); + return false; + } + + if (Params.ExecuteMode == YandexQuery::ExecuteMode::COMPILE) { + return true; + } + } + + // next phases can be async: optimize, validate, run + TProgram::TFutureStatus futureStatus; + switch (Params.ExecuteMode) { + case YandexQuery::ExecuteMode::EXPLAIN: + futureStatus = Program->OptimizeAsync(""); + break; + case YandexQuery::ExecuteMode::VALIDATE: + futureStatus = Program->ValidateAsync(""); + break; + case YandexQuery::ExecuteMode::RUN: + futureStatus = Program->RunAsync(""); + break; + default: + Issues.AddIssue(TStringBuilder() << "Unexpected execute mode " << static_cast<int>(Params.ExecuteMode)); return false; } @@ -1272,7 +1272,7 @@ private: } void Handle(NActors::TEvents::TEvUndelivered::TPtr&) { - Fail("TRunActor::OnUndelivered"); + Fail("TRunActor::OnUndelivered"); } TString FindTokenByName(const TString& tokenName) const { @@ -1283,7 +1283,7 @@ private: return token->second; } } - throw yexception() << "Token " << tokenName << " was not found in secure params"; + throw yexception() << "Token " << tokenName << " was not found in secure params"; } void AddIssueWithSubIssues(const TString& message, const NYql::TIssues& issues) { @@ -1315,7 +1315,7 @@ private: << " }"); } -private: +private: TRunActorParams Params; THashMap<TString, YandexQuery::Connection> YqConnections; @@ -1370,9 +1370,9 @@ private: IActor* CreateRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters, - TRunActorParams&& params + TRunActorParams&& params ) { - return new TRunActor(serviceCounters, std::move(params)); + return new TRunActor(serviceCounters, std::move(params)); } } /* NYq */ diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp index 1446cae1bb..7e0231c9d6 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.cpp +++ b/ydb/core/yq/libs/actors/run_actor_params.cpp @@ -26,12 +26,12 @@ TRunActorParams::TRunActorParams( const TString& userId, const TString& owner, const int64_t previousQueryRevision, - TVector<YandexQuery::Connection> connections, - TVector<YandexQuery::Binding> bindings, + TVector<YandexQuery::Connection> connections, + TVector<YandexQuery::Binding> bindings, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, THashMap<TString, TString> accountIdSignatures, YandexQuery::QueryContent::QueryType queryType, - YandexQuery::ExecuteMode executeMode, + YandexQuery::ExecuteMode executeMode, const TString& resultId, const YandexQuery::StateLoadMode stateLoadMode, const YandexQuery::StreamingDisposition& streamingDisposition, @@ -67,12 +67,12 @@ TRunActorParams::TRunActorParams( , UserId(userId) , Owner(owner) , PreviousQueryRevision(previousQueryRevision) - , Connections(std::move(connections)) - , Bindings(std::move(bindings)) - , CredentialsFactory(std::move(credentialsFactory)) - , AccountIdSignatures(std::move(accountIdSignatures)) + , Connections(std::move(connections)) + , Bindings(std::move(bindings)) + , CredentialsFactory(std::move(credentialsFactory)) + , AccountIdSignatures(std::move(accountIdSignatures)) , QueryType(queryType) - , ExecuteMode(executeMode) + , ExecuteMode(executeMode) , ResultId(resultId) , StateLoadMode(stateLoadMode) , StreamingDisposition(streamingDisposition) diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h index bad5c3c32b..32cd1648b2 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.h +++ b/ydb/core/yq/libs/actors/run_actor_params.h @@ -40,12 +40,12 @@ struct TRunActorParams { // TODO2 : Change name const TString& userId, const TString& owner, const int64_t previousQueryRevision, - TVector<YandexQuery::Connection> connections, - TVector<YandexQuery::Binding> bindings, + TVector<YandexQuery::Connection> connections, + TVector<YandexQuery::Binding> bindings, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, THashMap<TString, TString> accountIdSignatures, YandexQuery::QueryContent::QueryType queryType, - YandexQuery::ExecuteMode executeMode, + YandexQuery::ExecuteMode executeMode, const TString& resultId, const YandexQuery::StateLoadMode stateLoadMode, const YandexQuery::StreamingDisposition& streamingDisposition, @@ -61,8 +61,8 @@ struct TRunActorParams { // TODO2 : Change name const NMonitoring::TDynamicCounterPtr& clientCounters ); - TRunActorParams(const TRunActorParams& params) = default; - TRunActorParams(TRunActorParams&& params) = default; + TRunActorParams(const TRunActorParams& params) = default; + TRunActorParams(TRunActorParams&& params) = default; NYdb::TDriver Driver; NYql::IHTTPGateway::TPtr S3Gateway; @@ -87,11 +87,11 @@ struct TRunActorParams { // TODO2 : Change name const TString Owner; const int64_t PreviousQueryRevision; const TVector<YandexQuery::Connection> Connections; - const TVector<YandexQuery::Binding> Bindings; + const TVector<YandexQuery::Binding> Bindings; const NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; - const THashMap<TString, TString> AccountIdSignatures; + const THashMap<TString, TString> AccountIdSignatures; const YandexQuery::QueryContent::QueryType QueryType; - const YandexQuery::ExecuteMode ExecuteMode; + const YandexQuery::ExecuteMode ExecuteMode; const TString ResultId; const YandexQuery::StateLoadMode StateLoadMode; const YandexQuery::StreamingDisposition StreamingDisposition; diff --git a/ydb/core/yq/libs/actors/system_clusters.cpp b/ydb/core/yq/libs/actors/system_clusters.cpp index 2968ae6487..6b11425fc0 100644 --- a/ydb/core/yq/libs/actors/system_clusters.cpp +++ b/ydb/core/yq/libs/actors/system_clusters.cpp @@ -1,24 +1,24 @@ -#include "system_clusters.h" - +#include "system_clusters.h" + #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> -#include <util/generic/hash.h> - +#include <util/generic/hash.h> + namespace NYq { - + using namespace NYql; -void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken) { - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("logbroker"); - clusterCfg->SetEndpoint("logbroker.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); +void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken) { + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("logbroker"); + clusterCfg->SetEndpoint("logbroker.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); clusterCfg->SetName("logbroker_iam_no_sa"); clusterCfg->SetEndpoint("logbroker.yandex.net:2135"); clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111"); @@ -29,93 +29,93 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin } { const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("lbkx"); - clusterCfg->SetEndpoint("lbkx.logbroker.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.lbkx.logbroker.yandex.net:1111"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("logbroker_prestable"); - clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("logbroker_prestable_iam"); - clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); - clusterCfg->SetServiceAccountId("f6ooj9og003v61fjb821"); - clusterCfg->SetServiceAccountIdSignature("26YmpsY8SkqCNirlRduj8bqt4JQ="); - clusterCfg->SetAddBearerToToken(true); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("logbroker_prestable_iam_no_sa"); - clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); - clusterCfg->SetAddBearerToToken(true); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("lbkxt"); - clusterCfg->SetEndpoint("lbkxt.logbroker.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.lbkxt.logbroker.yandex.net:1111"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("datastreams_preprod"); - clusterCfg->SetEndpoint("lb.cc802lcdcldbsh7b5fd1.ydb.mdb.cloud-preprod.yandex.net:2135"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_DATA_STREAMS); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("yc-logbroker-preprod"); - clusterCfg->SetEndpoint("lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud-preprod.yandex.net:1111"); - clusterCfg->SetDatabase("/pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3"); + clusterCfg->SetName("lbkx"); + clusterCfg->SetEndpoint("lbkx.logbroker.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.lbkx.logbroker.yandex.net:1111"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("logbroker_prestable"); + clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("logbroker_prestable_iam"); + clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); + clusterCfg->SetServiceAccountId("f6ooj9og003v61fjb821"); + clusterCfg->SetServiceAccountIdSignature("26YmpsY8SkqCNirlRduj8bqt4JQ="); + clusterCfg->SetAddBearerToToken(true); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("logbroker_prestable_iam_no_sa"); + clusterCfg->SetEndpoint("logbroker-prestable.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.logbroker-prestable.yandex.net:1111"); + clusterCfg->SetAddBearerToToken(true); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("lbkxt"); + clusterCfg->SetEndpoint("lbkxt.logbroker.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.lbkxt.logbroker.yandex.net:1111"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("datastreams_preprod"); + clusterCfg->SetEndpoint("lb.cc802lcdcldbsh7b5fd1.ydb.mdb.cloud-preprod.yandex.net:2135"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_DATA_STREAMS); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("yc-logbroker-preprod"); + clusterCfg->SetEndpoint("lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud-preprod.yandex.net:1111"); + clusterCfg->SetDatabase("/pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3"); clusterCfg->SetUseSsl(true); - clusterCfg->SetServiceAccountId("bfbf4nc8c849ej11k3aq"); // audit-trails-reader-sa - clusterCfg->SetServiceAccountIdSignature("SyFk1SpKdQa6L8IJxnwvgC57yMQ="); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("yc-logbroker"); - clusterCfg->SetEndpoint("lb.etn03iai600jur7pipla.ydb.mdb.yandexcloud.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud.yandex.net:1111"); - clusterCfg->SetDatabase("/global/b1gvcqr959dbmi1jltep/etn03iai600jur7pipla"); + clusterCfg->SetServiceAccountId("bfbf4nc8c849ej11k3aq"); // audit-trails-reader-sa + clusterCfg->SetServiceAccountIdSignature("SyFk1SpKdQa6L8IJxnwvgC57yMQ="); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("yc-logbroker"); + clusterCfg->SetEndpoint("lb.etn03iai600jur7pipla.ydb.mdb.yandexcloud.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud.yandex.net:1111"); + clusterCfg->SetDatabase("/global/b1gvcqr959dbmi1jltep/etn03iai600jur7pipla"); clusterCfg->SetUseSsl(true); - clusterCfg->SetServiceAccountId("aje0ir9755dd4ia9mpu5"); - clusterCfg->SetServiceAccountIdSignature("xxx"); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutableSolomon()->AddClusterMapping(); - clusterCfg->SetName("solomon_prod"); - clusterCfg->SetCluster("solomon.yandex.net"); - clusterCfg->SetClusterType(TSolomonClusterConfig::SCT_SOLOMON); - clusterCfg->SetUseSsl(true); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), SolomonProviderName); - } -} + clusterCfg->SetServiceAccountId("aje0ir9755dd4ia9mpu5"); + clusterCfg->SetServiceAccountIdSignature("xxx"); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutableSolomon()->AddClusterMapping(); + clusterCfg->SetName("solomon_prod"); + clusterCfg->SetCluster("solomon.yandex.net"); + clusterCfg->SetClusterType(TSolomonClusterConfig::SCT_SOLOMON); + clusterCfg->SetUseSsl(true); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), SolomonProviderName); + } +} } //NYq diff --git a/ydb/core/yq/libs/actors/system_clusters.h b/ydb/core/yq/libs/actors/system_clusters.h index c842948fdc..5de7444b34 100644 --- a/ydb/core/yq/libs/actors/system_clusters.h +++ b/ydb/core/yq/libs/actors/system_clusters.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - + namespace NYq { void AddSystemClusters(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken); diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp index e353a194a4..a8da87066c 100644 --- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp +++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp @@ -1,78 +1,78 @@ -#include "table_bindings_from_bindings.h" - +#include "table_bindings_from_bindings.h" + #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/core/yq/libs/result_formatter/result_formatter.h> -#include <util/generic/vector.h> - +#include <util/generic/vector.h> + namespace NYq { - + using namespace NYql; -namespace { - -void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const YandexQuery::Binding& binding, const THashMap<TString, YandexQuery::Connection>& connections) { - TString clusterType; - TString path; - TString format; - TString compression; - TString schema; - switch (binding.content().setting().binding_case()) { - case YandexQuery::BindingSetting::kDataStreams: { - clusterType = PqProviderName; - auto yds = binding.content().setting().data_streams(); - path = yds.stream_name(); - format = yds.format(); - compression = yds.compression(); - schema = FormatSchema(yds.schema()); - break; - } - case YandexQuery::BindingSetting::kObjectStorage: { - clusterType = S3ProviderName; - const auto s3 = binding.content().setting().object_storage(); - if (s3.subset().empty()) { - throw yexception() << "No subsets in Object Storage binding " << binding.meta().id(); - } - - const auto& s = s3.subset(0); - path = s.path_pattern(); - format = s.format(); - compression = s.compression(); - schema = FormatSchema(s.schema()); - break; - } - - case YandexQuery::BindingSetting::BINDING_NOT_SET: { - throw yexception() << "BINDING_NOT_SET case for binding " << binding.meta().id() << ", name " << binding.content().name(); - } - // Do not add default. Adding a new binding should cause a compilation error - } - - auto connectionPtr = connections.FindPtr(binding.content().connection_id()); - if (!connectionPtr) { - throw yexception() << "Unable to resolve connection for binding " << binding.meta().id() << ", name " << binding.content().name() << ", connection id " << binding.content().connection_id(); - } - - NSQLTranslation::TTableBindingSettings bindSettings; - bindSettings.ClusterType = clusterType; - bindSettings.Settings["cluster"] = connectionPtr->content().name(); - bindSettings.Settings["path"] = path; - bindSettings.Settings["format"] = format; - // todo: fill format parameters - if (compression) { - bindSettings.Settings["compression"] = compression; - } - bindSettings.Settings["schema"] = schema; - - // todo: use visibility to fill either PrivateBindings or ScopedBindings - sqlSettings.PrivateBindings[binding.content().name()] = std::move(bindSettings); -} +namespace { +void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const YandexQuery::Binding& binding, const THashMap<TString, YandexQuery::Connection>& connections) { + TString clusterType; + TString path; + TString format; + TString compression; + TString schema; + switch (binding.content().setting().binding_case()) { + case YandexQuery::BindingSetting::kDataStreams: { + clusterType = PqProviderName; + auto yds = binding.content().setting().data_streams(); + path = yds.stream_name(); + format = yds.format(); + compression = yds.compression(); + schema = FormatSchema(yds.schema()); + break; + } + case YandexQuery::BindingSetting::kObjectStorage: { + clusterType = S3ProviderName; + const auto s3 = binding.content().setting().object_storage(); + if (s3.subset().empty()) { + throw yexception() << "No subsets in Object Storage binding " << binding.meta().id(); + } + + const auto& s = s3.subset(0); + path = s.path_pattern(); + format = s.format(); + compression = s.compression(); + schema = FormatSchema(s.schema()); + break; + } + + case YandexQuery::BindingSetting::BINDING_NOT_SET: { + throw yexception() << "BINDING_NOT_SET case for binding " << binding.meta().id() << ", name " << binding.content().name(); + } + // Do not add default. Adding a new binding should cause a compilation error + } + + auto connectionPtr = connections.FindPtr(binding.content().connection_id()); + if (!connectionPtr) { + throw yexception() << "Unable to resolve connection for binding " << binding.meta().id() << ", name " << binding.content().name() << ", connection id " << binding.content().connection_id(); + } + + NSQLTranslation::TTableBindingSettings bindSettings; + bindSettings.ClusterType = clusterType; + bindSettings.Settings["cluster"] = connectionPtr->content().name(); + bindSettings.Settings["path"] = path; + bindSettings.Settings["format"] = format; + // todo: fill format parameters + if (compression) { + bindSettings.Settings["compression"] = compression; + } + bindSettings.Settings["schema"] = schema; + + // todo: use visibility to fill either PrivateBindings or ScopedBindings + sqlSettings.PrivateBindings[binding.content().name()] = std::move(bindSettings); +} + } //namespace - -void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings) { - for (const auto& binding : bindings) { - FillBinding(sqlSettings, binding, connections); - } -} + +void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings) { + for (const auto& binding : bindings) { + FillBinding(sqlSettings, binding, connections); + } +} } //NYq diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h index 76343b06dc..def2290f13 100644 --- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h +++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/sql/settings/translation_settings.h> #include <ydb/public/api/protos/yq.pb.h> - + namespace NYq { -void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings); +void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings); } //NYq diff --git a/ydb/core/yq/libs/actors/task_get.cpp b/ydb/core/yq/libs/actors/task_get.cpp index d31517aa9f..aed8f37254 100644 --- a/ydb/core/yq/libs/actors/task_get.cpp +++ b/ydb/core/yq/libs/actors/task_get.cpp @@ -112,9 +112,9 @@ public: case YandexQuery::ConnectionSetting::kMonitoring: { return GetServiceAccountId(c.content().setting().monitoring().auth()); } - case YandexQuery::ConnectionSetting::kClickhouseCluster: { - return GetServiceAccountId(c.content().setting().clickhouse_cluster().auth()); - } + case YandexQuery::ConnectionSetting::kClickhouseCluster: { + return GetServiceAccountId(c.content().setting().clickhouse_cluster().auth()); + } // Do not replace with default. Adding a new connection should cause a compilation error case YandexQuery::ConnectionSetting::CONNECTION_NOT_SET: break; diff --git a/ydb/core/yq/libs/actors/ya.make b/ydb/core/yq/libs/actors/ya.make index 3638593814..dfb69fd2b5 100644 --- a/ydb/core/yq/libs/actors/ya.make +++ b/ydb/core/yq/libs/actors/ya.make @@ -3,21 +3,21 @@ OWNER(g:yq) LIBRARY() SRCS( - clusters_from_connections.cpp + clusters_from_connections.cpp database_resolver.cpp - error.cpp - nodes_health_check.cpp + error.cpp + nodes_health_check.cpp nodes_manager.cpp - pending_fetcher.cpp - pinger.cpp + pending_fetcher.cpp + pinger.cpp proxy.cpp - proxy_private.cpp - result_writer.cpp + proxy_private.cpp + result_writer.cpp run_actor.cpp run_actor_params.cpp - system_clusters.cpp - table_bindings_from_bindings.cpp - task_get.cpp + system_clusters.cpp + table_bindings_from_bindings.cpp + task_get.cpp task_ping.cpp task_result_write.cpp ) diff --git a/ydb/core/yq/libs/checkpointing/ut/ya.make b/ydb/core/yq/libs/checkpointing/ut/ya.make index cf9700d584..116c8d62c7 100644 --- a/ydb/core/yq/libs/checkpointing/ut/ya.make +++ b/ydb/core/yq/libs/checkpointing/ut/ya.make @@ -1,6 +1,6 @@ UNITTEST_FOR(ydb/core/yq/libs/checkpointing) -OWNER(g:yq) +OWNER(g:yq) SRCS( checkpoint_coordinator_ut.cpp @@ -17,4 +17,4 @@ SIZE(MEDIUM) YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/yq/libs/checkpointing/ya.make b/ydb/core/yq/libs/checkpointing/ya.make index 8a397f67ef..23eb5d7c03 100644 --- a/ydb/core/yq/libs/checkpointing/ya.make +++ b/ydb/core/yq/libs/checkpointing/ya.make @@ -1,5 +1,5 @@ OWNER( - g:yq + g:yq ) LIBRARY() diff --git a/ydb/core/yq/libs/common/database_token_builder.h b/ydb/core/yq/libs/common/database_token_builder.h index a3e00efab5..10f0d7cfe4 100644 --- a/ydb/core/yq/libs/common/database_token_builder.h +++ b/ydb/core/yq/libs/common/database_token_builder.h @@ -1,5 +1,5 @@ #pragma once - + #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/events/events.h> diff --git a/ydb/core/yq/libs/common/ya.make b/ydb/core/yq/libs/common/ya.make index 4622695700..0e2278ccaa 100644 --- a/ydb/core/yq/libs/common/ya.make +++ b/ydb/core/yq/libs/common/ya.make @@ -21,6 +21,6 @@ YQL_LAST_ABI_VERSION() END() -RECURSE_FOR_TESTS( +RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp index 783b924238..d7a7058de2 100644 --- a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp +++ b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp @@ -331,7 +331,7 @@ private: void SendEmptyResponse(TRequest& ev, std::string logText) { CPS_LOG_I(logText); - TResult result = {}; + TResult result = {}; auto event = std::make_unique<TEvResult>(result); NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); } diff --git a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto index e6ebff0c84..54f9be96a0 100644 --- a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto +++ b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto @@ -23,7 +23,7 @@ message QueryInternal { repeated YandexQuery.Binding binding = 6; YandexQuery.QueryAction action = 7; string ast = 8; // deprected and should not be used, will be removed in future - ExecuteMode execute_mode = 9; + ExecuteMode execute_mode = 9; StateLoadMode state_load_mode = 10; string cloud_id = 11; repeated Yq.Private.TopicConsumer created_topic_consumers = 12; diff --git a/ydb/core/yq/libs/control_plane_storage/ya.make b/ydb/core/yq/libs/control_plane_storage/ya.make index 8304e50db5..2acda2084f 100644 --- a/ydb/core/yq/libs/control_plane_storage/ya.make +++ b/ydb/core/yq/libs/control_plane_storage/ya.make @@ -24,11 +24,11 @@ PEERDIR( ydb/core/yq/libs/common ydb/core/yq/libs/config ydb/core/yq/libs/config/protos - ydb/core/yq/libs/control_plane_storage/events + ydb/core/yq/libs/control_plane_storage/events ydb/core/yq/libs/control_plane_storage/internal ydb/core/yq/libs/control_plane_storage/proto ydb/core/yq/libs/db_schema - ydb/core/yq/libs/graph_params/proto + ydb/core/yq/libs/graph_params/proto ydb/core/yq/libs/shared_resources ydb/core/yq/libs/ydb ydb/library/security @@ -44,6 +44,6 @@ END() RECURSE( events - internal + internal proto ) diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index 398725f7f4..93b2ed024e 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -331,7 +331,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries TVector<TString> filters; if (request.filter().name()) { queryBuilder.AddString("filter_name", request.filter().name()); - filters.push_back("`" NAME_COLUMN_NAME "` ILIKE '%' || $filter_name || '%'"); + filters.push_back("`" NAME_COLUMN_NAME "` ILIKE '%' || $filter_name || '%'"); } if (request.filter().query_type() != YandexQuery::QueryContent::QUERY_TYPE_UNSPECIFIED) { diff --git a/ydb/core/yq/libs/events/events.h b/ydb/core/yq/libs/events/events.h index b3c70e5c5f..7f11c52cc8 100644 --- a/ydb/core/yq/libs/events/events.h +++ b/ydb/core/yq/libs/events/events.h @@ -12,8 +12,8 @@ #include <library/cpp/actors/core/events.h> -#include <util/digest/multi.h> - +#include <util/digest/multi.h> + namespace NYq { using NYdb::NYq::TScope; @@ -113,7 +113,7 @@ struct TEvents { using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>; TFunction Handler; - explicit TEvDbFunctionRequest(const TFunction& handler) + explicit TEvDbFunctionRequest(const TFunction& handler) : Handler(handler) {} }; @@ -121,7 +121,7 @@ struct TEvents { struct TEvDbFunctionResponse : NActors::TEventLocal<TEvDbFunctionResponse, TEventIds::EvDbFunctionResponse> { NYdb::TStatus Status; - explicit TEvDbFunctionResponse(NYdb::TStatus status) + explicit TEvDbFunctionResponse(NYdb::TStatus status) : Status(status) {} }; @@ -130,42 +130,42 @@ struct TEvents { struct TEndpoint { TString Endpoint; TString Database; - bool Secure = false; + bool Secure = false; }; - THashMap<std::pair<TString, DatabaseType>, TEndpoint> DatabaseId2Endpoint; + THashMap<std::pair<TString, DatabaseType>, TEndpoint> DatabaseId2Endpoint; bool Success; - TEvEndpointResponse(THashMap<std::pair<TString, DatabaseType>, TEndpoint>&& res, bool success) - : DatabaseId2Endpoint(std::move(res)) + TEvEndpointResponse(THashMap<std::pair<TString, DatabaseType>, TEndpoint>&& res, bool success) + : DatabaseId2Endpoint(std::move(res)) , Success(success) { } }; - struct TDatabaseAuth { - TString StructuredToken; - bool AddBearerToToken = false; - - bool operator==(const TDatabaseAuth& other) const { - return std::tie(StructuredToken, AddBearerToToken) == std::tie(other.StructuredToken, other.AddBearerToToken); - } - bool operator!=(const TDatabaseAuth& other) const { - return !(*this == other); - } - }; - + struct TDatabaseAuth { + TString StructuredToken; + bool AddBearerToToken = false; + + bool operator==(const TDatabaseAuth& other) const { + return std::tie(StructuredToken, AddBearerToToken) == std::tie(other.StructuredToken, other.AddBearerToToken); + } + bool operator!=(const TDatabaseAuth& other) const { + return !(*this == other); + } + }; + struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> { - THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth> DatabaseIds; // DbId, DatabaseType => database auth + THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth> DatabaseIds; // DbId, DatabaseType => database auth TString YdbMvpEndpoint; TString MdbGateway; TString TraceId; bool MdbTransformHost; TEvEndpointRequest( - const THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth>& databaseIds, + const THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth>& databaseIds, const TString& ydbMvpEndpoint, const TString& mdbGateway, const TString& traceId, - bool mdbTransformHost) + bool mdbTransformHost) : DatabaseIds(databaseIds) , YdbMvpEndpoint(ydbMvpEndpoint) , MdbGateway(mdbGateway) @@ -175,7 +175,7 @@ struct TEvents { }; struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> { - explicit TEvDataStreamsReadRulesCreationResult(NYql::TIssues issues) + explicit TEvDataStreamsReadRulesCreationResult(NYql::TIssues issues) : Issues(std::move(issues)) { } @@ -193,7 +193,7 @@ struct TEvents { }; struct TEvQueryActionResult : NActors::TEventLocal<TEvQueryActionResult, TEventIds::EvQueryActionResult> { - explicit TEvQueryActionResult(YandexQuery::QueryAction action) + explicit TEvQueryActionResult(YandexQuery::QueryAction action) : Action(action) { } @@ -236,10 +236,10 @@ struct TEvents { }; } // namespace NYq - -template<> + +template<> struct THash<NYq::TEvents::TDatabaseAuth> { inline ui64 operator()(const NYq::TEvents::TDatabaseAuth& x) const noexcept { - return MultiHash(x.StructuredToken, x.AddBearerToToken); - } -}; + return MultiHash(x.StructuredToken, x.AddBearerToToken); + } +}; diff --git a/ydb/core/yq/libs/gateway/ya.make b/ydb/core/yq/libs/gateway/ya.make index 503e9b691b..06d00f1f0b 100644 --- a/ydb/core/yq/libs/gateway/ya.make +++ b/ydb/core/yq/libs/gateway/ya.make @@ -11,7 +11,7 @@ PEERDIR( library/cpp/monlib/dynamic_counters ydb/core/yq/libs/actors ydb/core/yq/libs/events - ydb/core/yq/libs/read_rule + ydb/core/yq/libs/read_rule ydb/core/yq/libs/shared_resources ydb/core/yq/libs/tasks_packer ydb/library/yql/providers/common/token_accessor/client diff --git a/ydb/core/yq/libs/hmac/hmac.cpp b/ydb/core/yq/libs/hmac/hmac.cpp index bbd9ab6c66..a445a7231b 100644 --- a/ydb/core/yq/libs/hmac/hmac.cpp +++ b/ydb/core/yq/libs/hmac/hmac.cpp @@ -1,21 +1,21 @@ -#include "hmac.h" - -#include <contrib/libs/openssl/include/openssl/hmac.h> -#include <library/cpp/string_utils/base64/base64.h> -#include <util/generic/yexception.h> - +#include "hmac.h" + +#include <contrib/libs/openssl/include/openssl/hmac.h> +#include <library/cpp/string_utils/base64/base64.h> +#include <util/generic/yexception.h> + namespace NYq { -TString HmacSha1(const TStringBuf data, const TStringBuf secret) { - unsigned char md_value[EVP_MAX_MD_SIZE]; - unsigned md_len = 0; - - Y_ENSURE(HMAC(EVP_sha1(), secret.data(), secret.size(), (unsigned char*)data.data(), data.size(), md_value, &md_len), - "could not encrypt data using HMAC SHA1 method"); - - return TString((char*)md_value, md_len); -} - -TString HmacSha1Base64(const TStringBuf data, const TStringBuf secret) { - return Base64Encode(HmacSha1(data, secret)); -} -} +TString HmacSha1(const TStringBuf data, const TStringBuf secret) { + unsigned char md_value[EVP_MAX_MD_SIZE]; + unsigned md_len = 0; + + Y_ENSURE(HMAC(EVP_sha1(), secret.data(), secret.size(), (unsigned char*)data.data(), data.size(), md_value, &md_len), + "could not encrypt data using HMAC SHA1 method"); + + return TString((char*)md_value, md_len); +} + +TString HmacSha1Base64(const TStringBuf data, const TStringBuf secret) { + return Base64Encode(HmacSha1(data, secret)); +} +} diff --git a/ydb/core/yq/libs/hmac/hmac.h b/ydb/core/yq/libs/hmac/hmac.h index cc4b95b361..bb58e1e804 100644 --- a/ydb/core/yq/libs/hmac/hmac.h +++ b/ydb/core/yq/libs/hmac/hmac.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/generic/strbuf.h> - +#pragma once + +#include <util/generic/string.h> +#include <util/generic/strbuf.h> + namespace NYq { - TString HmacSha1(const TStringBuf data, const TStringBuf secret); - TString HmacSha1Base64(const TStringBuf data, const TStringBuf secret); -} + TString HmacSha1(const TStringBuf data, const TStringBuf secret); + TString HmacSha1Base64(const TStringBuf data, const TStringBuf secret); +} diff --git a/ydb/core/yq/libs/hmac/ut/hmac_ut.cpp b/ydb/core/yq/libs/hmac/ut/hmac_ut.cpp index cdc79edbcb..369cbffbf9 100644 --- a/ydb/core/yq/libs/hmac/ut/hmac_ut.cpp +++ b/ydb/core/yq/libs/hmac/ut/hmac_ut.cpp @@ -1,18 +1,18 @@ #include <ydb/core/yq/libs/hmac/hmac.h> - -#include <library/cpp/string_utils/base64/base64.h> -#include <library/cpp/testing/unittest/registar.h> - + +#include <library/cpp/string_utils/base64/base64.h> +#include <library/cpp/testing/unittest/registar.h> + using namespace NYq; - -namespace { - constexpr TStringBuf SECRET = "AAAA"; - constexpr TStringBuf DATA = "BBBB"; -} - -Y_UNIT_TEST_SUITE(HmacSha) { - Y_UNIT_TEST(HmacSha1) { - UNIT_ASSERT_VALUES_EQUAL("7d54yAPz39TIawdgQBLcawAlXp8=", Base64Encode(HmacSha1(DATA, SECRET))); - UNIT_ASSERT_VALUES_EQUAL("7d54yAPz39TIawdgQBLcawAlXp8=", HmacSha1Base64(DATA, SECRET)); - } -} + +namespace { + constexpr TStringBuf SECRET = "AAAA"; + constexpr TStringBuf DATA = "BBBB"; +} + +Y_UNIT_TEST_SUITE(HmacSha) { + Y_UNIT_TEST(HmacSha1) { + UNIT_ASSERT_VALUES_EQUAL("7d54yAPz39TIawdgQBLcawAlXp8=", Base64Encode(HmacSha1(DATA, SECRET))); + UNIT_ASSERT_VALUES_EQUAL("7d54yAPz39TIawdgQBLcawAlXp8=", HmacSha1Base64(DATA, SECRET)); + } +} diff --git a/ydb/core/yq/libs/hmac/ut/ya.make b/ydb/core/yq/libs/hmac/ut/ya.make index e6cbf314ba..ea64fc0c6d 100644 --- a/ydb/core/yq/libs/hmac/ut/ya.make +++ b/ydb/core/yq/libs/hmac/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST_FOR(ydb/core/yq/libs/hmac) - + OWNER(g:yq) - -SRCS( - hmac_ut.cpp -) - -PEERDIR( + +SRCS( + hmac_ut.cpp +) + +PEERDIR( ydb/core/yq/libs/hmac -) - -END() +) + +END() diff --git a/ydb/core/yq/libs/hmac/ya.make b/ydb/core/yq/libs/hmac/ya.make index 3a83e0dcb3..d6c9306b22 100644 --- a/ydb/core/yq/libs/hmac/ya.make +++ b/ydb/core/yq/libs/hmac/ya.make @@ -1,18 +1,18 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - hmac.cpp -) - -PEERDIR( - contrib/libs/openssl - library/cpp/string_utils/base64 -) - -END() - -RECURSE_FOR_TESTS( - ut -) + +LIBRARY() + +SRCS( + hmac.cpp +) + +PEERDIR( + contrib/libs/openssl + library/cpp/string_utils/base64 +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.cpp b/ydb/core/yq/libs/result_formatter/result_formatter.cpp index 394832b7e5..098f2a2758 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter.cpp @@ -70,7 +70,7 @@ const NYql::TTypeAnnotationNode* MakeStructType( NYql::TExprContext& ctx) { TVector<const NYql::TItemExprType*> items; - items.reserve(i.size()); + items.reserve(i.size()); for (const auto& [k, v] : i) { items.push_back(ctx.MakeType<NYql::TItemExprType>(k, v)); } @@ -282,17 +282,17 @@ TType MakeType(NYdb::TTypeParser& parser, TContext& env) } struct TTypePair { - NKikimr::NMiniKQL::TType* MiniKQLType = nullptr; - const NYql::TTypeAnnotationNode* TypeAnnotation = nullptr; + NKikimr::NMiniKQL::TType* MiniKQLType = nullptr; + const NYql::TTypeAnnotationNode* TypeAnnotation = nullptr; }; -TTypePair FormatColumnType( +TTypePair FormatColumnType( NJson::TJsonValue& root, NYdb::TType type, NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, NYql::TExprContext& ctx) { - TTypePair result; + TTypePair result; NYdb::TTypeParser parser(type); result.MiniKQLType = MakeType<NKikimr::NMiniKQL::TType*>(parser, typeEnv); result.TypeAnnotation = MakeType<const NYql::TTypeAnnotationNode*>(parser, ctx); @@ -335,20 +335,20 @@ void FormatColumnValue( } // namespace -TString FormatSchema(const YandexQuery::Schema& schema) -{ - NYql::TExprContext ctx; - TVector<std::pair<TString, const NYql::TTypeAnnotationNode*>> typedColumns; - typedColumns.reserve(schema.column().size()); - for (const auto& c : schema.column()) { - NYdb::TTypeParser parser(NYdb::TType(c.type())); - auto typeAnnotation = MakeType<const NYql::TTypeAnnotationNode*>(parser, ctx); - typedColumns.emplace_back(c.name(), typeAnnotation); - } - +TString FormatSchema(const YandexQuery::Schema& schema) +{ + NYql::TExprContext ctx; + TVector<std::pair<TString, const NYql::TTypeAnnotationNode*>> typedColumns; + typedColumns.reserve(schema.column().size()); + for (const auto& c : schema.column()) { + NYdb::TTypeParser parser(NYdb::TType(c.type())); + auto typeAnnotation = MakeType<const NYql::TTypeAnnotationNode*>(parser, ctx); + typedColumns.emplace_back(c.name(), typeAnnotation); + } + return NYql::NCommon::WriteTypeToYson(MakeStructType(typedColumns, ctx), NYson::EYsonFormat::Text); -} - +} + void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet) { NYql::TExprContext ctx; diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.h b/ydb/core/yq/libs/result_formatter/result_formatter.h index 031196e050..8f2fd7ef2c 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.h +++ b/ydb/core/yq/libs/result_formatter/result_formatter.h @@ -11,6 +11,6 @@ namespace NYq { void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet); -TString FormatSchema(const YandexQuery::Schema& schema); +TString FormatSchema(const YandexQuery::Schema& schema); } // namespace NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp index bebbf68ab3..7b10e85e29 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp @@ -294,7 +294,7 @@ Y_UNIT_TEST_SUITE(ResultFormatter) { UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected either " << expected1 << " or " << expected2 << ", got " << actual); } - + Y_UNIT_TEST(VariantTuple) { Ydb::ResultSet rs; { @@ -393,25 +393,25 @@ Y_UNIT_TEST_SUITE(ResultFormatter) { UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); } - Y_UNIT_TEST(FormatEmptySchema) { - YandexQuery::Schema s; - auto result = FormatSchema(s); - UNIT_ASSERT_VALUES_EQUAL(result, R"(["StructType";[]])"); - } - - Y_UNIT_TEST(FormatNonEmptySchema) { - YandexQuery::Schema s; - auto& c1 = *s.Addcolumn(); - c1.set_name("key"); - c1.mutable_type()->set_type_id(Ydb::Type::STRING); - - auto& c2 = *s.Addcolumn(); - c2.set_name("value"); - Ydb::Type t2; - t2.set_type_id(Ydb::Type::UINT64); - *c2.mutable_type()->mutable_optional_type()->mutable_item() = t2; - - auto result = FormatSchema(s); - UNIT_ASSERT_VALUES_EQUAL(result, R"(["StructType";[["key";["DataType";"String"]];["value";["OptionalType";["DataType";"Uint64"]]]]])"); - } + Y_UNIT_TEST(FormatEmptySchema) { + YandexQuery::Schema s; + auto result = FormatSchema(s); + UNIT_ASSERT_VALUES_EQUAL(result, R"(["StructType";[]])"); + } + + Y_UNIT_TEST(FormatNonEmptySchema) { + YandexQuery::Schema s; + auto& c1 = *s.Addcolumn(); + c1.set_name("key"); + c1.mutable_type()->set_type_id(Ydb::Type::STRING); + + auto& c2 = *s.Addcolumn(); + c2.set_name("value"); + Ydb::Type t2; + t2.set_type_id(Ydb::Type::UINT64); + *c2.mutable_type()->mutable_optional_type()->mutable_item() = t2; + + auto result = FormatSchema(s); + UNIT_ASSERT_VALUES_EQUAL(result, R"(["StructType";[["key";["DataType";"String"]];["value";["OptionalType";["DataType";"Uint64"]]]]])"); + } } diff --git a/ydb/core/yq/libs/signer/signer.cpp b/ydb/core/yq/libs/signer/signer.cpp index 79f1abc2e4..7938736fc6 100644 --- a/ydb/core/yq/libs/signer/signer.cpp +++ b/ydb/core/yq/libs/signer/signer.cpp @@ -1,31 +1,31 @@ -#include "signer.h" +#include "signer.h" #include <ydb/core/yq/libs/hmac/hmac.h> - -#include <util/generic/yexception.h> -#include <util/stream/file.h> -#include <util/string/builder.h> - + +#include <util/generic/yexception.h> +#include <util/stream/file.h> +#include <util/string/builder.h> + namespace NYq { - -TSigner::TSigner(const TString& hmacSecret) - : HmacSecret(hmacSecret) { -} - -TString TSigner::Sign(Type type, const TString& value) const { - TStringBuilder b; - b << static_cast<unsigned char>(type) << value; - return HmacSha1Base64(b, HmacSecret); -} - -void TSigner::Verify(Type type, const TString& value, const TString& signature) const { - const auto expectedSignature = Sign(type, value); - if (signature != expectedSignature) { - ythrow yexception() << "Incorrect signature for value: " << value << ", signature: " << signature << ", expected signature: " << expectedSignature; - } -} - -TSigner::TPtr CreateSignerFromFile(const TString& secretFile) { - return new TSigner(TFileInput(secretFile).ReadAll()); -} - -} + +TSigner::TSigner(const TString& hmacSecret) + : HmacSecret(hmacSecret) { +} + +TString TSigner::Sign(Type type, const TString& value) const { + TStringBuilder b; + b << static_cast<unsigned char>(type) << value; + return HmacSha1Base64(b, HmacSecret); +} + +void TSigner::Verify(Type type, const TString& value, const TString& signature) const { + const auto expectedSignature = Sign(type, value); + if (signature != expectedSignature) { + ythrow yexception() << "Incorrect signature for value: " << value << ", signature: " << signature << ", expected signature: " << expectedSignature; + } +} + +TSigner::TPtr CreateSignerFromFile(const TString& secretFile) { + return new TSigner(TFileInput(secretFile).ReadAll()); +} + +} diff --git a/ydb/core/yq/libs/signer/signer.h b/ydb/core/yq/libs/signer/signer.h index 9c44b9f6b2..8ac29edb28 100644 --- a/ydb/core/yq/libs/signer/signer.h +++ b/ydb/core/yq/libs/signer/signer.h @@ -1,31 +1,31 @@ -#pragma once - -#include <util/generic/ptr.h> -#include <util/generic/string.h> - +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/string.h> + namespace NYq { -// keep in sync with token accessor logic -class TSigner : public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TSigner>; - - enum Type { - UNKNOWN = 0, - SERVICE_ACCOUNT_ID = 1 - }; - -public: - explicit TSigner(const TString& hmacSecret); - TString SignAccountId(const TString& serviceAccountId) const { - return Sign(SERVICE_ACCOUNT_ID, serviceAccountId); - } - - TString Sign(Type type, const TString& value) const; - void Verify(Type type, const TString& value, const TString& signature) const; - -private: - const TString HmacSecret; -}; - -TSigner::TPtr CreateSignerFromFile(const TString& secretFile); -} +// keep in sync with token accessor logic +class TSigner : public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TSigner>; + + enum Type { + UNKNOWN = 0, + SERVICE_ACCOUNT_ID = 1 + }; + +public: + explicit TSigner(const TString& hmacSecret); + TString SignAccountId(const TString& serviceAccountId) const { + return Sign(SERVICE_ACCOUNT_ID, serviceAccountId); + } + + TString Sign(Type type, const TString& value) const; + void Verify(Type type, const TString& value, const TString& signature) const; + +private: + const TString HmacSecret; +}; + +TSigner::TPtr CreateSignerFromFile(const TString& secretFile); +} diff --git a/ydb/core/yq/libs/signer/ut/signer_ut.cpp b/ydb/core/yq/libs/signer/ut/signer_ut.cpp index 7841b5780a..82858921d6 100644 --- a/ydb/core/yq/libs/signer/ut/signer_ut.cpp +++ b/ydb/core/yq/libs/signer/ut/signer_ut.cpp @@ -1,14 +1,14 @@ #include <ydb/core/yq/libs/signer/signer.h> -#include <library/cpp/testing/unittest/registar.h> - +#include <library/cpp/testing/unittest/registar.h> + using namespace NYq; - -Y_UNIT_TEST_SUITE(Signer) { - Y_UNIT_TEST(Basic) { - TSigner s("abc"); - UNIT_ASSERT_VALUES_EQUAL("b+UDr24riPj5pb7ha2D2AERFvXw=", s.Sign(TSigner::SERVICE_ACCOUNT_ID, "xyz012345")); - s.Verify(TSigner::SERVICE_ACCOUNT_ID, "xyz012345", "b+UDr24riPj5pb7ha2D2AERFvXw="); - UNIT_ASSERT_EXCEPTION_CONTAINS(s.Verify(TSigner::UNKNOWN, "xyz012345", "b+UDr24riPj5pb7ha2D2AERFvXw="), yexception, "Incorrect signature for value: xyz012345, signature: b+UDr24riPj5pb7ha2D2AERFvXw=, expected signature: 8O3pYqWEJLvL0TvwQzbEn7dbqzs="); - UNIT_ASSERT_EXCEPTION_CONTAINS(s.Verify(TSigner::SERVICE_ACCOUNT_ID, "xyz01234", "b+UDr24riPj5pb7ha2D2AERFvXw="), yexception, "Incorrect signature for value: xyz01234, signature: b+UDr24riPj5pb7ha2D2AERFvXw=, expected signature: N637rqjAbKHpvUPeBXRvGkdVKew="); - } -} + +Y_UNIT_TEST_SUITE(Signer) { + Y_UNIT_TEST(Basic) { + TSigner s("abc"); + UNIT_ASSERT_VALUES_EQUAL("b+UDr24riPj5pb7ha2D2AERFvXw=", s.Sign(TSigner::SERVICE_ACCOUNT_ID, "xyz012345")); + s.Verify(TSigner::SERVICE_ACCOUNT_ID, "xyz012345", "b+UDr24riPj5pb7ha2D2AERFvXw="); + UNIT_ASSERT_EXCEPTION_CONTAINS(s.Verify(TSigner::UNKNOWN, "xyz012345", "b+UDr24riPj5pb7ha2D2AERFvXw="), yexception, "Incorrect signature for value: xyz012345, signature: b+UDr24riPj5pb7ha2D2AERFvXw=, expected signature: 8O3pYqWEJLvL0TvwQzbEn7dbqzs="); + UNIT_ASSERT_EXCEPTION_CONTAINS(s.Verify(TSigner::SERVICE_ACCOUNT_ID, "xyz01234", "b+UDr24riPj5pb7ha2D2AERFvXw="), yexception, "Incorrect signature for value: xyz01234, signature: b+UDr24riPj5pb7ha2D2AERFvXw=, expected signature: N637rqjAbKHpvUPeBXRvGkdVKew="); + } +} diff --git a/ydb/core/yq/libs/signer/ut/ya.make b/ydb/core/yq/libs/signer/ut/ya.make index 2ead7125c4..e293f494c9 100644 --- a/ydb/core/yq/libs/signer/ut/ya.make +++ b/ydb/core/yq/libs/signer/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST_FOR(ydb/core/yq/libs/signer) - + OWNER(g:yq) - -SRCS( - signer_ut.cpp -) - -PEERDIR( + +SRCS( + signer_ut.cpp +) + +PEERDIR( ydb/core/yq/libs/signer -) - -END() +) + +END() diff --git a/ydb/core/yq/libs/signer/ya.make b/ydb/core/yq/libs/signer/ya.make index 9faf39df7b..dcceb1d861 100644 --- a/ydb/core/yq/libs/signer/ya.make +++ b/ydb/core/yq/libs/signer/ya.make @@ -1,17 +1,17 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - signer.cpp -) - -PEERDIR( + +LIBRARY() + +SRCS( + signer.cpp +) + +PEERDIR( ydb/core/yq/libs/hmac -) - -END() - -RECURSE_FOR_TESTS( - ut -) +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make index ccc88a55b6..44ba0ce687 100644 --- a/ydb/core/yq/libs/ya.make +++ b/ydb/core/yq/libs/ya.make @@ -21,7 +21,7 @@ RECURSE( mock pretty_printers private_client - read_rule + read_rule result_formatter shared_resources signer diff --git a/ydb/library/aclib/protos/aclib.proto b/ydb/library/aclib/protos/aclib.proto index 4978977d54..f565428299 100644 --- a/ydb/library/aclib/protos/aclib.proto +++ b/ydb/library/aclib/protos/aclib.proto @@ -1,7 +1,7 @@ package NACLibProto; option go_package = "a.yandex-team.ru/ydb/library/aclib/protos;aclibpb"; - + message TACE { optional uint32 AccessType = 1; optional uint32 AccessRight = 2; diff --git a/ydb/library/aclib/protos/ya.make b/ydb/library/aclib/protos/ya.make index 59135589cc..6b8165eb4b 100644 --- a/ydb/library/aclib/protos/ya.make +++ b/ydb/library/aclib/protos/ya.make @@ -5,7 +5,7 @@ OWNER( g:kikimr ) -INCLUDE_TAGS(GO_PROTO) +INCLUDE_TAGS(GO_PROTO) SRCS( aclib.proto diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index 53d7722dee..c0e9972fd1 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -422,7 +422,7 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed } } -void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr) { +void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr) { switch (type->GetKind()) { case TType::EKind::Void: case TType::EKind::EmptyList: @@ -465,9 +465,9 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK auto structType = static_cast<TStructType*>(type); res.MutableStruct()->Reserve(structType->GetMembersCount()); for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; - auto memberType = structType->GetMemberType(memberIndex); - ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.MutableStruct()->Add()); + auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; + auto memberType = structType->GetMemberType(memberIndex); + ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.MutableStruct()->Add()); } break; @@ -518,7 +518,7 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK } } -void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) { +void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) { switch (type->GetKind()) { case TType::EKind::Void: case TType::EKind::EmptyList: @@ -573,9 +573,9 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Yd auto structType = static_cast<TStructType*>(type); res.mutable_items()->Reserve(structType->GetMembersCount()); for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { - auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; - auto memberType = structType->GetMemberType(memberIndex); - ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.mutable_items()->Add()); + auto memberIndex = (!columnOrder || columnOrder->empty()) ? index : (*columnOrder)[index]; + auto memberType = structType->GetMemberType(memberIndex); + ExportValueToProtoImpl(memberType, value.GetElement(memberIndex), *res.mutable_items()->Add()); } break; @@ -752,8 +752,8 @@ void ExportTypeToProto(TType* type, Ydb::Type& res) { ExportTypeToProtoImpl(type, res); } -void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder) { - ExportValueToProtoImpl(type, value, res, columnOrder); +void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder) { + ExportValueToProtoImpl(type, value, res, columnOrder); } template <typename ValueType> class TBufferArray { @@ -1179,8 +1179,8 @@ void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res) { ExportTypeToProtoImpl(type, res); } -void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder) { - ExportValueToProtoImpl(type, value, res, columnOrder); +void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder) { + ExportValueToProtoImpl(type, value, res, columnOrder); } diff --git a/ydb/library/mkql_proto/mkql_proto.h b/ydb/library/mkql_proto/mkql_proto.h index 716f8530d3..57b913552d 100644 --- a/ydb/library/mkql_proto/mkql_proto.h +++ b/ydb/library/mkql_proto/mkql_proto.h @@ -10,12 +10,12 @@ namespace NKikimr::NMiniKQL { class THolderFactory; void ExportTypeToProto(TType* type, NKikimrMiniKQL::TType& res); -void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr); +void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr); void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output); void ExportTypeToProto(TType* type, Ydb::Type& res); -void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr); +void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr); TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env); diff --git a/ydb/library/mkql_proto/mkql_proto_ut.cpp b/ydb/library/mkql_proto/mkql_proto_ut.cpp index fdda085f18..1b8adfc94b 100644 --- a/ydb/library/mkql_proto/mkql_proto_ut.cpp +++ b/ydb/library/mkql_proto/mkql_proto_ut.cpp @@ -506,71 +506,71 @@ Variant { "}\n"); } - Y_UNIT_TEST(TestExportStructEmptyColumnOrder) { - const TVector<ui32> emptyColumnOrder; - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - // struct fields are in alpha order - "Struct {\n" - " Int32: 42\n" - "}\n" - "Struct {\n" - " Bytes: \"abc\"\n" - "}\n", - &emptyColumnOrder); - } - - Y_UNIT_TEST(TestExportStructWithColumnOrder) { - const TVector<ui32> columnOrder = {1, 0}; - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - std::vector<std::pair<std::string_view, TRuntimeNode>> items; - items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto pgmReturn = pgmBuilder.NewStruct(items); - return pgmReturn; - }, - "Struct {\n" - " Bytes: \"abc\"\n" - "}\n" - "Struct {\n" - " Int32: 42\n" - "}\n", - &columnOrder); - } - - Y_UNIT_TEST(TestExportStructColumnOrderAffectsTopLevelOnly) { - const TVector<ui32> columnOrder = {1, 0}; - TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { - std::vector<std::pair<std::string_view, TRuntimeNode>> items1; - items1.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); - items1.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); - auto struct1 = pgmBuilder.NewStruct(items1); - - std::vector<std::pair<std::string_view, TRuntimeNode>> items2; - items2.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(777) }); - items2.push_back({ "y", struct1 }); - auto pgmReturn = pgmBuilder.NewStruct(items2); - return pgmReturn; - }, - "Struct {\n" - " Struct {\n" - " Int32: 42\n" - " }\n" - " Struct {\n" - " Bytes: \"abc\"\n" - " }\n" - "}\n" - "Struct {\n" - " Int32: 777\n" - "}\n", - &columnOrder); - } - + Y_UNIT_TEST(TestExportStructEmptyColumnOrder) { + const TVector<ui32> emptyColumnOrder; + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + std::vector<std::pair<std::string_view, TRuntimeNode>> items; + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + // struct fields are in alpha order + "Struct {\n" + " Int32: 42\n" + "}\n" + "Struct {\n" + " Bytes: \"abc\"\n" + "}\n", + &emptyColumnOrder); + } + + Y_UNIT_TEST(TestExportStructWithColumnOrder) { + const TVector<ui32> columnOrder = {1, 0}; + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + std::vector<std::pair<std::string_view, TRuntimeNode>> items; + items.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto pgmReturn = pgmBuilder.NewStruct(items); + return pgmReturn; + }, + "Struct {\n" + " Bytes: \"abc\"\n" + "}\n" + "Struct {\n" + " Int32: 42\n" + "}\n", + &columnOrder); + } + + Y_UNIT_TEST(TestExportStructColumnOrderAffectsTopLevelOnly) { + const TVector<ui32> columnOrder = {1, 0}; + TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { + std::vector<std::pair<std::string_view, TRuntimeNode>> items1; + items1.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(42) }); + items1.push_back({ "y", pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("abc") }); + auto struct1 = pgmBuilder.NewStruct(items1); + + std::vector<std::pair<std::string_view, TRuntimeNode>> items2; + items2.push_back({ "x", pgmBuilder.NewDataLiteral<i32>(777) }); + items2.push_back({ "y", struct1 }); + auto pgmReturn = pgmBuilder.NewStruct(items2); + return pgmReturn; + }, + "Struct {\n" + " Struct {\n" + " Int32: 42\n" + " }\n" + " Struct {\n" + " Bytes: \"abc\"\n" + " }\n" + "}\n" + "Struct {\n" + " Int32: 777\n" + "}\n", + &columnOrder); + } + Y_UNIT_TEST(TestExportDict) { TestExportValue<NKikimrMiniKQL::TValue>([](TProgramBuilder& pgmBuilder) { auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id), diff --git a/ydb/library/mkql_proto/ut/helpers/helpers.h b/ydb/library/mkql_proto/ut/helpers/helpers.h index e5094e8085..dd9c1f8513 100644 --- a/ydb/library/mkql_proto/ut/helpers/helpers.h +++ b/ydb/library/mkql_proto/ut/helpers/helpers.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/mkql_program_builder.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> @@ -31,7 +31,7 @@ void TestExportType(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> set } template <typename TProtoValue> -void TestExportValue(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> setup, const TString& expectedString, const TVector<ui32>* columnOrder = nullptr) { +void TestExportValue(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> setup, const TString& expectedString, const TVector<ui32>* columnOrder = nullptr) { auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); auto randomProvider = CreateDeterministicRandomProvider(1); auto timeProvider = CreateDeterministicTimeProvider(1); @@ -49,7 +49,7 @@ void TestExportValue(std::function<TRuntimeNode(TProgramBuilder& pgmBuilder)> se auto pattern = MakeComputationPattern(explorer, pgmReturn, {}, opts); auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); TProtoValue res; - ExportValueToProto(pgmReturn.GetStaticType(), graph->GetValue(), res, columnOrder); + ExportValueToProto(pgmReturn.GetStaticType(), graph->GetValue(), res, columnOrder); using ::google::protobuf::TextFormat; TString str; TextFormat::PrintToString(res, &str); diff --git a/ydb/library/yql/ast/yql_ast_ut.cpp b/ydb/library/yql/ast/yql_ast_ut.cpp index 23e140653d..7124ac94ae 100644 --- a/ydb/library/yql/ast/yql_ast_ut.cpp +++ b/ydb/library/yql/ast/yql_ast_ut.cpp @@ -349,15 +349,15 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { "string@@)"; ParseAndPrint(s4, s4); - TString s5 = "(@@\n" - "one@\n" - "two@@@@\n" - "four@@@@@@@@\n" - "@@@@two\n" - "@one\n" - "@@)"; - - TAstParseResult ast = ParseAst(s5); + TString s5 = "(@@\n" + "one@\n" + "two@@@@\n" + "four@@@@@@@@\n" + "@@@@two\n" + "@one\n" + "@@)"; + + TAstParseResult ast = ParseAst(s5); UNIT_ASSERT(ast.IsOk()); UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); @@ -372,9 +372,9 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { "@@two\n" "@one\n"; UNIT_ASSERT_STRINGS_EQUAL(atom->GetContent(), expected); - - TString printResult = ast.Root->ToString(); - UNIT_ASSERT_STRINGS_EQUAL(s5, printResult); + + TString printResult = ast.Root->ToString(); + UNIT_ASSERT_STRINGS_EQUAL(s5, printResult); } Y_UNIT_TEST(UnicodePrettyPrint) { diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 82f6c5812a..87278567b2 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -885,39 +885,39 @@ namespace { return {lambda}; } - bool CompileSetPackageVersion(const TAstNode& node, TContext& ctx) { - if (node.GetChildrenCount() != 3) { - ctx.AddError(node, "Expected list of size 3"); - return false; - } - - const auto name = node.GetChild(1); - if (name->IsAtom() || name->GetChildrenCount() != 2 || !name->GetChild(0)->IsAtom() || !name->GetChild(1)->IsAtom() || + bool CompileSetPackageVersion(const TAstNode& node, TContext& ctx) { + if (node.GetChildrenCount() != 3) { + ctx.AddError(node, "Expected list of size 3"); + return false; + } + + const auto name = node.GetChild(1); + if (name->IsAtom() || name->GetChildrenCount() != 2 || !name->GetChild(0)->IsAtom() || !name->GetChild(1)->IsAtom() || name->GetChild(0)->GetContent() != TStringBuf("quote")) { - ctx.AddError(*name, "Expected quoted atom for package name"); - return false; - } - - const auto versionNode = node.GetChild(2); - if (versionNode->IsAtom() || versionNode->GetChildrenCount() != 2 || !versionNode->GetChild(0)->IsAtom() || !versionNode->GetChild(1)->IsAtom() || + ctx.AddError(*name, "Expected quoted atom for package name"); + return false; + } + + const auto versionNode = node.GetChild(2); + if (versionNode->IsAtom() || versionNode->GetChildrenCount() != 2 || !versionNode->GetChild(0)->IsAtom() || !versionNode->GetChild(1)->IsAtom() || versionNode->GetChild(0)->GetContent() != TStringBuf("quote")) { ctx.AddError(*versionNode, "Expected quoted atom for package version"); - return false; - } - - ui32 version = 0; - if (!versionNode->GetChild(1)->IsAtom() || !TryFromString(versionNode->GetChild(1)->GetContent(), version)) { - ctx.AddError(*versionNode, TString("Expected package version as number, bad content ") + versionNode->GetChild(1)->GetContent()); - return false; - } - - if (ctx.ModuleResolver && !ctx.ModuleResolver->SetPackageDefaultVersion(TString(name->GetChild(1)->GetContent()), version)) { - ctx.AddError(*versionNode, TStringBuilder() << "Unable to specify version " << version << " for package " << name->GetChild(1)->GetContent()); - return false; - } - return true; - } - + return false; + } + + ui32 version = 0; + if (!versionNode->GetChild(1)->IsAtom() || !TryFromString(versionNode->GetChild(1)->GetContent(), version)) { + ctx.AddError(*versionNode, TString("Expected package version as number, bad content ") + versionNode->GetChild(1)->GetContent()); + return false; + } + + if (ctx.ModuleResolver && !ctx.ModuleResolver->SetPackageDefaultVersion(TString(name->GetChild(1)->GetContent()), version)) { + ctx.AddError(*versionNode, TStringBuilder() << "Unable to specify version " << version << " for package " << name->GetChild(1)->GetContent()); + return false; + } + return true; + } + TExprNode::TPtr CompileBind(const TAstNode& node, TContext& ctx) { if (node.GetChildrenCount() != 3) { ctx.AddError(node, "Expected list of size 3"); @@ -1177,7 +1177,7 @@ namespace { if (!CompileLibraryDef(*node, ctx)) return {}; } else if (firstChild->GetContent() == TStringBuf("set_package_version")) { - if (!CompileSetPackageVersion(*node, ctx)) + if (!CompileSetPackageVersion(*node, ctx)) return {}; } } @@ -1240,14 +1240,14 @@ namespace { continue; } else if (firstChild->GetContent() == TStringBuf("set_package_version")) { - if (!topLevel) { - ctx.AddError(*firstChild, "set_package_version statements are only allowed on top level block"); + if (!topLevel) { + ctx.AddError(*firstChild, "set_package_version statements are only allowed on top level block"); return {}; - } - - continue; + } + + continue; } else { - ctx.AddError(*firstChild, ToString("expected either let, return or import, but have ") + firstChild->GetContent()); + ctx.AddError(*firstChild, ToString("expected either let, return or import, but have ") + firstChild->GetContent()); return {}; } } diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index d6eb544f9a..613b2dccc3 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -2246,7 +2246,7 @@ struct TExprContext : private TNonCopyable { TSingletonTypeCache SingletonTypeCache; std::unordered_set<const TTypeAnnotationNode*, TTypeAnnotationNode::THash, TTypeAnnotationNode::TEqual> TypeSet; std::unordered_set<const TConstraintNode*, TConstraintNode::THash, TConstraintNode::TEqual> ConstraintSet; - std::unordered_map<const TTypeAnnotationNode*, TExprNode::TPtr> TypeAsNodeCache; + std::unordered_map<const TTypeAnnotationNode*, TExprNode::TPtr> TypeAsNodeCache; std::unordered_set<TStringBuf, THash<TStringBuf>> DisabledConstraints; ui64 NextUniqueId = 0; @@ -2264,7 +2264,7 @@ struct TExprContext : private TNonCopyable { bool Frozen; - explicit TExprContext(ui64 nextUniqueId = 0ULL); + explicit TExprContext(ui64 nextUniqueId = 0ULL); ~TExprContext(); ui64 AllocateNextUniqueId() { diff --git a/ydb/library/yql/ast/yql_expr_builder.cpp b/ydb/library/yql/ast/yql_expr_builder.cpp index 4b22308b99..7bd5e2e81a 100644 --- a/ydb/library/yql/ast/yql_expr_builder.cpp +++ b/ydb/library/yql/ast/yql_expr_builder.cpp @@ -132,7 +132,7 @@ TExprNodeBuilder& TExprNodeBuilder::Add(ui32 index, const TExprNode::TPtr& child Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << ", actual: " << index); - Y_ENSURE(child, "child should not be nullptr"); + Y_ENSURE(child, "child should not be nullptr"); Container->Children_.push_back(child); return *this; } @@ -142,7 +142,7 @@ TExprNodeBuilder& TExprNodeBuilder::Add(ui32 index, TExprNode::TPtr&& child) { Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << ", actual: " << index); - Y_ENSURE(child, "child should not be nullptr"); + Y_ENSURE(child, "child should not be nullptr"); Container->Children_.push_back(std::move(child)); return *this; } diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp index a32a6c3dab..f3eb4a5bd9 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp @@ -632,21 +632,21 @@ TExprNode::TPtr ExpandExtract(const TExprNode::TPtr& node, TExprContext& ctx) { .Seal().Build(); } -std::vector<TExprNode::TListType> GroupNodeChildrenByType(const TExprNode::TPtr& node) { - std::vector<TExprNode::TListType> groups; - std::map<const TTypeAnnotationNode*, ui32> typeToGroup; - for (ui32 i = 0; i < node->ChildrenSize(); ++i) { - auto child = node->Child(i); - auto groupIndex = typeToGroup.emplace(child->GetTypeAnn(), groups.size()).first->second; - if (groupIndex >= groups.size()) { - YQL_ENSURE(groupIndex == groups.size()); - groups.resize(groupIndex + 1); - } - groups[groupIndex].push_back(child); - } - return groups; -} - +std::vector<TExprNode::TListType> GroupNodeChildrenByType(const TExprNode::TPtr& node) { + std::vector<TExprNode::TListType> groups; + std::map<const TTypeAnnotationNode*, ui32> typeToGroup; + for (ui32 i = 0; i < node->ChildrenSize(); ++i) { + auto child = node->Child(i); + auto groupIndex = typeToGroup.emplace(child->GetTypeAnn(), groups.size()).first->second; + if (groupIndex >= groups.size()) { + YQL_ENSURE(groupIndex == groups.size()); + groups.resize(groupIndex + 1); + } + groups[groupIndex].push_back(child); + } + return groups; +} + template <bool Ordered> TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); @@ -659,7 +659,7 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T auto remapList = [&ctx, &nulls, resultStructType](TExprNode::TPtr input, const TTypeAnnotationNode* inputType) -> TExprNode::TPtr { auto pos = input->Pos(); auto arg = ctx.NewArgument(pos, "item"); - auto inputStructType = inputType->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + auto inputStructType = inputType->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); TExprNode::TListType bodyItems; ui32 resultIndex = 0; for (auto& item : resultStructType->GetItems()) { @@ -724,8 +724,8 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T } else { auto pos = group[0]->Pos(); remapped = remapList(ctx.NewCallable(pos, Ordered ? "Merge" : "Extend", std::move(group)), typeAnn); - } - + } + if (!remapped) { return node; } diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp index b74abc1c34..e0aae6ab80 100644 --- a/ydb/library/yql/core/facade/yql_facade.cpp +++ b/ydb/library/yql/core/facade/yql_facade.cpp @@ -51,8 +51,8 @@ public: : Storage_(storage) {} - TString Load(const TString& url, const TString& token) override { - auto file = Storage_->PutUrl(url, token); + TString Load(const TString& url, const TString& token) override { + auto file = Storage_->PutUrl(url, token); return TFileInput(file->GetPath()).ReadAll(); } @@ -86,26 +86,26 @@ TProgram::TStatus SyncExecution( return status; } -std::function<TMaybe<TString>(const TString&)> BuildCompositeTokenResolver(TVector<std::function<TMaybe<TString>(const TString&)>>&& children) { - if (children.empty()) { - return {}; - } - - if (children.size() == 1) { - return std::move(children[0]); - } - - return[children = std::move(children)](const TString& url)->TMaybe<TString> { - for (auto& c : children) { - if (auto r = c(url)) { - return r; - } - } - - return Nothing(); - }; -} - +std::function<TMaybe<TString>(const TString&)> BuildCompositeTokenResolver(TVector<std::function<TMaybe<TString>(const TString&)>>&& children) { + if (children.empty()) { + return {}; + } + + if (children.size() == 1) { + return std::move(children[0]); + } + + return[children = std::move(children)](const TString& url)->TMaybe<TString> { + for (auto& c : children) { + if (auto r = c(url)) { + return r; + } + } + + return Nothing(); + }; +} + } // namspace /////////////////////////////////////////////////////////////////////////////// @@ -135,12 +135,12 @@ void TProgramFactory::EnableRangeComputeFor() { EnableRangeComputeFor_ = true; } -void TProgramFactory::AddUserDataTable(const TUserDataTable& userDataTable) { - for (auto& p : userDataTable) { - if (!UserDataTable_.emplace(p).second) { - ythrow yexception() << "UserDataTable already has user data block with key " << p.first; - } - } +void TProgramFactory::AddUserDataTable(const TUserDataTable& userDataTable) { + for (auto& p : userDataTable) { + if (!UserDataTable_.emplace(p).second) { + ythrow yexception() << "UserDataTable already has user data block with key " << p.first; + } + } } void TProgramFactory::AddCredentialsTable(TCredentialTablePtr credentialTable) { @@ -163,11 +163,11 @@ void TProgramFactory::SetUdfResolver(IUdfResolver::TPtr udfResolver) { UdfResolver_ = udfResolver; } -void TProgramFactory::SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet) { - UdfIndex_ = std::move(udfIndex); - UdfIndexPackageSet_ = std::move(udfIndexPackageSet); -} - +void TProgramFactory::SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet) { + UdfIndex_ = std::move(udfIndex); + UdfIndexPackageSet_ = std::move(udfIndexPackageSet); +} + void TProgramFactory::SetFileStorage(TFileStoragePtr fileStorage) { FileStorage_ = fileStorage; } @@ -189,15 +189,15 @@ TProgramPtr TProgramFactory::Create( CreateDeterministicRandomProvider(1) : CreateDefaultRandomProvider(); auto timeProvider = UseRepeatableRandomAndTimeProviders_ ? CreateDeterministicTimeProvider(10000000) : CreateDefaultTimeProvider(); - - TUdfIndex::TPtr udfIndex = UdfIndex_ ? UdfIndex_->Clone() : nullptr; - TUdfIndexPackageSet::TPtr udfIndexPackageSet = UdfIndexPackageSet_ ? UdfIndexPackageSet_->Clone() : nullptr; - IModuleResolver::TPtr moduleResolver = Modules_ ? Modules_->CreateMutableChild() : nullptr; - auto udfResolver = udfIndex ? NCommon::CreateUdfResolverWithIndex(udfIndex, UdfResolver_, FileStorage_) : UdfResolver_; - - // make UserDataTable_ copy here + + TUdfIndex::TPtr udfIndex = UdfIndex_ ? UdfIndex_->Clone() : nullptr; + TUdfIndexPackageSet::TPtr udfIndexPackageSet = UdfIndexPackageSet_ ? UdfIndexPackageSet_->Clone() : nullptr; + IModuleResolver::TPtr moduleResolver = Modules_ ? Modules_->CreateMutableChild() : nullptr; + auto udfResolver = udfIndex ? NCommon::CreateUdfResolverWithIndex(udfIndex, UdfResolver_, FileStorage_) : UdfResolver_; + + // make UserDataTable_ copy here return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_, - UserDataTable_, CredentialTables_, UserCredentials_, moduleResolver, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, + UserDataTable_, CredentialTables_, UserCredentials_, moduleResolver, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_); } @@ -210,13 +210,13 @@ TProgram::TProgram( const TIntrusivePtr<ITimeProvider> timeProvider, ui64 nextUniqueId, const TVector<TDataProviderInitializer>& dataProvidersInit, - const TUserDataTable& userDataTable, + const TUserDataTable& userDataTable, const TVector<TCredentialTablePtr>& credentialTables, const TUserCredentials& userCredentials, const IModuleResolver::TPtr& modules, const IUdfResolver::TPtr& udfResolver, - const TUdfIndex::TPtr& udfIndex, - const TUdfIndexPackageSet::TPtr& udfIndexPackageSet, + const TUdfIndex::TPtr& udfIndex, + const TUdfIndexPackageSet::TPtr& udfIndexPackageSet, const TFileStoragePtr& fileStorage, const TGatewaysConfig* gatewaysConfig, const TString& filename, @@ -233,10 +233,10 @@ TProgram::TProgram( , CredentialTables_(credentialTables) , UserCredentials_(userCredentials) , UdfResolver_(udfResolver) - , UdfIndex_(udfIndex) - , UdfIndexPackageSet_(udfIndexPackageSet) + , UdfIndex_(udfIndex) + , UdfIndexPackageSet_(udfIndexPackageSet) , FileStorage_(fileStorage) - , UserDataStorage_(MakeIntrusive<TUserDataStorage>(fileStorage, userDataTable, udfResolver, udfIndex)) + , UserDataStorage_(MakeIntrusive<TUserDataStorage>(fileStorage, userDataTable, udfResolver, udfIndex)) , GatewaysConfig_(gatewaysConfig) , Filename_(filename) , SourceCode_(sourceCode) @@ -251,11 +251,11 @@ TProgram::TProgram( , EnableRangeComputeFor_(enableRangeComputeFor) { if (SessionId_.empty()) { - SessionId_ = CreateGuidAsString(); - } - + SessionId_ = CreateGuidAsString(); + } + if (auto modules = dynamic_cast<TModuleResolver*>(Modules_.get())) { - modules->AttachUserData(UserDataStorage_); + modules->AttachUserData(UserDataStorage_); modules->SetUrlLoader(new TUrlLoader(FileStorage_)); } OperationOptions_.Runner = runner; @@ -277,31 +277,31 @@ void TProgram::ConfigureYsonResultFormat(NYson::EYsonFormat format) { } void TProgram::SetValidateOptions(NUdf::EValidateMode validateMode) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); ValidateMode_ = validateMode; } -void TProgram::SetDisableNativeUdfSupport(bool disable) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); - DisableNativeUdfSupport_ = disable; -} - +void TProgram::SetDisableNativeUdfSupport(bool disable) { + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + DisableNativeUdfSupport_ = disable; +} + void TProgram::SetUseTableMetaFromGraph(bool use) { Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); UseTableMetaFromGraph_ = use; } -TTypeAnnotationContextPtr TProgram::GetAnnotationContext() const { - Y_ENSURE(TypeCtx_, "TypeCtx_ is not created"); - return TypeCtx_; -} - -TTypeAnnotationContextPtr TProgram::ProvideAnnotationContext(const TString& username) { +TTypeAnnotationContextPtr TProgram::GetAnnotationContext() const { + Y_ENSURE(TypeCtx_, "TypeCtx_ is not created"); + return TypeCtx_; +} + +TTypeAnnotationContextPtr TProgram::ProvideAnnotationContext(const TString& username) { if (!TypeCtx_) { - TypeCtx_ = BuildTypeAnnotationContext(username); + TypeCtx_ = BuildTypeAnnotationContext(username); TypeCtx_->OperationOptions = OperationOptions_; TypeCtx_->ValidateMode = ValidateMode_; - TypeCtx_->DisableNativeUdfSupport = DisableNativeUdfSupport_; + TypeCtx_->DisableNativeUdfSupport = DisableNativeUdfSupport_; TypeCtx_->UseTableMetaFromGraph = UseTableMetaFromGraph_; } @@ -316,14 +316,14 @@ IPlanBuilder& TProgram::GetPlanBuilder() { return *PlanBuilder_; } -void TProgram::SetParametersYson(const TString& parameters) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); - auto node = NYT::NodeFromYsonString(parameters); +void TProgram::SetParametersYson(const TString& parameters) { + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + auto node = NYT::NodeFromYsonString(parameters); YQL_ENSURE(node.IsMap()); for (const auto& x : node.AsMap()) { YQL_ENSURE(x.second.IsMap()); YQL_ENSURE(x.second.HasKey("Data")); - YQL_ENSURE(x.second.Size() == 1); + YQL_ENSURE(x.second.Size() == 1); } OperationOptions_.ParametersYson = node; @@ -332,18 +332,18 @@ void TProgram::SetParametersYson(const TString& parameters) { } } -bool TProgram::ExtractQueryParametersMetadata() { - auto& types = *GetAnnotationContext(); - NYT::TNode params = NYT::TNode::CreateMap(); - Y_ENSURE(ExprCtx_); - if (!ExtractParametersMetaAsYson(ExprRoot_, types, *ExprCtx_, params)) { - return false; - } - - ExtractedQueryParametersMetadataYson_ = NYT::NodeToYsonString(params, ResultFormat_); - return true; -} - +bool TProgram::ExtractQueryParametersMetadata() { + auto& types = *GetAnnotationContext(); + NYT::TNode params = NYT::TNode::CreateMap(); + Y_ENSURE(ExprCtx_); + if (!ExtractParametersMetaAsYson(ExprRoot_, types, *ExprCtx_, params)) { + return false; + } + + ExtractedQueryParametersMetadataYson_ = NYT::NodeToYsonString(params, ResultFormat_); + return true; +} + bool TProgram::FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules) { if (!astRes.Issues.Empty()) { if (!ExprCtx_) { @@ -374,19 +374,19 @@ bool TProgram::FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRul return true; } -TString TProgram::GetSessionId() const { - with_lock(SessionIdLock_) { - return SessionId_; - } -} - -TString TProgram::TakeSessionId() { - // post-condition: SessionId_ will be empty - with_lock(SessionIdLock_) { - return std::move(SessionId_); - } -} - +TString TProgram::GetSessionId() const { + with_lock(SessionIdLock_) { + return SessionId_; + } +} + +TString TProgram::TakeSessionId() { + // post-condition: SessionId_ will be empty + with_lock(SessionIdLock_) { + return std::move(SessionId_); + } +} + bool TProgram::ParseYql() { YQL_PROFILE_FUNC(TRACE); YQL_ENSURE(SourceSyntax_ == ESourceSyntax::Unknown); @@ -417,18 +417,18 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) return FillParseResult(SqlToYql(SourceCode_, settings, &warningRules), &warningRules); } -bool TProgram::Compile(const TString& username) { +bool TProgram::Compile(const TString& username) { YQL_PROFILE_FUNC(TRACE); Y_ENSURE(AstRoot_, "Program not parsed yet"); if (!ExprCtx_) { ExprCtx_.Reset(new TExprContext(NextUniqueId_)); } - - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_)) { - return false; - } - TypeCtx_->IsReadOnly = true; + + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_)) { + return false; + } + TypeCtx_->IsReadOnly = true; if (Modules_.get()) { auto libs = UserDataStorage_->GetLibraries(); for (auto lib : libs) { @@ -437,7 +437,7 @@ bool TProgram::Compile(const TString& username) { } } } - + if (!CompileExpr(*AstRoot_, ExprRoot_, *ExprCtx_, Modules_.get(), 0, SyntaxVersion_)) { return false; } @@ -551,7 +551,7 @@ TProgram::TStatus TProgram::Validate(const TString& username, IOutputStream* exp } TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutputStream* exprOut, bool withTypes) { - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } TypeCtx_->IsReadOnly = true; @@ -585,7 +585,7 @@ TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutput } return openSession.Apply([this](const TFuture<void>& f) { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { f.GetValue(); } catch (const std::exception& e) { @@ -616,7 +616,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsync( IOutputStream* exprOut, bool withTypes) { - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } TypeCtx_->IsReadOnly = true; @@ -657,7 +657,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsync( return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); return openSession.Apply([this](const TFuture<void>& f) { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { f.GetValue(); } catch (const std::exception& e) { @@ -680,7 +680,7 @@ TProgram::TStatus TProgram::OptimizeWithConfig( TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig( const TString& username, const IPipelineConfigurator& pipelineConf) { - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } TypeCtx_->IsReadOnly = true; @@ -726,7 +726,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig( return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); return openSession.Apply([this](const TFuture<void>& f) { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { f.GetValue(); } catch (const std::exception& e) { @@ -757,7 +757,7 @@ TProgram::TFutureStatus TProgram::RunAsync( IOutputStream* exprOut, bool withTypes) { - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } TypeCtx_->IsReadOnly = false; @@ -808,7 +808,7 @@ TProgram::TFutureStatus TProgram::RunAsync( SavedExprRoot_ = ExprRoot_; return openSession.Apply([this](const TFuture<void>& f) { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { f.GetValue(); } catch (const std::exception& e) { @@ -831,7 +831,7 @@ TProgram::TStatus TProgram::RunWithConfig( TProgram::TFutureStatus TProgram::RunAsyncWithConfig( const TString& username, const IPipelineConfigurator& pipelineConf) { - if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { + if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } TypeCtx_->IsReadOnly = false; @@ -880,7 +880,7 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig( SavedExprRoot_ = ExprRoot_; return openSession.Apply([this](const TFuture<void>& f) { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { f.GetValue(); } catch (const std::exception& e) { @@ -1190,7 +1190,7 @@ TMaybe<TString> TProgram::GetDiscoveredData() { } TProgram::TFutureStatus TProgram::ContinueAsync() { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); return AsyncTransformWithFallback(true); } @@ -1215,9 +1215,9 @@ void TProgram::CleanupLastSession() { } void TProgram::CloseLastSession() { - YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); + YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); - TString sessionId = TakeSessionId(); + TString sessionId = TakeSessionId(); if (sessionId.empty()) { return; } @@ -1244,35 +1244,35 @@ TString TProgram::ResultsAsString() const { return resultOut.Str(); } -TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& username) { +TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& username) { auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); - - typeAnnotationContext->UserDataStorage = UserDataStorage_; + + typeAnnotationContext->UserDataStorage = UserDataStorage_; typeAnnotationContext->Credentials = CredentialTables_; typeAnnotationContext->UserCredentials = UserCredentials_; typeAnnotationContext->Modules = Modules_; typeAnnotationContext->UdfResolver = UdfResolver_; - typeAnnotationContext->UdfIndex = UdfIndex_; - typeAnnotationContext->UdfIndexPackageSet = UdfIndexPackageSet_; + typeAnnotationContext->UdfIndex = UdfIndex_; + typeAnnotationContext->UdfIndexPackageSet = UdfIndexPackageSet_; typeAnnotationContext->RandomProvider = RandomProvider_; typeAnnotationContext->TimeProvider = TimeProvider_; if (DiagnosticFormat_) { typeAnnotationContext->Diagnostics = true; } - if (UdfIndex_ && UdfIndexPackageSet_) { - // setup default versions at the beginning - // could be overridden by pragma later - UdfIndexPackageSet_->AddResourcesTo(UdfIndex_); - } - + if (UdfIndex_ && UdfIndexPackageSet_) { + // setup default versions at the beginning + // could be overridden by pragma later + UdfIndexPackageSet_->AddResourcesTo(UdfIndex_); + } + PlanBuilder_ = CreatePlanBuilder(*typeAnnotationContext); THashSet<TString> providerNames; TVector<TString> fullResultDataSinks; - TVector<std::function<TMaybe<TString>(const TString& url)>> tokenResolvers; + TVector<std::function<TMaybe<TString>(const TString& url)>> tokenResolvers; for (const auto& dpi : DataProvidersInit_) { auto dp = dpi( - username, + username, SessionId_, GatewaysConfig_, FunctionRegistry_, @@ -1291,10 +1291,10 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us if (dp.Sink) { typeAnnotationContext->AddDataSink(dp.Names, dp.Sink); } - - if (dp.TokenResolver) { - tokenResolvers.push_back(dp.TokenResolver); - } + + if (dp.TokenResolver) { + tokenResolvers.push_back(dp.TokenResolver); + } if (dp.SupportFullResultDataSink) { fullResultDataSinks.insert(fullResultDataSinks.end(), dp.Names.begin(), dp.Names.end()); @@ -1343,7 +1343,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us typeAnnotationContext->AddDataSource(ConfigProviderName, configProvider); } - typeAnnotationContext->UserDataStorage->SetTokenResolver(BuildCompositeTokenResolver(std::move(tokenResolvers))); + typeAnnotationContext->UserDataStorage->SetTokenResolver(BuildCompositeTokenResolver(std::move(tokenResolvers))); return typeAnnotationContext; } diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h index 1f8f47681d..c954c08266 100644 --- a/ydb/library/yql/core/facade/yql_facade.h +++ b/ydb/library/yql/core/facade/yql_facade.h @@ -44,13 +44,13 @@ public: const TVector<TDataProviderInitializer>& dataProvidersInit, const TString& runner); - void AddUserDataTable(const TUserDataTable& userDataTable); + void AddUserDataTable(const TUserDataTable& userDataTable); void AddCredentialsTable(TCredentialTablePtr credentialTable); void SetUserCredentials(const TUserCredentials& userCredentials); void SetGatewaysConfig(const TGatewaysConfig* gatewaysConfig); void SetModules(IModuleResolver::TPtr modules); void SetUdfResolver(IUdfResolver::TPtr udfResolver); - void SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet); + void SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet); void SetFileStorage(TFileStoragePtr fileStorage); void EnableRangeComputeFor(); @@ -70,14 +70,14 @@ private: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; const ui64 NextUniqueId_; TVector<TDataProviderInitializer> DataProvidersInit_; - TUserDataTable UserDataTable_; + TUserDataTable UserDataTable_; TVector<TCredentialTablePtr> CredentialTables_; TUserCredentials UserCredentials_; const TGatewaysConfig* GatewaysConfig_; IModuleResolver::TPtr Modules_; IUdfResolver::TPtr UdfResolver_; - TUdfIndex::TPtr UdfIndex_; - TUdfIndexPackageSet::TPtr UdfIndexPackageSet_; + TUdfIndex::TPtr UdfIndex_; + TUdfIndexPackageSet::TPtr UdfIndexPackageSet_; TFileStoragePtr FileStorage_; TString Runner_; bool EnableRangeComputeFor_ = false; @@ -100,7 +100,7 @@ public: bool ParseSql(); bool ParseSql(const NSQLTranslation::TTranslationSettings& settings); - bool Compile(const TString& username); + bool Compile(const TString& username); TStatus Discover(const TString& username); @@ -169,13 +169,13 @@ public: } inline TIssues CompletedIssues() const { - if (ExprCtx_) { - return ExprCtx_->IssueManager.GetCompletedIssues(); - } else { + if (ExprCtx_) { + return ExprCtx_->IssueManager.GetCompletedIssues(); + } else { return {}; - } - } - + } + } + void Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan = false); inline void PrintErrorsTo(IOutputStream& out) const { @@ -231,11 +231,11 @@ public: NYson::EYsonFormat GetOutputFormat() const { return OutputFormat_; } void SetValidateOptions(NUdf::EValidateMode validateMode); - void SetDisableNativeUdfSupport(bool disable); + void SetDisableNativeUdfSupport(bool disable); void SetUseTableMetaFromGraph(bool use); void SetProgressWriter(TOperationProgressWriter writer) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); ProgressWriter_ = ThreadSafeProgressWriter(writer); } @@ -245,7 +245,7 @@ public: } void SetOperationId(const TString& id) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); OperationOptions_.Id = id; } @@ -255,7 +255,7 @@ public: } void SetOperationTitle(const TString& title) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); if (!title.Contains("YQL")) { ythrow yexception() << "Please mention YQL in the title '" << title << "'"; } @@ -269,23 +269,23 @@ public: } void SetQueryName(const TString& name) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); OperationOptions_.QueryName = name; } void SetOperationAttrsYson(const TString& attrs) { - Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); + Y_ENSURE(!TypeCtx_, "TypeCtx_ already created"); OperationOptions_.AttrsYson = attrs; } - void SetParametersYson(const TString& parameters); - // should be used after Compile phase - bool ExtractQueryParametersMetadata(); - - const TString& GetExtractedQueryParametersMetadataYson() const { - return ExtractedQueryParametersMetadataYson_; - } + void SetParametersYson(const TString& parameters); + // should be used after Compile phase + bool ExtractQueryParametersMetadata(); + const TString& GetExtractedQueryParametersMetadataYson() const { + return ExtractedQueryParametersMetadataYson_; + } + void EnableResultPosition() { SupportsResultPosition_ = true; } @@ -299,13 +299,13 @@ private: const TIntrusivePtr<ITimeProvider> timeProvider, ui64 nextUniqueId, const TVector<TDataProviderInitializer>& dataProvidersInit, - const TUserDataTable& userDataTable, + const TUserDataTable& userDataTable, const TVector<TCredentialTablePtr>& credentialTables, const TUserCredentials& userCredentials, const IModuleResolver::TPtr& modules, const IUdfResolver::TPtr& udfResolver, - const TUdfIndex::TPtr& udfIndex, - const TUdfIndexPackageSet::TPtr& udfIndexPackageSet, + const TUdfIndex::TPtr& udfIndex, + const TUdfIndexPackageSet::TPtr& udfIndexPackageSet, const TFileStoragePtr& fileStorage, const TGatewaysConfig* gatewaysConfig, const TString& filename, @@ -314,9 +314,9 @@ private: const TString& runner, bool enableRangeComputeFor); - TTypeAnnotationContextPtr BuildTypeAnnotationContext(const TString& username); - TTypeAnnotationContextPtr GetAnnotationContext() const; - TTypeAnnotationContextPtr ProvideAnnotationContext(const TString& username); + TTypeAnnotationContextPtr BuildTypeAnnotationContext(const TString& username); + TTypeAnnotationContextPtr GetAnnotationContext() const; + TTypeAnnotationContextPtr ProvideAnnotationContext(const TString& username); bool CollectUsedClusters(); NThreading::TFuture<void> OpenSession(const TString& username); @@ -329,12 +329,12 @@ private: TFutureStatus RemoteKikimrRun(const TString& cluster, const IPipelineConfigurator* pipelineConf); bool FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules = nullptr); - TString GetSessionId() const; - TString TakeSessionId(); + TString GetSessionId() const; + TString TakeSessionId(); NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransformWithFallback(bool applyAsyncChanges); -private: +private: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; const TIntrusivePtr<IRandomProvider> RandomProvider_; const TIntrusivePtr<ITimeProvider> TimeProvider_; @@ -345,10 +345,10 @@ private: TVector<TCredentialTablePtr> CredentialTables_; TUserCredentials UserCredentials_; const IUdfResolver::TPtr UdfResolver_; - const TUdfIndex::TPtr UdfIndex_; - const TUdfIndexPackageSet::TPtr UdfIndexPackageSet_; + const TUdfIndex::TPtr UdfIndex_; + const TUdfIndexPackageSet::TPtr UdfIndexPackageSet_; const TFileStoragePtr FileStorage_; - const TUserDataStorage::TPtr UserDataStorage_; + const TUserDataStorage::TPtr UserDataStorage_; const TGatewaysConfig* GatewaysConfig_; TString Filename_; TString SourceCode_; @@ -361,7 +361,7 @@ private: const IModuleResolver::TPtr Modules_; TExprNode::TPtr ExprRoot_; TExprNode::TPtr SavedExprRoot_; - mutable TAdaptiveLock SessionIdLock_; + mutable TAdaptiveLock SessionIdLock_; TString SessionId_; TTypeAnnotationContextPtr TypeCtx_; TAutoPtr<IPlanBuilder> PlanBuilder_; @@ -372,7 +372,7 @@ private: NYson::EYsonFormat OutputFormat_; TMaybe<NYson::EYsonFormat> DiagnosticFormat_; NUdf::EValidateMode ValidateMode_ = NUdf::EValidateMode::None; - bool DisableNativeUdfSupport_ = false; + bool DisableNativeUdfSupport_ = false; bool UseTableMetaFromGraph_ = false; TMaybe<TSet<TString>> UsedClusters_; TMaybe<TSet<TString>> UsedProviders_; @@ -383,7 +383,7 @@ private: IOutputStream* ExprStream_ = nullptr; IOutputStream* PlanStream_ = nullptr; TOperationProgressWriter ProgressWriter_ = [](const TOperationProgress&) {}; - TString ExtractedQueryParametersMetadataYson_; + TString ExtractedQueryParametersMetadataYson_; const bool EnableRangeComputeFor_; }; diff --git a/ydb/library/yql/core/file_storage/download_stream.cpp b/ydb/library/yql/core/file_storage/download_stream.cpp index ebd7c5616f..83bf7d4059 100644 --- a/ydb/library/yql/core/file_storage/download_stream.cpp +++ b/ydb/library/yql/core/file_storage/download_stream.cpp @@ -1,19 +1,19 @@ -#include "download_stream.h" - -namespace NYql { - -TDownloadStream::TDownloadStream(IInputStream& delegatee) - : Delegatee_(delegatee) -{ -} - -size_t TDownloadStream::DoRead(void* buf, size_t len) { - try { - return Delegatee_.Read(buf, len); - } catch (const std::exception& e) { - // just change type of the exception - throw TDownloadError() << e.what(); - } -} - -} +#include "download_stream.h" + +namespace NYql { + +TDownloadStream::TDownloadStream(IInputStream& delegatee) + : Delegatee_(delegatee) +{ +} + +size_t TDownloadStream::DoRead(void* buf, size_t len) { + try { + return Delegatee_.Read(buf, len); + } catch (const std::exception& e) { + // just change type of the exception + throw TDownloadError() << e.what(); + } +} + +} diff --git a/ydb/library/yql/core/file_storage/download_stream.h b/ydb/library/yql/core/file_storage/download_stream.h index 7f41509068..93d5006a5d 100644 --- a/ydb/library/yql/core/file_storage/download_stream.h +++ b/ydb/library/yql/core/file_storage/download_stream.h @@ -1,21 +1,21 @@ -#pragma once - -#include <util/generic/yexception.h> -#include <util/stream/input.h> - -namespace NYql { - -class TDownloadError : public yexception { -}; - -class TDownloadStream : public IInputStream { -public: - explicit TDownloadStream(IInputStream& delegatee); - -private: - size_t DoRead(void* buf, size_t len) override; - -private: - IInputStream& Delegatee_; -}; -} +#pragma once + +#include <util/generic/yexception.h> +#include <util/stream/input.h> + +namespace NYql { + +class TDownloadError : public yexception { +}; + +class TDownloadStream : public IInputStream { +public: + explicit TDownloadStream(IInputStream& delegatee); + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + IInputStream& Delegatee_; +}; +} diff --git a/ydb/library/yql/core/file_storage/file_storage.cpp b/ydb/library/yql/core/file_storage/file_storage.cpp index 631adbc688..8d6fd89a48 100644 --- a/ydb/library/yql/core/file_storage/file_storage.cpp +++ b/ydb/library/yql/core/file_storage/file_storage.cpp @@ -1,7 +1,7 @@ #include "file_storage.h" #include "storage.h" -#include "url_mapper.h" -#include "url_meta.h" +#include "url_mapper.h" +#include "url_meta.h" #include "download_stream.h" #include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h> @@ -13,64 +13,64 @@ #include <ydb/library/yql/utils/md5_stream.h> #include <ydb/library/yql/utils/retry.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <library/cpp/cache/cache.h> #include <library/cpp/digest/md5/md5.h> #include <library/cpp/threading/future/async.h> -#include <util/generic/guid.h> +#include <util/generic/guid.h> #include <util/generic/yexception.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/stream/null.h> #include <util/system/fs.h> -#include <util/system/fstat.h> -#include <util/system/guard.h> +#include <util/system/fstat.h> +#include <util/system/guard.h> #include <util/system/shellcommand.h> #include <util/system/sysstat.h> -#include <util/system/utime.h> +#include <util/system/utime.h> namespace NYql { -class TFileStorageImpl: public IFileStorage { +class TFileStorageImpl: public IFileStorage { public: - explicit TFileStorageImpl(const TFileStorageConfig& params) - : Storage(params.GetMaxFiles(), ui64(params.GetMaxSizeMb()) << 20ull, params.GetPath()) - , Config(params) - , QueueStarted(0) + explicit TFileStorageImpl(const TFileStorageConfig& params) + : Storage(params.GetMaxFiles(), ui64(params.GetMaxSizeMb()) << 20ull, params.GetPath()) + , Config(params) + , QueueStarted(0) { try { - for (const auto& sc : params.GetCustomSchemes()) { - Mapper.AddMapping(sc.GetPattern(), sc.GetTargetUrl()); + for (const auto& sc : params.GetCustomSchemes()) { + Mapper.AddMapping(sc.GetPattern(), sc.GetTargetUrl()); } } catch (const yexception& e) { - ythrow yexception() << "FileStorage: " << e.what(); + ythrow yexception() << "FileStorage: " << e.what(); } - - auto numThreads = params.GetThreads(); - if (1 == numThreads) { + + auto numThreads = params.GetThreads(); + if (1 == numThreads) { MtpQueue.Reset(new TFakeThreadPool()); - } else { + } else { MtpQueue.Reset(new TSimpleThreadPool(TThreadPoolParams{"FileStorage"})); - } - - // do not call MtpQueue->Start here as we have to do it _after_ fork - } - - ~TFileStorageImpl() { - MtpQueue->Stop(); + } + + // do not call MtpQueue->Start here as we have to do it _after_ fork } + ~TFileStorageImpl() { + MtpQueue->Stop(); + } + void AddDownloader(IDownloaderPtr downloader) override { Downloaders.push_back(std::move(downloader)); - } - + } + TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) override { YQL_LOG(INFO) << "PutFile to cache: " << file; - const auto md5 = MD5::File(file); - const TString storageFileName = md5 + ".file"; - auto lock = MultiResourceLock.Acquire(storageFileName); - return Storage.Put(storageFileName, outFileName, md5, [&file, &md5](const TFsPath& dstFile) { + const auto md5 = MD5::File(file); + const TString storageFileName = md5 + ".file"; + auto lock = MultiResourceLock.Acquire(storageFileName); + return Storage.Put(storageFileName, outFileName, md5, [&file, &md5](const TFsPath& dstFile) { NFs::HardLinkOrCopy(file, dstFile); i64 length = GetFileLength(dstFile.c_str()); if (length == -1) { @@ -128,11 +128,11 @@ public: } TFileLinkPtr PutInline(const TString& data) override { - const auto md5 = MD5::Calc(data); - const TString storageFileName = md5 + ".file"; + const auto md5 = MD5::Calc(data); + const TString storageFileName = md5 + ".file"; YQL_LOG(INFO) << "PutInline to cache. md5=" << md5; - auto lock = MultiResourceLock.Acquire(storageFileName); - return Storage.Put(storageFileName, TString(), md5, [&data, &md5](const TFsPath& dstFile) { + auto lock = MultiResourceLock.Acquire(storageFileName); + return Storage.Put(storageFileName, TString(), md5, [&data, &md5](const TFsPath& dstFile) { TStringInput in(data); TFile outFile(dstFile, CreateAlways | ARW | AX); TUnbufferedFileOutput out(outFile); @@ -150,54 +150,54 @@ public: } TFileLinkPtr PutUrl(const TString& urlStr, const TString& oauthToken) override { - try { - TString convertedUrl; - if (!Mapper.MapUrl(urlStr, convertedUrl)) { - convertedUrl = urlStr; - } + try { + TString convertedUrl; + if (!Mapper.MapUrl(urlStr, convertedUrl)) { + convertedUrl = urlStr; + } YQL_LOG(INFO) << "PutUrl to cache: " << convertedUrl; - THttpURL url = ParseURL(convertedUrl); + THttpURL url = ParseURL(convertedUrl); for (const auto& d: Downloaders) { if (d->Accept(url)) { return PutUrl(url, oauthToken, d); } - } + } ythrow yexception() << "Unsupported url: " << convertedUrl; - } catch (const std::exception& e) { + } catch (const std::exception& e) { YQL_LOG(ERROR) << "Failed to download file by URL \"" << urlStr << "\", details: " << e.what(); - YQL_LOG_CTX_THROW yexception() << "FileStorage: Failed to download file by URL \"" << urlStr << "\", details: " << e.what(); + YQL_LOG_CTX_THROW yexception() << "FileStorage: Failed to download file by URL \"" << urlStr << "\", details: " << e.what(); } } - NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName = {}) override { - StartQueueOnce(); - return NThreading::Async([=]() { - return this->PutFile(file, outFileName); - }, *MtpQueue); - } - - NThreading::TFuture<TFileLinkPtr> PutInlineAsync(const TString& data) override { - StartQueueOnce(); - return NThreading::Async([=]() { - return this->PutInline(data); - }, *MtpQueue); - } - - NThreading::TFuture<TFileLinkPtr> PutUrlAsync(const TString& url, const TString& oauthToken) override { - StartQueueOnce(); - return NThreading::Async([=]() { - return this->PutUrl(url, oauthToken); - }, *MtpQueue); - } - + NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName = {}) override { + StartQueueOnce(); + return NThreading::Async([=]() { + return this->PutFile(file, outFileName); + }, *MtpQueue); + } + + NThreading::TFuture<TFileLinkPtr> PutInlineAsync(const TString& data) override { + StartQueueOnce(); + return NThreading::Async([=]() { + return this->PutInline(data); + }, *MtpQueue); + } + + NThreading::TFuture<TFileLinkPtr> PutUrlAsync(const TString& url, const TString& oauthToken) override { + StartQueueOnce(); + return NThreading::Async([=]() { + return this->PutUrl(url, oauthToken); + }, *MtpQueue); + } + TFsPath GetRoot() const override { - return Storage.GetRoot(); + return Storage.GetRoot(); } TFsPath GetTemp() const override { - return Storage.GetTemp(); + return Storage.GetTemp(); } const TFileStorageConfig& GetConfig() const override { @@ -205,38 +205,38 @@ public: } private: - void StartQueueOnce() { - // we shall call Start only once - if (AtomicTryLock(&QueueStarted)) { - MtpQueue->Start(Config.GetThreads()); - } - } - + void StartQueueOnce() { + // we shall call Start only once + if (AtomicTryLock(&QueueStarted)) { + MtpQueue->Start(Config.GetThreads()); + } + } + TFileLinkPtr PutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) { - return WithRetry<TDownloadError>(Config.GetRetryCount(), [&, this]() { + return WithRetry<TDownloadError>(Config.GetRetryCount(), [&, this]() { return this->DoPutUrl(url, oauthToken, downloader); - }, [&](const auto& e, int attempt, int attemptCount) { + }, [&](const auto& e, int attempt, int attemptCount) { YQL_LOG(WARN) << "Error while downloading url " << url.PrintS() << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what(); - Sleep(TDuration::MilliSeconds(Config.GetRetryDelayMs())); - }); - } - + Sleep(TDuration::MilliSeconds(Config.GetRetryDelayMs())); + }); + } + TFileLinkPtr DoPutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) { - const auto urlMetaFile = BuildUrlMetaFileName(url); - auto lock = MultiResourceLock.Acquire(urlMetaFile); // let's use meta file as lock name - - TUrlMeta urlMeta; - urlMeta.TryReadFrom(GetRoot() / urlMetaFile); - - TFileLinkPtr oldContentLink = nullptr; - if (urlMeta.ContentFile) { - oldContentLink = Storage.HardlinkFromStorage(urlMeta.ContentFile, urlMeta.Md5, ""); - } - - if (!oldContentLink) { - urlMeta = TUrlMeta(); - } - + const auto urlMetaFile = BuildUrlMetaFileName(url); + auto lock = MultiResourceLock.Acquire(urlMetaFile); // let's use meta file as lock name + + TUrlMeta urlMeta; + urlMeta.TryReadFrom(GetRoot() / urlMetaFile); + + TFileLinkPtr oldContentLink = nullptr; + if (urlMeta.ContentFile) { + oldContentLink = Storage.HardlinkFromStorage(urlMeta.ContentFile, urlMeta.Md5, ""); + } + + if (!oldContentLink) { + urlMeta = TUrlMeta(); + } + YQL_LOG(INFO) << "UrlMeta: " << urlMetaFile << ", ETag=" << urlMeta.ETag << ", ContentFile=" << urlMeta.ContentFile << ", Md5=" << urlMeta.Md5 << ", LastModified=" << urlMeta.LastModified; @@ -262,41 +262,41 @@ private: else if (urlMeta.LastModified && lastModified) { YQL_LOG(INFO) << "LastModified for url " << url.PrintS() << " has been changed from " << urlMeta.LastModified << " to " << lastModified << ". We have to download new version"; } - + // todo: remove oldContentLink ? const auto urlContentFile = BuildUrlContentFileName(url, etag, lastModified); TFileLinkPtr result = Storage.Put(urlContentFile, TString(), TString(), puller); - - // save meta using rename for atomicity - urlMeta.ETag = etag; - urlMeta.LastModified = lastModified; - urlMeta.ContentFile = result->GetStorageFileName(); - urlMeta.Md5 = result->GetMd5(); - auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); - urlMeta.SaveTo(metaTmpFile); - - Storage.MoveToStorage(metaTmpFile, urlMetaFile); - - return result; - } - - static TString BuildUrlMetaFileName(const THttpURL& url) { - return MD5::Calc(url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url_meta"; - } - - static TString BuildUrlContentFileName(const THttpURL& url, const TString& etag, const TString& lastModified) { - TString needle = etag ? etag : lastModified; - return MD5::Calc(needle + url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url"; + + // save meta using rename for atomicity + urlMeta.ETag = etag; + urlMeta.LastModified = lastModified; + urlMeta.ContentFile = result->GetStorageFileName(); + urlMeta.Md5 = result->GetMd5(); + auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); + urlMeta.SaveTo(metaTmpFile); + + Storage.MoveToStorage(metaTmpFile, urlMetaFile); + + return result; } + static TString BuildUrlMetaFileName(const THttpURL& url) { + return MD5::Calc(url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url_meta"; + } + + static TString BuildUrlContentFileName(const THttpURL& url, const TString& etag, const TString& lastModified) { + TString needle = etag ? etag : lastModified; + return MD5::Calc(needle + url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url"; + } + private: - TStorage Storage; - const TFileStorageConfig Config; + TStorage Storage; + const TFileStorageConfig Config; std::vector<IDownloaderPtr> Downloaders; - TUrlMapper Mapper; - TAtomic QueueStarted; + TUrlMapper Mapper; + TAtomic QueueStarted; THolder<IThreadPool> MtpQueue; - TMultiResourceLock MultiResourceLock; + TMultiResourceLock MultiResourceLock; }; TFileStoragePtr CreateFileStorage(const TFileStorageConfig& params) { diff --git a/ydb/library/yql/core/file_storage/file_storage.h b/ydb/library/yql/core/file_storage/file_storage.h index 9133d57b1b..28fef171c4 100644 --- a/ydb/library/yql/core/file_storage/file_storage.h +++ b/ydb/library/yql/core/file_storage/file_storage.h @@ -4,7 +4,7 @@ #include <library/cpp/threading/future/future.h> #include <library/cpp/uri/http_url.h> - + #include <util/folder/path.h> #include <util/generic/ptr.h> #include <util/generic/string.h> @@ -29,12 +29,12 @@ struct IFileStorage: public TThrRefBase { virtual TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) = 0; virtual TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) = 0; virtual TFileLinkPtr PutInline(const TString& data) = 0; - virtual TFileLinkPtr PutUrl(const TString& url, const TString& oauthToken) = 0; - // async versions - virtual NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName = {}) = 0; - virtual NThreading::TFuture<TFileLinkPtr> PutInlineAsync(const TString& data) = 0; - virtual NThreading::TFuture<TFileLinkPtr> PutUrlAsync(const TString& url, const TString& oauthToken) = 0; - + virtual TFileLinkPtr PutUrl(const TString& url, const TString& oauthToken) = 0; + // async versions + virtual NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName = {}) = 0; + virtual NThreading::TFuture<TFileLinkPtr> PutInlineAsync(const TString& data) = 0; + virtual NThreading::TFuture<TFileLinkPtr> PutUrlAsync(const TString& url, const TString& oauthToken) = 0; + virtual TFsPath GetRoot() const = 0; virtual TFsPath GetTemp() const = 0; virtual const TFileStorageConfig& GetConfig() const = 0; diff --git a/ydb/library/yql/core/file_storage/file_storage_ut.cpp b/ydb/library/yql/core/file_storage/file_storage_ut.cpp index 5103fdd31d..96f08292d5 100644 --- a/ydb/library/yql/core/file_storage/file_storage_ut.cpp +++ b/ydb/library/yql/core/file_storage/file_storage_ut.cpp @@ -1,501 +1,501 @@ -#include "file_storage.h" +#include "file_storage.h" #include <ydb/library/yql/core/file_storage/ut/test_http_server.h> #include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h> #include <ydb/library/yql/core/file_storage/http_download/http_download.h> - + #include <library/cpp/threading/future/future.h> #include <library/cpp/threading/future/async.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/stream/file.h> -#include <util/system/tempfile.h> + +#include <util/stream/file.h> +#include <util/system/tempfile.h> #include <util/thread/pool.h> - -using namespace NYql; -using namespace NThreading; - + +using namespace NYql; +using namespace NThreading; + Y_UNIT_TEST_SUITE(TFileStorageTests) { - - static TString ReadFileContent(const TString& path) { - return TIFStream(path).ReadAll(); - } - + + static TString ReadFileContent(const TString& path) { + return TIFStream(path).ReadAll(); + } + static TFileStoragePtr CreateTestFS(const TFileStorageConfig& params = {}, const std::vector<TString>& extraPatterns = {}) { TFileStoragePtr fs = CreateFileStorage(params); fs->AddDownloader(MakeHttpDownloader(false, params, extraPatterns)); return fs; } - static std::unique_ptr<TTestHttpServer> CreateTestHttpServer() { - TPortManager pm; - const ui16 port = pm.GetPort(); - auto result = std::make_unique<TTestHttpServer>(port); - result->Start(); - - return result; - } - - static void RemoveUrlMeta(const TFsPath& root) { - TVector<TFsPath> children; - root.List(children); - - auto it = FindIf(children, [](auto& c) { - return c.GetExtension() == "url_meta"; - }); - - UNIT_ASSERT(it != children.end()); - it->DeleteIfExists(); - } - - TString MakeWeakETag(const TString& strongETag) { - return "W/" + strongETag; - } - + static std::unique_ptr<TTestHttpServer> CreateTestHttpServer() { + TPortManager pm; + const ui16 port = pm.GetPort(); + auto result = std::make_unique<TTestHttpServer>(port); + result->Start(); + + return result; + } + + static void RemoveUrlMeta(const TFsPath& root) { + TVector<TFsPath> children; + root.List(children); + + auto it = FindIf(children, [](auto& c) { + return c.GetExtension() == "url_meta"; + }); + + UNIT_ASSERT(it != children.end()); + it->DeleteIfExists(); + } + + TString MakeWeakETag(const TString& strongETag) { + return "W/" + strongETag; + } + Y_UNIT_TEST(PutUrlNoTokenNoETag) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto& request) { - UNIT_ASSERT_VALUES_EQUAL(request.OAuthToken, ""); - ++downloadCount; - return TTestHttpServer::TReply::Ok(currentContent); - }); - + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto& request) { + UNIT_ASSERT_VALUES_EQUAL(request.OAuthToken, ""); + ++downloadCount; + return TTestHttpServer::TReply::Ok(currentContent); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - auto link1 = fs->PutUrl(url, {}); - currentContent = "UNUSED FILE"; // just to be just second download will drop data - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); // assume file was not consumed completely second time - UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); - const TString abcMd5 = "902fbdd2b1df0c4f70b4a5d23525e932"; - UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), abcMd5); - UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link2->GetMd5()); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); - - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - } - + + auto url = server->GetUrl(); + auto link1 = fs->PutUrl(url, {}); + currentContent = "UNUSED FILE"; // just to be just second download will drop data + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); // assume file was not consumed completely second time + UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); + const TString abcMd5 = "902fbdd2b1df0c4f70b4a5d23525e932"; + UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), abcMd5); + UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link2->GetMd5()); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); + + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + } + Y_UNIT_TEST(PutUrlNoAccessForBadToken) { - auto server = CreateTestHttpServer(); - - TString okToken = "TOKEN_1"; - TString badToken = "TOKEN_2"; - - server->SetRequestHandler([&](auto& request) { - if (request.OAuthToken != okToken) { - return TTestHttpServer::TReply::Forbidden(); - } - - const TString content = "ABC"; - return TTestHttpServer::TReply::Ok(content); - }); - + auto server = CreateTestHttpServer(); + + TString okToken = "TOKEN_1"; + TString badToken = "TOKEN_2"; + + server->SetRequestHandler([&](auto& request) { + if (request.OAuthToken != okToken) { + return TTestHttpServer::TReply::Forbidden(); + } + + const TString content = "ABC"; + return TTestHttpServer::TReply::Ok(content); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, badToken), std::exception, "Failed to fetch url"); - - auto link1 = fs->PutUrl(url, okToken); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - - // still no access, we downloaded resource though - UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, badToken), std::exception, "Failed to fetch url"); - } - + + auto url = server->GetUrl(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, badToken), std::exception, "Failed to fetch url"); + + auto link1 = fs->PutUrl(url, okToken); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + + // still no access, we downloaded resource though + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, badToken), std::exception, "Failed to fetch url"); + } + Y_UNIT_TEST(PutUrlETagChange) { - auto server = CreateTestHttpServer(); - - TString currentETag = "TAG_1"; - TString currentContent = "ABC"; - - int downloadCount = 0; - server->SetRequestHandler([&](auto& request) { - if (request.IfNoneMatch == currentETag) { - return TTestHttpServer::TReply::NotModified(currentETag); - } - - ++downloadCount; - return TTestHttpServer::TReply::OkETag(currentContent, currentETag); - }); - + auto server = CreateTestHttpServer(); + + TString currentETag = "TAG_1"; + TString currentContent = "ABC"; + + int downloadCount = 0; + server->SetRequestHandler([&](auto& request) { + if (request.IfNoneMatch == currentETag) { + return TTestHttpServer::TReply::NotModified(currentETag); + } + + ++downloadCount; + return TTestHttpServer::TReply::OkETag(currentContent, currentETag); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - auto link1 = fs->PutUrl(url, {}); - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); - - // change etag: - currentETag = "TAG_2"; - currentContent = "XYZPQAZWSXEDC"; // change length as well - - auto link3 = fs->PutUrl(url, {}); - auto link4 = fs->PutUrl(url, {}); - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - - // check contents: - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); - UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); - } - - Y_UNIT_TEST(PutUrlLastModifiedChange) { - auto server = CreateTestHttpServer(); - - TString currentLastModified = "Wed, 05 Jun 2019 13:39:26 GMT"; - TString currentContent = "ABC"; - - int downloadCount = 0; - server->SetRequestHandler([&](auto& request) { - if (request.IfModifiedSince == currentLastModified) { - return TTestHttpServer::TReply::NotModified({}, currentLastModified); - } - - ++downloadCount; - return TTestHttpServer::TReply::OkLastModified(currentContent, currentLastModified); - }); - + + auto url = server->GetUrl(); + + auto link1 = fs->PutUrl(url, {}); + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); + + // change etag: + currentETag = "TAG_2"; + currentContent = "XYZPQAZWSXEDC"; // change length as well + + auto link3 = fs->PutUrl(url, {}); + auto link4 = fs->PutUrl(url, {}); + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + + // check contents: + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); + UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); + } + + Y_UNIT_TEST(PutUrlLastModifiedChange) { + auto server = CreateTestHttpServer(); + + TString currentLastModified = "Wed, 05 Jun 2019 13:39:26 GMT"; + TString currentContent = "ABC"; + + int downloadCount = 0; + server->SetRequestHandler([&](auto& request) { + if (request.IfModifiedSince == currentLastModified) { + return TTestHttpServer::TReply::NotModified({}, currentLastModified); + } + + ++downloadCount; + return TTestHttpServer::TReply::OkLastModified(currentContent, currentLastModified); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - auto link1 = fs->PutUrl(url, {}); - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); - - // change LastModified: - currentLastModified = "Wed, 06 Jun 2019 13:39:26 GMT"; - currentContent = "XYZPQAZWSXEDC"; // change length as well - - auto link3 = fs->PutUrl(url, {}); - auto link4 = fs->PutUrl(url, {}); - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - - // check contents: - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); - UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); - } - + + auto url = server->GetUrl(); + + auto link1 = fs->PutUrl(url, {}); + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); + + // change LastModified: + currentLastModified = "Wed, 06 Jun 2019 13:39:26 GMT"; + currentContent = "XYZPQAZWSXEDC"; // change length as well + + auto link3 = fs->PutUrl(url, {}); + auto link4 = fs->PutUrl(url, {}); + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + + // check contents: + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); + UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); + } + Y_UNIT_TEST(PutUrlETagChangeButNoSupportForIfNoneMatch) { - auto server = CreateTestHttpServer(); - - TString currentETag = "TAG_1"; - TString currentContent = "ABC"; - - int downloadCount = 0; - server->SetRequestHandler([&](auto& request) { - Y_UNUSED(request); - ++downloadCount; - return TTestHttpServer::TReply::OkETag(currentContent, currentETag); - }); - + auto server = CreateTestHttpServer(); + + TString currentETag = "TAG_1"; + TString currentContent = "ABC"; + + int downloadCount = 0; + server->SetRequestHandler([&](auto& request) { + Y_UNUSED(request); + ++downloadCount; + return TTestHttpServer::TReply::OkETag(currentContent, currentETag); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - auto link1 = fs->PutUrl(url, {}); - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - - // change etag: - currentETag = "TAG_2"; - currentContent = "XYZPQAZWSXEDC"; // change length as well - - auto link3 = fs->PutUrl(url, {}); - auto link4 = fs->PutUrl(url, {}); - UNIT_ASSERT_VALUES_EQUAL(4, downloadCount); - - // check contents: - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); - UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); - } - + + auto url = server->GetUrl(); + + auto link1 = fs->PutUrl(url, {}); + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + + // change etag: + currentETag = "TAG_2"; + currentContent = "XYZPQAZWSXEDC"; // change length as well + + auto link3 = fs->PutUrl(url, {}); + auto link4 = fs->PutUrl(url, {}); + UNIT_ASSERT_VALUES_EQUAL(4, downloadCount); + + // check contents: + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link3->GetMd5()); + UNIT_ASSERT_VALUES_EQUAL(link3->GetMd5(), link4->GetMd5()); + } + Y_UNIT_TEST(PutUrlWeakETagChange) { - auto server = CreateTestHttpServer(); - - TString currentETag = "TAG_1"; - TString currentContent = "ABC"; - - int downloadCount = 0; - server->SetRequestHandler([&](auto& request) { - if (request.IfNoneMatch == currentETag) { - return TTestHttpServer::TReply::NotModified(MakeWeakETag(currentETag)); - } - - ++downloadCount; - return TTestHttpServer::TReply::OkETag(currentContent, MakeWeakETag(currentETag)); - }); - + auto server = CreateTestHttpServer(); + + TString currentETag = "TAG_1"; + TString currentContent = "ABC"; + + int downloadCount = 0; + server->SetRequestHandler([&](auto& request) { + if (request.IfNoneMatch == currentETag) { + return TTestHttpServer::TReply::NotModified(MakeWeakETag(currentETag)); + } + + ++downloadCount; + return TTestHttpServer::TReply::OkETag(currentContent, MakeWeakETag(currentETag)); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - auto link1 = fs->PutUrl(url, {}); - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); - - // change etag: - currentETag = "TAG_2"; - currentContent = "XYZPQAZWSXEDC"; // change length as well - - auto link3 = fs->PutUrl(url, {}); - auto link4 = fs->PutUrl(url, {}); - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - - // check contents: - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); - } - + + auto url = server->GetUrl(); + + auto link1 = fs->PutUrl(url, {}); + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(1, downloadCount); + + // change etag: + currentETag = "TAG_2"; + currentContent = "XYZPQAZWSXEDC"; // change length as well + + auto link3 = fs->PutUrl(url, {}); + auto link4 = fs->PutUrl(url, {}); + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + + // check contents: + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link3->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("XYZPQAZWSXEDC", ReadFileContent(link4->GetPath())); + } + Y_UNIT_TEST(SecondPutUrlNoETagButFileRemoved) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto& request) { - UNIT_ASSERT(!request.IfNoneMatch); - ++downloadCount; - return TTestHttpServer::TReply::Ok(currentContent); - }); - + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto& request) { + UNIT_ASSERT(!request.IfNoneMatch); + ++downloadCount; + return TTestHttpServer::TReply::Ok(currentContent); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - auto link1 = fs->PutUrl(url, {}); - - auto storageFile = fs->GetRoot() / link1->GetStorageFileName(); - storageFile.DeleteIfExists(); - - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); - - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - } - + + auto url = server->GetUrl(); + auto link1 = fs->PutUrl(url, {}); + + auto storageFile = fs->GetRoot() / link1->GetStorageFileName(); + storageFile.DeleteIfExists(); + + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); + + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + } + Y_UNIT_TEST(SecondPutUrlETagButFileRemoved) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentETag = "TAG_1"; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto& request) { - UNIT_ASSERT(!request.IfNoneMatch); - ++downloadCount; - return TTestHttpServer::TReply::OkETag(currentContent, currentETag); - }); - + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentETag = "TAG_1"; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto& request) { + UNIT_ASSERT(!request.IfNoneMatch); + ++downloadCount; + return TTestHttpServer::TReply::OkETag(currentContent, currentETag); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - auto link1 = fs->PutUrl(url, {}); - - auto storageFile = fs->GetRoot() / link1->GetStorageFileName(); - storageFile.DeleteIfExists(); - - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); - - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - } - + + auto url = server->GetUrl(); + auto link1 = fs->PutUrl(url, {}); + + auto storageFile = fs->GetRoot() / link1->GetStorageFileName(); + storageFile.DeleteIfExists(); + + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); + + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + } + Y_UNIT_TEST(SecondPutUrlETagButMetaRemoved) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentETag = "TAG_1"; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto& request) { - UNIT_ASSERT(!request.IfNoneMatch); - ++downloadCount; - return TTestHttpServer::TReply::OkETag(currentContent, currentETag); - }); - + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentETag = "TAG_1"; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto& request) { + UNIT_ASSERT(!request.IfNoneMatch); + ++downloadCount; + return TTestHttpServer::TReply::OkETag(currentContent, currentETag); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - auto link1 = fs->PutUrl(url, {}); - - RemoveUrlMeta(fs->GetRoot()); - - auto link2 = fs->PutUrl(url, {}); - - UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); - UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); - UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link2->GetMd5()); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); - - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); - UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); - } - + + auto url = server->GetUrl(); + auto link1 = fs->PutUrl(url, {}); + + RemoveUrlMeta(fs->GetRoot()); + + auto link2 = fs->PutUrl(url, {}); + + UNIT_ASSERT_VALUES_EQUAL(2, downloadCount); + UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link2->GetStorageFileName()); + UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link2->GetMd5()); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link2->GetPath()); + + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link1->GetPath())); + UNIT_ASSERT_VALUES_EQUAL("ABC", ReadFileContent(link2->GetPath())); + } + Y_UNIT_TEST(Md5ForPutFiles) { - TString currentContent = "ABC"; - + TString currentContent = "ABC"; + TFileStoragePtr fs = CreateTestFS(); - - TTempFileHandle h1; - h1.Write("ABC", 3); - const TString abcMd5 = "902fbdd2b1df0c4f70b4a5d23525e932"; - // no hardlinks to existing file in storage - auto link1 = fs->PutInline("ABCD"); - auto link2 = fs->PutFile(h1.GetName()); - - // hardlinks - TTempFileHandle h2; - h2.Write("ABCD", 4); - auto link3 = fs->PutInline("ABC"); - auto link4 = fs->PutFile(h2.GetName()); - - UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link4->GetMd5()); - UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link4->GetStorageFileName()); - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link4->GetPath()); - - UNIT_ASSERT_VALUES_EQUAL(link2->GetMd5(), link3->GetMd5()); - UNIT_ASSERT_VALUES_EQUAL(link2->GetStorageFileName(), link3->GetStorageFileName()); - UNIT_ASSERT_VALUES_UNEQUAL(link2->GetPath(), link3->GetPath()); - - UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link2->GetMd5()); - UNIT_ASSERT_VALUES_UNEQUAL(link3->GetMd5(), link4->GetMd5()); - } - + + TTempFileHandle h1; + h1.Write("ABC", 3); + const TString abcMd5 = "902fbdd2b1df0c4f70b4a5d23525e932"; + // no hardlinks to existing file in storage + auto link1 = fs->PutInline("ABCD"); + auto link2 = fs->PutFile(h1.GetName()); + + // hardlinks + TTempFileHandle h2; + h2.Write("ABCD", 4); + auto link3 = fs->PutInline("ABC"); + auto link4 = fs->PutFile(h2.GetName()); + + UNIT_ASSERT_VALUES_EQUAL(link1->GetMd5(), link4->GetMd5()); + UNIT_ASSERT_VALUES_EQUAL(link1->GetStorageFileName(), link4->GetStorageFileName()); + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetPath(), link4->GetPath()); + + UNIT_ASSERT_VALUES_EQUAL(link2->GetMd5(), link3->GetMd5()); + UNIT_ASSERT_VALUES_EQUAL(link2->GetStorageFileName(), link3->GetStorageFileName()); + UNIT_ASSERT_VALUES_UNEQUAL(link2->GetPath(), link3->GetPath()); + + UNIT_ASSERT_VALUES_UNEQUAL(link1->GetMd5(), link2->GetMd5()); + UNIT_ASSERT_VALUES_UNEQUAL(link3->GetMd5(), link4->GetMd5()); + } + Y_UNIT_TEST(NoUrlDownloadRetryOnBadCode) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto& request) { - UNIT_ASSERT(!request.IfNoneMatch); - ++downloadCount; - return TTestHttpServer::TReply::Forbidden(); - }); - - TFileStorageConfig params; - params.SetRetryCount(3); + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto& request) { + UNIT_ASSERT(!request.IfNoneMatch); + ++downloadCount; + return TTestHttpServer::TReply::Forbidden(); + }); + + TFileStorageConfig params; + params.SetRetryCount(3); TFileStoragePtr fs = CreateTestFS(params); - - auto url = server->GetUrl(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "FileStorage: Failed to download file by URL"); - // currently this test does not work and we retry even if forbidden code is returned - // todo: optimize this - UNIT_ASSERT_VALUES_EQUAL(3, downloadCount); - } - - Y_UNIT_TEST(PutEmptyFiles) { - auto server = CreateTestHttpServer(); - - TString currentETag = "TAG_1"; - TString currentContent = ""; - - server->SetRequestHandler([&](auto& ) { - return TTestHttpServer::TReply::OkETag(currentContent, currentETag, 0); - }); - + + auto url = server->GetUrl(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "FileStorage: Failed to download file by URL"); + // currently this test does not work and we retry even if forbidden code is returned + // todo: optimize this + UNIT_ASSERT_VALUES_EQUAL(3, downloadCount); + } + + Y_UNIT_TEST(PutEmptyFiles) { + auto server = CreateTestHttpServer(); + + TString currentETag = "TAG_1"; + TString currentContent = ""; + + server->SetRequestHandler([&](auto& ) { + return TTestHttpServer::TReply::OkETag(currentContent, currentETag, 0); + }); + TFileStoragePtr fs = CreateTestFS(); - - auto url = server->GetUrl(); - - auto link1 = fs->PutUrl(url, {}); - auto link2 = fs->PutInline(""); - - TTempFileHandle tmpFile; - auto link3 = fs->PutFile(tmpFile.GetName()); - const TString emptyStringMd5 = "d41d8cd98f00b204e9800998ecf8427e"; - - for (auto link : { link1, link2, link3 }) { - UNIT_ASSERT_VALUES_EQUAL("", ReadFileContent(link->GetPath())); - UNIT_ASSERT_VALUES_EQUAL(emptyStringMd5, link->GetMd5()); - } - } - - Y_UNIT_TEST(BadContentLength) { - auto server = CreateTestHttpServer(); - - int downloadCount = 0; - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto&) { - ++downloadCount; - return TTestHttpServer::TReply::Ok(currentContent, 10); - }); - - TFileStorageConfig params; - params.SetRetryCount(3); + + auto url = server->GetUrl(); + + auto link1 = fs->PutUrl(url, {}); + auto link2 = fs->PutInline(""); + + TTempFileHandle tmpFile; + auto link3 = fs->PutFile(tmpFile.GetName()); + const TString emptyStringMd5 = "d41d8cd98f00b204e9800998ecf8427e"; + + for (auto link : { link1, link2, link3 }) { + UNIT_ASSERT_VALUES_EQUAL("", ReadFileContent(link->GetPath())); + UNIT_ASSERT_VALUES_EQUAL(emptyStringMd5, link->GetMd5()); + } + } + + Y_UNIT_TEST(BadContentLength) { + auto server = CreateTestHttpServer(); + + int downloadCount = 0; + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto&) { + ++downloadCount; + return TTestHttpServer::TReply::Ok(currentContent, 10); + }); + + TFileStorageConfig params; + params.SetRetryCount(3); TFileStoragePtr fs = CreateTestFS(params); - - auto url = server->GetUrl(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "Size mismatch while downloading url http"); - UNIT_ASSERT_VALUES_EQUAL(3, downloadCount); - } - - Y_UNIT_TEST(AllowedUrls) { - auto server = CreateTestHttpServer(); - - TString currentContent = "ABC"; - server->SetRequestHandler([&](auto&) { - return TTestHttpServer::TReply::Ok(currentContent); - }); - - auto url = server->GetUrl(); - + + auto url = server->GetUrl(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "Size mismatch while downloading url http"); + UNIT_ASSERT_VALUES_EQUAL(3, downloadCount); + } + + Y_UNIT_TEST(AllowedUrls) { + auto server = CreateTestHttpServer(); + + TString currentContent = "ABC"; + server->SetRequestHandler([&](auto&) { + return TTestHttpServer::TReply::Ok(currentContent); + }); + + auto url = server->GetUrl(); + { // not in whitelist TFileStorageConfig params; params.AddAllowedUrlPatterns("^XXXX$"); TFileStoragePtr fs = CreateTestFS(params); - + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "It is not allowed to download url http://localhost:"); } - + { // have in whitelist TFileStorageConfig params; params.SetSocketTimeoutMs(4000); params.AddAllowedUrlPatterns("^http://localhost:"); TFileStoragePtr fs = CreateTestFS(params); - + auto link = fs->PutUrl(url, {}); UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath())); } - + { // have eaxtra url in whitelist TFileStorageConfig params; @@ -506,21 +506,21 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) { auto link = fs->PutUrl(url, {}); UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath())); } - } - - Y_UNIT_TEST(SocketTimeout) { - auto server = CreateTestHttpServer(); - - server->SetRequestHandler([&](auto& ) { - Sleep(TDuration::Seconds(2)); - return TTestHttpServer::TReply::Ok("ABC"); - }); - - TFileStorageConfig params; - params.SetSocketTimeoutMs(1000); + } + + Y_UNIT_TEST(SocketTimeout) { + auto server = CreateTestHttpServer(); + + server->SetRequestHandler([&](auto& ) { + Sleep(TDuration::Seconds(2)); + return TTestHttpServer::TReply::Ok("ABC"); + }); + + TFileStorageConfig params; + params.SetSocketTimeoutMs(1000); TFileStoragePtr fs = CreateTestFS(params); - - auto url = server->GetUrl(); - UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "can not read from socket input stream"); - } -} + + auto url = server->GetUrl(); + UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "can not read from socket input stream"); + } +} diff --git a/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp b/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp index f60ec107be..a30bf53829 100644 --- a/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp +++ b/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp @@ -1,34 +1,34 @@ #include <ydb/library/yql/core/file_storage/http_download/pattern_group.h> - -namespace NYql { - -TPatternGroup::TPatternGroup(const TVector<TString>& patterns) { - for (auto& p : patterns) { - Add(p); - } -} - -void TPatternGroup::Add(const TString& pattern) { - auto it = CompiledPatterns.find(pattern); - if (it != CompiledPatterns.end()) { - return; - } - - CompiledPatterns.emplace(pattern, TRegExMatch(pattern)); -} - -bool TPatternGroup::IsEmpty() const { - return CompiledPatterns.empty(); -} - -bool TPatternGroup::Match(const TString& s) const { - for (auto& p : CompiledPatterns) { - if (p.second.Match(s.c_str())) { - return true; - } - } - - return false; -} - -} + +namespace NYql { + +TPatternGroup::TPatternGroup(const TVector<TString>& patterns) { + for (auto& p : patterns) { + Add(p); + } +} + +void TPatternGroup::Add(const TString& pattern) { + auto it = CompiledPatterns.find(pattern); + if (it != CompiledPatterns.end()) { + return; + } + + CompiledPatterns.emplace(pattern, TRegExMatch(pattern)); +} + +bool TPatternGroup::IsEmpty() const { + return CompiledPatterns.empty(); +} + +bool TPatternGroup::Match(const TString& s) const { + for (auto& p : CompiledPatterns) { + if (p.second.Match(s.c_str())) { + return true; + } + } + + return false; +} + +} diff --git a/ydb/library/yql/core/file_storage/http_download/pattern_group.h b/ydb/library/yql/core/file_storage/http_download/pattern_group.h index 53d0d64a7f..38007e5a95 100644 --- a/ydb/library/yql/core/file_storage/http_download/pattern_group.h +++ b/ydb/library/yql/core/file_storage/http_download/pattern_group.h @@ -1,20 +1,20 @@ -#pragma once - +#pragma once + #include <library/cpp/regex/pcre/regexp.h> -#include <util/generic/map.h> -#include <util/generic/vector.h> - -namespace NYql { - -class TPatternGroup { -public: - explicit TPatternGroup(const TVector<TString>& patterns = {}); - void Add(const TString& pattern); - bool IsEmpty() const; - bool Match(const TString& s) const; - -private: - TMap<TString, TRegExMatch> CompiledPatterns; -}; - -} +#include <util/generic/map.h> +#include <util/generic/vector.h> + +namespace NYql { + +class TPatternGroup { +public: + explicit TPatternGroup(const TVector<TString>& patterns = {}); + void Add(const TString& pattern); + bool IsEmpty() const; + bool Match(const TString& s) const; + +private: + TMap<TString, TRegExMatch> CompiledPatterns; +}; + +} diff --git a/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp b/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp index 1b917252c1..cda028351c 100644 --- a/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp +++ b/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp @@ -1,36 +1,36 @@ -#include "pattern_group.h" +#include "pattern_group.h" #include <library/cpp/testing/unittest/registar.h> - -using namespace NYql; - -namespace { -void Check(const TPatternGroup& g, const TString& url, bool expectedResult) { - UNIT_ASSERT_C(g.Match(url) == expectedResult, url); -} - -void CheckSuccess(const TPatternGroup& g, const TString& url) { - Check(g, url, true); -} - -void CheckFailure(const TPatternGroup& g, const TString& url) { - Check(g, url, false); -} -} - -Y_UNIT_TEST_SUITE(TPatternGroupTests) { - Y_UNIT_TEST(All) { - TPatternGroup g; - UNIT_ASSERT(g.IsEmpty()); - - g.Add(R"XXX(^https://[\w-]+\.yt\.yandex-team\.ru/)XXX"); - g.Add(R"XXX(^https://(paste|storage|nirvana|grafana|bb|fml)\.yandex-team\.ru/)XXX"); - UNIT_ASSERT(!g.IsEmpty()); - - CheckSuccess(g, "https://seneca-sas.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); - CheckSuccess(g, "https://paste.yandex-team.ru/727063/text"); - - CheckFailure(g, "https://hahn.yt.yandex-team.ru:80/api/v3/read_file"); - CheckFailure(g, "http://ya.ru"); - } -} + +using namespace NYql; + +namespace { +void Check(const TPatternGroup& g, const TString& url, bool expectedResult) { + UNIT_ASSERT_C(g.Match(url) == expectedResult, url); +} + +void CheckSuccess(const TPatternGroup& g, const TString& url) { + Check(g, url, true); +} + +void CheckFailure(const TPatternGroup& g, const TString& url) { + Check(g, url, false); +} +} + +Y_UNIT_TEST_SUITE(TPatternGroupTests) { + Y_UNIT_TEST(All) { + TPatternGroup g; + UNIT_ASSERT(g.IsEmpty()); + + g.Add(R"XXX(^https://[\w-]+\.yt\.yandex-team\.ru/)XXX"); + g.Add(R"XXX(^https://(paste|storage|nirvana|grafana|bb|fml)\.yandex-team\.ru/)XXX"); + UNIT_ASSERT(!g.IsEmpty()); + + CheckSuccess(g, "https://seneca-sas.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); + CheckSuccess(g, "https://paste.yandex-team.ru/727063/text"); + + CheckFailure(g, "https://hahn.yt.yandex-team.ru:80/api/v3/read_file"); + CheckFailure(g, "http://ya.ru"); + } +} diff --git a/ydb/library/yql/core/file_storage/proto/file_storage.proto b/ydb/library/yql/core/file_storage/proto/file_storage.proto index c8fa642932..67c461b072 100644 --- a/ydb/library/yql/core/file_storage/proto/file_storage.proto +++ b/ydb/library/yql/core/file_storage/proto/file_storage.proto @@ -10,13 +10,13 @@ message TFileStorageConfig { optional uint32 MaxFiles = 2 [default = 1000]; // Maximum number of files in the storage optional uint32 MaxSizeMb = 3 [default = 100]; // Maximum total size of all files in the storage repeated TSchemeTranslate CustomSchemes = 4; - optional uint32 Threads = 5 [default = 1]; // Number of download threads for async downloading - optional uint32 RetryCount = 6 [default = 0]; // Number of additional attempts to download file - optional uint32 RetryDelayMs = 7 [default = 1000]; // Delay in ms between attempts to download file - repeated string AllowedUrlPatterns = 8; // Whitelist of url regexps; disabled if empty + optional uint32 Threads = 5 [default = 1]; // Number of download threads for async downloading + optional uint32 RetryCount = 6 [default = 0]; // Number of additional attempts to download file + optional uint32 RetryDelayMs = 7 [default = 1000]; // Delay in ms between attempts to download file + repeated string AllowedUrlPatterns = 8; // Whitelist of url regexps; disabled if empty repeated string ExternalAllowedUrlPatterns = 12; // Whitelist of url regexps for external users; disabled if empty - optional uint32 SocketTimeoutMs = 9 [default = 300000]; - optional string ArcSshUser = 10; - optional string ArcSshPkPath = 11; + optional uint32 SocketTimeoutMs = 9 [default = 300000]; + optional string ArcSshUser = 10; + optional string ArcSshPkPath = 11; optional string ArcTokenPath = 13; } diff --git a/ydb/library/yql/core/file_storage/storage.cpp b/ydb/library/yql/core/file_storage/storage.cpp index 247bff5242..7a146694ed 100644 --- a/ydb/library/yql/core/file_storage/storage.cpp +++ b/ydb/library/yql/core/file_storage/storage.cpp @@ -5,13 +5,13 @@ #include <ydb/library/yql/utils/proc_alive.h> #include <library/cpp/digest/md5/md5.h> - + #include <util/folder/dirut.h> #include <util/generic/algorithm.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> #include <util/generic/ptr.h> -#include <util/generic/utility.h> +#include <util/generic/utility.h> #include <util/system/file.h> #include <util/system/file_lock.h> #include <util/system/fs.h> @@ -52,20 +52,20 @@ TFsPath ToFilePath(const TString& path) } // namespace -TFileLink::TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy) +TFileLink::TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy) : Path(path) , StorageFileName(storageFileName) , Size(size) - , Md5(md5) - , DeleteOnDestroy(deleteOnDestroy) + , Md5(md5) + , DeleteOnDestroy(deleteOnDestroy) { } TFileLink::~TFileLink() { - if (!DeleteOnDestroy) { - return; - } - + if (!DeleteOnDestroy) { + return; + } + YQL_LOG(INFO) << "Destroying TFileLink for " << Path.GetPath().Quote(); try { Path.ForceDelete(); @@ -75,37 +75,37 @@ TFileLink::~TFileLink() { } TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5, bool deleteOnDestroy) { - if (!path.Exists()) { - ythrow yexception() << "Unable to create file link for non-existent file " << path.GetPath().Quote(); - } - - TString effectiveMd5 = md5; - if (!effectiveMd5) { - effectiveMd5 = MD5::File(path); - } - const i64 size = GetFileLength(path); - if (size == -1) { - ythrow yexception() << "Unable to get size for file " << path.GetPath().Quote(); - } - + if (!path.Exists()) { + ythrow yexception() << "Unable to create file link for non-existent file " << path.GetPath().Quote(); + } + + TString effectiveMd5 = md5; + if (!effectiveMd5) { + effectiveMd5 = MD5::File(path); + } + const i64 size = GetFileLength(path); + if (size == -1) { + ythrow yexception() << "Unable to get size for file " << path.GetPath().Quote(); + } + return new TFileLink(path, effectiveMd5, size, effectiveMd5, deleteOnDestroy); -} - -bool SetCacheFilePermissionsNoThrow(const TString& path) { +} + +bool SetCacheFilePermissionsNoThrow(const TString& path) { return Chmod(path.data(), MODE0755) == 0; -} - -void SetCacheFilePermissions(const TString& path) { - SetFilePermissions(path, MODE0755); -} - -void SetFilePermissions(const TString& path, int mode) { +} + +void SetCacheFilePermissions(const TString& path) { + SetFilePermissions(path, MODE0755); +} + +void SetFilePermissions(const TString& path, int mode) { if (Chmod(path.data(), mode)) { TFileStat fstat(path.data()); - ythrow TSystemError() << "Failed to chmod file " << path.Quote() << ", uid = " << fstat.Uid << ", mode = " << fstat.Mode << ", new mode = " << mode; - } -} - + ythrow TSystemError() << "Failed to chmod file " << path.Quote() << ", uid = " << fstat.Uid << ", mode = " << fstat.Mode << ", new mode = " << mode; + } +} + class TStorage::TImpl: public TIntrusiveListItem<TImpl> { public: class TAtforkReinit { @@ -135,8 +135,8 @@ public: private: void Reinit() { with_lock (Mutex) { - for (auto& v : Registered) { - v.ResetRandom(); + for (auto& v : Registered) { + v.ResetRandom(); } } } @@ -160,8 +160,8 @@ public: StorageDir.PathSplit(); ProcessTempDir.PathSplit(); - StorageDir.MkDirs(MODE0711); - ProcessTempDir.MkDirs(MODE0711); + StorageDir.MkDirs(MODE0711); + ProcessTempDir.MkDirs(MODE0711); #ifdef _linux_ ProcessTempDirLock.Reset(new TFileLock(ProcessTempDir / ".lockfile")); ProcessTempDirLock->Acquire(); @@ -198,16 +198,16 @@ public: return ProcessTempDir; } - TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TStorage::TDataPuller& puller) { + TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TStorage::TDataPuller& puller) { bool newFileAdded = false; - TFileLinkPtr result = HardlinkFromStorage(storageFileName, md5, outFileName); - if (!result) { - TFsPath storageFile = StorageDir / storageFileName; - TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); - Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); - - ui64 fileSize = 0; - TString pullerMd5; // overrides input arg 'md5' + TFileLinkPtr result = HardlinkFromStorage(storageFileName, md5, outFileName); + if (!result) { + TFsPath storageFile = StorageDir / storageFileName; + TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); + Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); + + ui64 fileSize = 0; + TString pullerMd5; // overrides input arg 'md5' try { std::tie(fileSize, pullerMd5) = puller(hardlinkFile); } catch (...) { @@ -215,97 +215,97 @@ public: NFs::Remove(hardlinkFile); throw; } - Y_ENSURE(hardlinkFile.Exists(), "FileStorage: cannot put not existing temporary path"); - Y_ENSURE(hardlinkFile.IsFile(), "FileStorage: cannot put non-file temporary path"); + Y_ENSURE(hardlinkFile.Exists(), "FileStorage: cannot put not existing temporary path"); + Y_ENSURE(hardlinkFile.IsFile(), "FileStorage: cannot put non-file temporary path"); SetCacheFilePermissionsNoThrow(hardlinkFile); - if (NFs::HardLink(hardlinkFile, storageFile)) { - AtomicIncrement(CurrentFiles); - AtomicAdd(CurrentSize, fileSize); - } - // Ignore HardLink fail. Another process managed to download before us + if (NFs::HardLink(hardlinkFile, storageFile)) { + AtomicIncrement(CurrentFiles); + AtomicAdd(CurrentSize, fileSize); + } + // Ignore HardLink fail. Another process managed to download before us TouchFile(storageFile.c_str()); - newFileAdded = true; - result = MakeIntrusive<TFileLink>(hardlinkFile, storageFileName, fileSize, pullerMd5); + newFileAdded = true; + result = MakeIntrusive<TFileLink>(hardlinkFile, storageFileName, fileSize, pullerMd5); } - YQL_LOG(INFO) << "Using " << (newFileAdded ? "new" : "existing") << " storage file " << result->GetStorageFileName().Quote() - << ", temp path: " << result->GetPath().GetPath().Quote() - << ", size: " << result->GetSize(); + YQL_LOG(INFO) << "Using " << (newFileAdded ? "new" : "existing") << " storage file " << result->GetStorageFileName().Quote() + << ", temp path: " << result->GetPath().GetPath().Quote() + << ", size: " << result->GetSize(); if (newFileAdded) { Cleanup(); } - return result; + return result; } - TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) { - TFsPath storageFile = StorageDir / existingStorageFileName; - TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); - Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); - - if (!NFs::HardLink(storageFile, hardlinkFile)) { - return nullptr; - } - + TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) { + TFsPath storageFile = StorageDir / existingStorageFileName; + TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); + Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); + + if (!NFs::HardLink(storageFile, hardlinkFile)) { + return nullptr; + } + TouchFile(storageFile.c_str()); - SetCacheFilePermissionsNoThrow(hardlinkFile); - + SetCacheFilePermissionsNoThrow(hardlinkFile); + const i64 fileSize = GetFileLength(hardlinkFile); if (fileSize < 0) { ythrow yexception() << "Unable to get size for file " << hardlinkFile.GetPath().Quote(); } - TString md5 = storageFileMd5; - if (!md5) { - // could happen rarely - YQL_LOG(WARN) << "Rebuilding MD5 for file " << hardlinkFile.GetPath().Quote() << ", storage file " << existingStorageFileName << ". Usually it means file was downloaded via HTTP by another process and we just hardlinked it"; - md5 = MD5::File(hardlinkFile); - } - - return new TFileLink(hardlinkFile, existingStorageFileName, fileSize, md5); - } - - void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) { - TFsPath dstStorageFile = StorageDir / dstStorageFileName; - const bool prevFileExisted = dstStorageFile.Exists(); + TString md5 = storageFileMd5; + if (!md5) { + // could happen rarely + YQL_LOG(WARN) << "Rebuilding MD5 for file " << hardlinkFile.GetPath().Quote() << ", storage file " << existingStorageFileName << ". Usually it means file was downloaded via HTTP by another process and we just hardlinked it"; + md5 = MD5::File(hardlinkFile); + } + + return new TFileLink(hardlinkFile, existingStorageFileName, fileSize, md5); + } + + void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) { + TFsPath dstStorageFile = StorageDir / dstStorageFileName; + const bool prevFileExisted = dstStorageFile.Exists(); const i64 prevFileSize = Max<i64>(0, GetFileLength(dstStorageFile.c_str())); - - if (!NFs::Rename(src, dstStorageFile)) { - ythrow yexception() << "Failed to rename file from " << src << " to " << dstStorageFile; - } + + if (!NFs::Rename(src, dstStorageFile)) { + ythrow yexception() << "Failed to rename file from " << src << " to " << dstStorageFile; + } SetCacheFilePermissionsNoThrow(dstStorageFile); - + const i64 newFileSize = Max<i64>(0, GetFileLength(dstStorageFile.c_str())); - - if (!prevFileExisted) { - AtomicIncrement(CurrentFiles); - } - - AtomicAdd(CurrentSize, newFileSize - prevFileSize); - } - - bool RemoveFromStorage(const TString& existingStorageFileName) { - TFsPath storageFile = StorageDir / existingStorageFileName; - if (!storageFile.Exists()) { - // can't update statistics - // not sure we had this file at all - return false; - } - - // file could be removed by another process, handle this situation + + if (!prevFileExisted) { + AtomicIncrement(CurrentFiles); + } + + AtomicAdd(CurrentSize, newFileSize - prevFileSize); + } + + bool RemoveFromStorage(const TString& existingStorageFileName) { + TFsPath storageFile = StorageDir / existingStorageFileName; + if (!storageFile.Exists()) { + // can't update statistics + // not sure we had this file at all + return false; + } + + // file could be removed by another process, handle this situation const i64 prevFileSize = Max<i64>(0, GetFileLength(storageFile.c_str())); - const bool result = NFs::Remove(storageFile); - - if (result || !storageFile.Exists()) { - AtomicDecrement(CurrentFiles); - AtomicAdd(CurrentSize, -prevFileSize); - } - - return result; - } - + const bool result = NFs::Remove(storageFile); + + if (result || !storageFile.Exists()) { + AtomicDecrement(CurrentFiles); + AtomicAdd(CurrentSize, -prevFileSize); + } + + return result; + } + ui64 GetOccupiedSize() const { return AtomicGet(CurrentSize); } @@ -314,12 +314,12 @@ public: return AtomicGet(CurrentFiles); } - TString GetTempName() { - with_lock(RndLock) { - return Rnd.GenGuid(); - } - } - + TString GetTempName() { + with_lock(RndLock) { + return Rnd.GenGuid(); + } + } + private: void LoadStats() { TVector<TString> names; @@ -337,7 +337,7 @@ private: ++actualFiles; actualSize += stat.Size; } else if (stat.IsDir() && TryFromString(name, oldPid)) { - if (!IsProcessAlive(oldPid)) { + if (!IsProcessAlive(oldPid)) { // cleanup of previously not cleaned hardlinks directory try { #ifdef _linux_ @@ -427,9 +427,9 @@ private: } void ResetRandom() { - with_lock(RndLock) { - Rnd.ResetSeed(); - } + with_lock(RndLock) { + Rnd.ResetSeed(); + } } private: @@ -443,7 +443,7 @@ private: TAtomic CurrentFiles = 0; TAtomic CurrentSize = 0; TMutex RndLock; - TRandGuid Rnd; + TRandGuid Rnd; }; TStorage::TStorage(size_t maxFiles, ui64 maxSize, const TString& storagePath) @@ -465,26 +465,26 @@ TFsPath TStorage::GetTemp() const return Impl->GetTemp(); } -TFileLinkPtr TStorage::Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller) -{ - return Impl->Put(storageFileName, outFileName, md5, puller); -} - -TFileLinkPtr TStorage::HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) -{ - return Impl->HardlinkFromStorage(existingStorageFileName, storageFileMd5, outFileName); -} - -void TStorage::MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) +TFileLinkPtr TStorage::Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller) { - return Impl->MoveToStorage(src, dstStorageFileName); -} - -bool TStorage::RemoveFromStorage(const TString& existingStorageFileName) -{ - return Impl->RemoveFromStorage(existingStorageFileName); + return Impl->Put(storageFileName, outFileName, md5, puller); } +TFileLinkPtr TStorage::HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) +{ + return Impl->HardlinkFromStorage(existingStorageFileName, storageFileMd5, outFileName); +} + +void TStorage::MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) +{ + return Impl->MoveToStorage(src, dstStorageFileName); +} + +bool TStorage::RemoveFromStorage(const TString& existingStorageFileName) +{ + return Impl->RemoveFromStorage(existingStorageFileName); +} + ui64 TStorage::GetOccupiedSize() const { return Impl->GetOccupiedSize(); @@ -495,8 +495,8 @@ size_t TStorage::GetCount() const return Impl->GetCount(); } -TString TStorage::GetTempName() -{ - return Impl->GetTempName(); -} +TString TStorage::GetTempName() +{ + return Impl->GetTempName(); +} } // NYql diff --git a/ydb/library/yql/core/file_storage/storage.h b/ydb/library/yql/core/file_storage/storage.h index 8ab42a69a1..ed77bb5320 100644 --- a/ydb/library/yql/core/file_storage/storage.h +++ b/ydb/library/yql/core/file_storage/storage.h @@ -11,26 +11,26 @@ namespace NYql { class TFileLink: public TThrRefBase { public: - TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy = true); + TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy = true); ~TFileLink(); const TString& GetStorageFileName() const { return StorageFileName; } const TFsPath& GetPath() const { return Path; } ui64 GetSize() const { return Size; } - const TString& GetMd5() const { return Md5; } + const TString& GetMd5() const { return Md5; } private: - const TFsPath Path; - const TString StorageFileName; - const ui64 Size; - const TString Md5; - const bool DeleteOnDestroy; + const TFsPath Path; + const TString StorageFileName; + const ui64 Size; + const TString Md5; + const bool DeleteOnDestroy; }; using TFileLinkPtr = TIntrusivePtr<TFileLink>; TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5 = "", bool deleteOnDestroy = false); - + /* File cache with limits by count and by occupied size. The same file can be added multiple times. It is safe to add the same file from multiple threads/processes simultaneously. @@ -39,8 +39,8 @@ TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5 = "", bo class TStorage { public: // This function is called by the storage to transfer user data to the provided temporary file path - // Returns content size and md5 - using TDataPuller = std::function<std::pair<ui64, TString> (const TFsPath& dstPath)>; + // Returns content size and md5 + using TDataPuller = std::function<std::pair<ui64, TString> (const TFsPath& dstPath)>; /* Constructs the storage with the specified limits. storagePath can be empty - a temporary directory will be used in this case. @@ -53,29 +53,29 @@ public: // Returns temp storage directory TFsPath GetTemp() const; // Puts the passed data to the storage with the specified storage file name. - // The second argument outFileName specifies a name of temporary link returned from the Put(). If empty, then random guid is used. - // Provide valid md5 if it is known in advance, otherwise pass "". It will be overridden by puller result + // The second argument outFileName specifies a name of temporary link returned from the Put(). If empty, then random guid is used. + // Provide valid md5 if it is known in advance, otherwise pass "". It will be overridden by puller result // The provided puller does the actual transfer of the data to a storage file. It can take a long time. - TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller); - // Returns nullptr on error - TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName); - void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName); - bool RemoveFromStorage(const TString& existingStorageFileName); + TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller); + // Returns nullptr on error + TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName); + void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName); + bool RemoveFromStorage(const TString& existingStorageFileName); // Total size of all files in the cache ui64 GetOccupiedSize() const; // Total count of files in the cache size_t GetCount() const; - TString GetTempName(); - + TString GetTempName(); + private: class TImpl; THolder<TImpl> Impl; }; -constexpr int MODE0711 = S_IRWXU | S_IXGRP | S_IXOTH; -constexpr int MODE0744 = S_IRWXU | S_IRGRP | S_IROTH; - -void SetCacheFilePermissions(const TString& path); -bool SetCacheFilePermissionsNoThrow(const TString& path); -void SetFilePermissions(const TString& path, int mode); +constexpr int MODE0711 = S_IRWXU | S_IXGRP | S_IXOTH; +constexpr int MODE0744 = S_IRWXU | S_IRGRP | S_IROTH; + +void SetCacheFilePermissions(const TString& path); +bool SetCacheFilePermissionsNoThrow(const TString& path); +void SetFilePermissions(const TString& path, int mode); } // NYql diff --git a/ydb/library/yql/core/file_storage/storage_ut.cpp b/ydb/library/yql/core/file_storage/storage_ut.cpp index 4229abd219..8cad105539 100644 --- a/ydb/library/yql/core/file_storage/storage_ut.cpp +++ b/ydb/library/yql/core/file_storage/storage_ut.cpp @@ -15,7 +15,7 @@ using namespace NYql; using namespace NThreading; static TString DATA = "1234567890"; -static TString DATA_MD5 = "e807f1fcf82d132f9bb018ca6738a19f"; +static TString DATA_MD5 = "e807f1fcf82d132f9bb018ca6738a19f"; Y_UNIT_TEST_SUITE(TStorageTests) { @@ -43,17 +43,17 @@ Y_UNIT_TEST_SUITE(TStorageTests) { } }; - TFileLinkPtr PutFile(const TString& name, TStorage& storage, const TString& tmpOutName = {}, int sleepSeconds = 0) { - return storage.Put(name, tmpOutName, DATA_MD5, [&](const TFsPath& storagePath) { - Sleep(TDuration::Seconds(sleepSeconds)); + TFileLinkPtr PutFile(const TString& name, TStorage& storage, const TString& tmpOutName = {}, int sleepSeconds = 0) { + return storage.Put(name, tmpOutName, DATA_MD5, [&](const TFsPath& storagePath) { + Sleep(TDuration::Seconds(sleepSeconds)); TStringInput in(DATA); - TUnbufferedFileOutput out(storagePath); - return std::make_pair(TransferData(&in, &out), DATA_MD5); + TUnbufferedFileOutput out(storagePath); + return std::make_pair(TransferData(&in, &out), DATA_MD5); }); } TFileLinkPtr PutFileWithSleep(const TString& name, TStorage& storage) { - return PutFile(name, storage, {}, 1); + return PutFile(name, storage, {}, 1); } Y_UNIT_TEST(Put) { diff --git a/ydb/library/yql/core/file_storage/url_mapper.cpp b/ydb/library/yql/core/file_storage/url_mapper.cpp index 0c646ed0d6..596f6bb6e2 100644 --- a/ydb/library/yql/core/file_storage/url_mapper.cpp +++ b/ydb/library/yql/core/file_storage/url_mapper.cpp @@ -1,19 +1,19 @@ -#include "url_mapper.h" - -namespace NYql { - -void TUrlMapper::AddMapping(const TString& pattern, const TString& targetUrl) { - CustomSchemes.push_back(TCustomScheme(pattern, targetUrl)); -} - -bool TUrlMapper::MapUrl(const TString& url, TString& mappedUrl) const { - for (const auto& sc : CustomSchemes) { +#include "url_mapper.h" + +namespace NYql { + +void TUrlMapper::AddMapping(const TString& pattern, const TString& targetUrl) { + CustomSchemes.push_back(TCustomScheme(pattern, targetUrl)); +} + +bool TUrlMapper::MapUrl(const TString& url, TString& mappedUrl) const { + for (const auto& sc : CustomSchemes) { if (sc.Pattern.Match(url.data())) { mappedUrl = TRegExSubst(sc.TargetUrlSubst).Replace(url.data()); - return true; - } - } - return false; -} - -} + return true; + } + } + return false; +} + +} diff --git a/ydb/library/yql/core/file_storage/url_mapper.h b/ydb/library/yql/core/file_storage/url_mapper.h index f6438ecca3..9a5e3191e6 100644 --- a/ydb/library/yql/core/file_storage/url_mapper.h +++ b/ydb/library/yql/core/file_storage/url_mapper.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include <library/cpp/regex/pcre/regexp.h> -#include <util/generic/vector.h> - -namespace NYql { - -class TUrlMapper { -public: - void AddMapping(const TString& pattern, const TString& targetUrl); - bool MapUrl(const TString& url, TString& mappedUrl) const; - -private: - struct TCustomScheme { - TCustomScheme(const TString& pattern, const TString& url) - : Pattern(pattern) - , TargetUrlHolder(url) +#include <util/generic/vector.h> + +namespace NYql { + +class TUrlMapper { +public: + void AddMapping(const TString& pattern, const TString& targetUrl); + bool MapUrl(const TString& url, TString& mappedUrl) const; + +private: + struct TCustomScheme { + TCustomScheme(const TString& pattern, const TString& url) + : Pattern(pattern) + , TargetUrlHolder(url) , TargetUrlSubst(pattern.data()) { if (0 == TargetUrlSubst.ParseReplacement(TargetUrlHolder.data())) { - ythrow yexception() << "Bad url replacement: " << TargetUrlHolder; - } - } - TRegExMatch Pattern; - TString TargetUrlHolder; - TRegExSubst TargetUrlSubst; - }; - -private: - TVector<TCustomScheme> CustomSchemes; -}; - -} + ythrow yexception() << "Bad url replacement: " << TargetUrlHolder; + } + } + TRegExMatch Pattern; + TString TargetUrlHolder; + TRegExSubst TargetUrlSubst; + }; + +private: + TVector<TCustomScheme> CustomSchemes; +}; + +} diff --git a/ydb/library/yql/core/file_storage/url_mapper_ut.cpp b/ydb/library/yql/core/file_storage/url_mapper_ut.cpp index 287caf054c..7a44bac5ec 100644 --- a/ydb/library/yql/core/file_storage/url_mapper_ut.cpp +++ b/ydb/library/yql/core/file_storage/url_mapper_ut.cpp @@ -1,57 +1,57 @@ -#include "url_mapper.h" +#include "url_mapper.h" #include <library/cpp/testing/unittest/registar.h> - -using namespace NYql; - -namespace { -void RemapAndCheck(const TUrlMapper& m, const TString& input, const TString& expectedOutput) { - TString output; - UNIT_ASSERT(m.MapUrl(input, output)); - UNIT_ASSERT_VALUES_EQUAL(output, expectedOutput); -} -} - + +using namespace NYql; + +namespace { +void RemapAndCheck(const TUrlMapper& m, const TString& input, const TString& expectedOutput) { + TString output; + UNIT_ASSERT(m.MapUrl(input, output)); + UNIT_ASSERT_VALUES_EQUAL(output, expectedOutput); +} +} + Y_UNIT_TEST_SUITE(TUrlMapperTests) { Y_UNIT_TEST(All) { - TUrlMapper m; - m.AddMapping("sbr:(?://)?(\\d+)", "http://proxy.sandbox.yandex-team.ru/$1"); - - m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2&transaction_id=$3"); - m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2"); - + TUrlMapper m; + m.AddMapping("sbr:(?://)?(\\d+)", "http://proxy.sandbox.yandex-team.ru/$1"); + + m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2&transaction_id=$3"); + m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2"); + m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3"); m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3"); - + m.AddMapping("https?://a\\.yandex-team\\.ru/arc/(.+)/arcadia/(.+)\\?rev=(\\d+)", "arc://$2?rev=$3&branch=$1"); - TString tmp; - UNIT_ASSERT(!m.MapUrl("http://ya.ru", tmp)); - UNIT_ASSERT(!m.MapUrl("http://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home", tmp)); - - RemapAndCheck(m, "sbr:654321", "http://proxy.sandbox.yandex-team.ru/654321"); - RemapAndCheck(m, "sbr://654321", "http://proxy.sandbox.yandex-team.ru/654321"); - RemapAndCheck(m, "yt://hahn/statbox/home/ngc224/nile/tmp/libexamples-pager_ctr.soYDyrqeVMhp", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/ngc224/nile/tmp/libexamples-pager_ctr.soYDyrqeVMhp"); - // transaction guid : 4 groups of hex - RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190-dd3c4184", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164f64-3fe0190-dd3c4184"); - // transaction guid : 0-0-0-0 - RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=0-0-0-0", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=0-0-0-0"); - // transaction guid with bad char z, it will be dropped - RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190-dd3c418z", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164f64-3fe0190-dd3c418"); - // transaction guid with capitals - RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164F64-3FE0190-DD3C4184A", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164F64-3FE0190-DD3C4184A"); - // transaction guid with 3 groups instead of 4 - RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190"); - RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/#page=navigation&path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt"); + TString tmp; + UNIT_ASSERT(!m.MapUrl("http://ya.ru", tmp)); + UNIT_ASSERT(!m.MapUrl("http://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home", tmp)); + + RemapAndCheck(m, "sbr:654321", "http://proxy.sandbox.yandex-team.ru/654321"); + RemapAndCheck(m, "sbr://654321", "http://proxy.sandbox.yandex-team.ru/654321"); + RemapAndCheck(m, "yt://hahn/statbox/home/ngc224/nile/tmp/libexamples-pager_ctr.soYDyrqeVMhp", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/ngc224/nile/tmp/libexamples-pager_ctr.soYDyrqeVMhp"); + // transaction guid : 4 groups of hex + RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190-dd3c4184", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164f64-3fe0190-dd3c4184"); + // transaction guid : 0-0-0-0 + RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=0-0-0-0", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=0-0-0-0"); + // transaction guid with bad char z, it will be dropped + RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190-dd3c418z", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164f64-3fe0190-dd3c418"); + // transaction guid with capitals + RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164F64-3FE0190-DD3C4184A", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt&transaction_id=9797-164F64-3FE0190-DD3C4184A"); + // transaction guid with 3 groups instead of 4 + RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190"); + RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/#page=navigation&path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt"); RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/navigation?path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt"); - - // with dash - RemapAndCheck(m, "yt://seneca-sas/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca-sas.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); - // with underscore and digits - RemapAndCheck(m, "yt://seneca_sas0123456789/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca_sas0123456789.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); + + // with dash + RemapAndCheck(m, "yt://seneca-sas/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca-sas.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); + // with underscore and digits + RemapAndCheck(m, "yt://seneca_sas0123456789/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca_sas0123456789.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating"); RemapAndCheck(m, "https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=trunk"); RemapAndCheck(m, "https://a.yandex-team.ru/arc/branches/yql/yql-stable-2019-08-16/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=branches/yql/yql-stable-2019-08-16"); UNIT_ASSERT(!m.MapUrl("https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make", tmp)); - } -} + } +} diff --git a/ydb/library/yql/core/file_storage/url_meta.cpp b/ydb/library/yql/core/file_storage/url_meta.cpp index 2dd973ea06..064856c650 100644 --- a/ydb/library/yql/core/file_storage/url_meta.cpp +++ b/ydb/library/yql/core/file_storage/url_meta.cpp @@ -1,68 +1,68 @@ -#include "url_meta.h" +#include "url_meta.h" #include <library/cpp/http/io/headers.h> -#include <util/stream/file.h> -#include <util/system/file.h> - -namespace NYql { - -void TUrlMeta::SaveTo(const TString& path) { - THttpHeaders headers; - if (ETag) { - headers.AddHeader(THttpInputHeader("ETag", ETag)); - } - - if (ContentFile) { - headers.AddHeader(THttpInputHeader("ContentFile", ContentFile)); - } - - if (Md5) { - headers.AddHeader(THttpInputHeader("Md5", Md5)); - } - - if (LastModified) { - headers.AddHeader(THttpInputHeader("LastModified", LastModified)); - } - - - TFile outFile(path, CreateAlways | ARW | AX); - TUnbufferedFileOutput out(outFile); - headers.OutTo(&out); -} - -void TUrlMeta::TryReadFrom(const TString& path) { - ETag.clear(); - LastModified.clear(); - ContentFile.clear(); - - TFileHandle fileHandle(path, OpenExisting | RdOnly | Seq); - if (!fileHandle.IsOpen()) { - return; - } - - TFile file(fileHandle.Release(), path); - TIFStream in(file); - THttpHeaders headers(&in); - for (auto it = headers.Begin(); it != headers.End(); ++it) { +#include <util/stream/file.h> +#include <util/system/file.h> + +namespace NYql { + +void TUrlMeta::SaveTo(const TString& path) { + THttpHeaders headers; + if (ETag) { + headers.AddHeader(THttpInputHeader("ETag", ETag)); + } + + if (ContentFile) { + headers.AddHeader(THttpInputHeader("ContentFile", ContentFile)); + } + + if (Md5) { + headers.AddHeader(THttpInputHeader("Md5", Md5)); + } + + if (LastModified) { + headers.AddHeader(THttpInputHeader("LastModified", LastModified)); + } + + + TFile outFile(path, CreateAlways | ARW | AX); + TUnbufferedFileOutput out(outFile); + headers.OutTo(&out); +} + +void TUrlMeta::TryReadFrom(const TString& path) { + ETag.clear(); + LastModified.clear(); + ContentFile.clear(); + + TFileHandle fileHandle(path, OpenExisting | RdOnly | Seq); + if (!fileHandle.IsOpen()) { + return; + } + + TFile file(fileHandle.Release(), path); + TIFStream in(file); + THttpHeaders headers(&in); + for (auto it = headers.Begin(); it != headers.End(); ++it) { if (it->Name() == TStringBuf("ETag")) { - ETag = it->Value(); - continue; - } - + ETag = it->Value(); + continue; + } + if (it->Name() == TStringBuf("ContentFile")) { - ContentFile = it->Value(); - continue; - } - + ContentFile = it->Value(); + continue; + } + if (it->Name() == TStringBuf("Md5")) { - Md5 = it->Value(); - continue; - } - + Md5 = it->Value(); + continue; + } + if (it->Name() == TStringBuf("LastModified")) { - LastModified = it->Value(); - continue; - } - } -} - -} // NYql + LastModified = it->Value(); + continue; + } + } +} + +} // NYql diff --git a/ydb/library/yql/core/file_storage/url_meta.h b/ydb/library/yql/core/file_storage/url_meta.h index 3121628d45..f1c7eb5839 100644 --- a/ydb/library/yql/core/file_storage/url_meta.h +++ b/ydb/library/yql/core/file_storage/url_meta.h @@ -1,17 +1,17 @@ -#pragma once - -#include <util/generic/string.h> - -namespace NYql { - -struct TUrlMeta { - TString ETag; - TString ContentFile; - TString Md5; - TString LastModified; - - void SaveTo(const TString& path); - void TryReadFrom(const TString& path); -}; - -} +#pragma once + +#include <util/generic/string.h> + +namespace NYql { + +struct TUrlMeta { + TString ETag; + TString ContentFile; + TString Md5; + TString LastModified; + + void SaveTo(const TString& path); + void TryReadFrom(const TString& path); +}; + +} diff --git a/ydb/library/yql/core/file_storage/ut/test_http_server.cpp b/ydb/library/yql/core/file_storage/ut/test_http_server.cpp index c296fc4bef..727b50127d 100644 --- a/ydb/library/yql/core/file_storage/ut/test_http_server.cpp +++ b/ydb/library/yql/core/file_storage/ut/test_http_server.cpp @@ -1,152 +1,152 @@ -#include "test_http_server.h" - +#include "test_http_server.h" + #include <library/cpp/http/misc/httpcodes.h> #include <library/cpp/http/server/http_ex.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/system/atomic_ops.h> - -namespace NYql { - -class TTestHttpServer::TImpl : public THttpServer::ICallBack { - class TRequestProcessor : public THttpClientRequestEx { - public: - explicit TRequestProcessor(TImpl* parent) - : Parent_(parent) - { - Y_UNUSED(Parent_); - } - - bool Reply(void* /*tsr*/) override { - if (!ProcessHeaders()) { + +#include <util/system/atomic_ops.h> + +namespace NYql { + +class TTestHttpServer::TImpl : public THttpServer::ICallBack { + class TRequestProcessor : public THttpClientRequestEx { + public: + explicit TRequestProcessor(TImpl* parent) + : Parent_(parent) + { + Y_UNUSED(Parent_); + } + + bool Reply(void* /*tsr*/) override { + if (!ProcessHeaders()) { return true; - } - - if (!RequestString.StartsWith("GET ")) { + } + + if (!RequestString.StartsWith("GET ")) { return true; - } - - TRequest r; - for (auto& p : ParsedHeaders) { - if (p.first == "Authorization" && p.second.StartsWith("OAuth ")) { - r.OAuthToken = p.second.substr(strlen("OAuth ")); - continue; - } - - if (p.first == "If-None-Match") { - r.IfNoneMatch = p.second; - continue; - } - - if (p.first == "If-Modified-Since") { - r.IfModifiedSince = p.second; - continue; - } - } - - auto reply = Parent_->ProcessNextRequest(r); - - switch (reply.Code) { - case HTTP_OK: - Output() << "HTTP/1.1 200 Ok\r\n"; - break; - - case HTTP_NOT_MODIFIED: - Output() << "HTTP/1.1 304 Not modified\r\n"; - break; - - case HTTP_FORBIDDEN: - Output() << "HTTP/1.1 403 Forbidden\r\n"; - break; - - default: + } + + TRequest r; + for (auto& p : ParsedHeaders) { + if (p.first == "Authorization" && p.second.StartsWith("OAuth ")) { + r.OAuthToken = p.second.substr(strlen("OAuth ")); + continue; + } + + if (p.first == "If-None-Match") { + r.IfNoneMatch = p.second; + continue; + } + + if (p.first == "If-Modified-Since") { + r.IfModifiedSince = p.second; + continue; + } + } + + auto reply = Parent_->ProcessNextRequest(r); + + switch (reply.Code) { + case HTTP_OK: + Output() << "HTTP/1.1 200 Ok\r\n"; + break; + + case HTTP_NOT_MODIFIED: + Output() << "HTTP/1.1 304 Not modified\r\n"; + break; + + case HTTP_FORBIDDEN: + Output() << "HTTP/1.1 403 Forbidden\r\n"; + break; + + default: return true; - } - - if (reply.ETag) { - Output() << "ETag: " + reply.ETag + "\r\n"; - } - - if (reply.LastModified) { - Output() << "Last-Modified: " + reply.LastModified + "\r\n"; - } - - if (reply.Content || reply.ContentLength) { - const int length = reply.ContentLength.GetOrElse(reply.Content.length()); - Output() << "Content-Length: " << length << "\r\n"; - } - - Output() << "\r\n"; - if (reply.Content) { - Output() << reply.Content; - } - - Output().Finish(); - - return true; - } - - private: - TImpl* Parent_ = nullptr; - }; - -public: - explicit TImpl(int port) - : HttpServer_(this, THttpServer::TOptions(port)) - , Port_(port) - { - - } - - TClientRequest* CreateClient() override { - return new TRequestProcessor(this); - } - - void Start() { - UNIT_ASSERT(HttpServer_.Start()); - } - - void Stop() { - HttpServer_.Stop(); - } - - TString GetUrl() const { - return "http://localhost:" + ToString(Port_); - } - - void SetRequestHandler(TRequestHandler handler) { - RequestHandler_ = std::move(handler); - } - -private: - TReply ProcessNextRequest(const TRequest& request) { - return RequestHandler_(request); - } - -private: - THttpServer HttpServer_; - const int Port_; - TRequestHandler RequestHandler_; -}; - -TTestHttpServer::TTestHttpServer(int port) - : Impl_(new TImpl(port)) { -} - -TTestHttpServer::~TTestHttpServer() { - Impl_->Stop(); -} - -void TTestHttpServer::Start() { - Impl_->Start(); -} - -TString TTestHttpServer::GetUrl() const { - return Impl_->GetUrl(); -} - -void TTestHttpServer::SetRequestHandler(TRequestHandler handler) { - return Impl_->SetRequestHandler(std::move(handler)); -} - -} + } + + if (reply.ETag) { + Output() << "ETag: " + reply.ETag + "\r\n"; + } + + if (reply.LastModified) { + Output() << "Last-Modified: " + reply.LastModified + "\r\n"; + } + + if (reply.Content || reply.ContentLength) { + const int length = reply.ContentLength.GetOrElse(reply.Content.length()); + Output() << "Content-Length: " << length << "\r\n"; + } + + Output() << "\r\n"; + if (reply.Content) { + Output() << reply.Content; + } + + Output().Finish(); + + return true; + } + + private: + TImpl* Parent_ = nullptr; + }; + +public: + explicit TImpl(int port) + : HttpServer_(this, THttpServer::TOptions(port)) + , Port_(port) + { + + } + + TClientRequest* CreateClient() override { + return new TRequestProcessor(this); + } + + void Start() { + UNIT_ASSERT(HttpServer_.Start()); + } + + void Stop() { + HttpServer_.Stop(); + } + + TString GetUrl() const { + return "http://localhost:" + ToString(Port_); + } + + void SetRequestHandler(TRequestHandler handler) { + RequestHandler_ = std::move(handler); + } + +private: + TReply ProcessNextRequest(const TRequest& request) { + return RequestHandler_(request); + } + +private: + THttpServer HttpServer_; + const int Port_; + TRequestHandler RequestHandler_; +}; + +TTestHttpServer::TTestHttpServer(int port) + : Impl_(new TImpl(port)) { +} + +TTestHttpServer::~TTestHttpServer() { + Impl_->Stop(); +} + +void TTestHttpServer::Start() { + Impl_->Start(); +} + +TString TTestHttpServer::GetUrl() const { + return Impl_->GetUrl(); +} + +void TTestHttpServer::SetRequestHandler(TRequestHandler handler) { + return Impl_->SetRequestHandler(std::move(handler)); +} + +} diff --git a/ydb/library/yql/core/file_storage/ut/test_http_server.h b/ydb/library/yql/core/file_storage/ut/test_http_server.h index 170b45ca62..00198584c5 100644 --- a/ydb/library/yql/core/file_storage/ut/test_http_server.h +++ b/ydb/library/yql/core/file_storage/ut/test_http_server.h @@ -1,77 +1,77 @@ -#pragma once - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> - +#pragma once + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> + #include <library/cpp/http/misc/httpcodes.h> #include <library/cpp/threading/future/future.h> -#include <functional> - -namespace NYql { - -class TTestHttpServer { -public: - struct TReply { - int Code = 0; - TString ETag; - TString LastModified; - TString Content; - TMaybe<int> ContentLength; - - static TReply Ok(const TString& content, TMaybe<int> contentLength = {}) { - TReply r; - r.Code = HTTP_OK; - r.Content = content; - r.ContentLength = contentLength; - return r; - } - - static TReply OkETag(const TString& content, const TString& etag, TMaybe<int> contentLength = {}) { - TReply r = Ok(content, contentLength); - r.ETag = etag; - return r; - } - - static TReply OkLastModified(const TString& content, const TString& lastModified, TMaybe<int> contentLength = {}) { - TReply r = Ok(content, contentLength); - r.LastModified = lastModified; - return r; - } - - static TReply NotModified(const TString& etag = {}, const TString& lastModified = {}) { - TReply r; - r.Code = HTTP_NOT_MODIFIED; - r.ETag = etag; - r.LastModified = lastModified; - return r; - } - - static TReply Forbidden() { - TReply r; - r.Code = HTTP_FORBIDDEN; - return r; - } - }; - - struct TRequest { - TString OAuthToken; - TString IfNoneMatch; - TString IfModifiedSince; - }; - - typedef std::function<TReply(const TRequest& request)> TRequestHandler; - -public: - explicit TTestHttpServer(int port); - ~TTestHttpServer(); - void Start(); - TString GetUrl() const; - void SetRequestHandler(TRequestHandler handler); - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - -} +#include <functional> + +namespace NYql { + +class TTestHttpServer { +public: + struct TReply { + int Code = 0; + TString ETag; + TString LastModified; + TString Content; + TMaybe<int> ContentLength; + + static TReply Ok(const TString& content, TMaybe<int> contentLength = {}) { + TReply r; + r.Code = HTTP_OK; + r.Content = content; + r.ContentLength = contentLength; + return r; + } + + static TReply OkETag(const TString& content, const TString& etag, TMaybe<int> contentLength = {}) { + TReply r = Ok(content, contentLength); + r.ETag = etag; + return r; + } + + static TReply OkLastModified(const TString& content, const TString& lastModified, TMaybe<int> contentLength = {}) { + TReply r = Ok(content, contentLength); + r.LastModified = lastModified; + return r; + } + + static TReply NotModified(const TString& etag = {}, const TString& lastModified = {}) { + TReply r; + r.Code = HTTP_NOT_MODIFIED; + r.ETag = etag; + r.LastModified = lastModified; + return r; + } + + static TReply Forbidden() { + TReply r; + r.Code = HTTP_FORBIDDEN; + return r; + } + }; + + struct TRequest { + TString OAuthToken; + TString IfNoneMatch; + TString IfModifiedSince; + }; + + typedef std::function<TReply(const TRequest& request)> TRequestHandler; + +public: + explicit TTestHttpServer(int port); + ~TTestHttpServer(); + void Start(); + TString GetUrl() const; + void SetRequestHandler(TRequestHandler handler); + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +} diff --git a/ydb/library/yql/core/file_storage/ut/ya.make b/ydb/library/yql/core/file_storage/ut/ya.make index c19f58203b..831ea0f36f 100644 --- a/ydb/library/yql/core/file_storage/ut/ya.make +++ b/ydb/library/yql/core/file_storage/ut/ya.make @@ -10,11 +10,11 @@ ENDIF() SRCS( - file_storage_ut.cpp + file_storage_ut.cpp sized_cache_ut.cpp storage_ut.cpp - test_http_server.cpp - url_mapper_ut.cpp + test_http_server.cpp + url_mapper_ut.cpp ) PEERDIR( diff --git a/ydb/library/yql/core/file_storage/ya.make b/ydb/library/yql/core/file_storage/ya.make index f25592bfb7..2f204cce32 100644 --- a/ydb/library/yql/core/file_storage/ya.make +++ b/ydb/library/yql/core/file_storage/ya.make @@ -3,18 +3,18 @@ LIBRARY() OWNER(g:yql) SRCS( - download_stream.cpp - download_stream.h + download_stream.cpp + download_stream.h file_storage.cpp - file_storage.h + file_storage.h sized_cache.cpp - sized_cache.h - storage.cpp - storage.h - url_mapper.cpp - url_mapper.h - url_meta.cpp - url_meta.h + sized_cache.h + storage.cpp + storage.h + url_mapper.cpp + url_mapper.h + url_meta.cpp + url_meta.h ) PEERDIR( diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index a2ed91d640..434f094af5 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -84,12 +84,12 @@ message TIssuesIds { // yt YT_TABLE_PATH_RECORD_FOR_TMP = 3001; YT_INFER_SCHEMA = 3002; - YT_ACCESS_DENIED = 3003; - YT_TABLE_NOT_FOUND = 3004; + YT_ACCESS_DENIED = 3003; + YT_TABLE_NOT_FOUND = 3004; YT_NATIVE_PRAGMA = 3005; - YT_VIEW_NOT_FOUND = 3006; - YT_SCHEMELESS_TABLE = 3007; - YT_ENTRY_NOT_TABLE_OR_VIEW = 3008; + YT_VIEW_NOT_FOUND = 3006; + YT_SCHEMELESS_TABLE = 3007; + YT_ENTRY_NOT_TABLE_OR_VIEW = 3008; YT_CONCURRENT_TABLE_MODIF = 3009; YT_LATE_TABLE_XLOCK = 3010; YT_SORT_ORDER_CHANGE = 3011; diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index 897d5d6f9a..9c1c915cd2 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -324,7 +324,7 @@ ids { code: YQL_MISSING_IS_BEFORE_NOT_NULL severity: S_WARNING } -ids { +ids { code: YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX severity: S_WARNING } @@ -365,17 +365,17 @@ ids { severity: S_WARNING } ids { - code: YT_VIEW_NOT_FOUND - severity: S_ERROR -} -ids { - code: YT_SCHEMELESS_TABLE - severity: S_ERROR -} -ids { - code: YT_ENTRY_NOT_TABLE_OR_VIEW - severity: S_ERROR -} + code: YT_VIEW_NOT_FOUND + severity: S_ERROR +} +ids { + code: YT_SCHEMELESS_TABLE + severity: S_ERROR +} +ids { + code: YT_ENTRY_NOT_TABLE_OR_VIEW + severity: S_ERROR +} ids { code: YT_CONCURRENT_TABLE_MODIF severity: S_ERROR diff --git a/ydb/library/yql/core/services/mounts/yql_mounts.cpp b/ydb/library/yql/core/services/mounts/yql_mounts.cpp index 7d0e221799..6928878ea1 100644 --- a/ydb/library/yql/core/services/mounts/yql_mounts.cpp +++ b/ydb/library/yql/core/services/mounts/yql_mounts.cpp @@ -99,15 +99,15 @@ namespace NYql { } - TUserDataTable GetYqlDefaultMounts() { - TUserDataTable userData; - AddLibraryFromResource(userData, "/lib/yql/aggregate.yql"); - AddLibraryFromResource(userData, "/lib/yql/window.yql"); - AddLibraryFromResource(userData, "/lib/yql/id.yql"); - AddLibraryFromResource(userData, "/lib/yql/sqr.yql"); - AddLibraryFromResource(userData, "/lib/yql/core.yql"); - - return userData; + TUserDataTable GetYqlDefaultMounts() { + TUserDataTable userData; + AddLibraryFromResource(userData, "/lib/yql/aggregate.yql"); + AddLibraryFromResource(userData, "/lib/yql/window.yql"); + AddLibraryFromResource(userData, "/lib/yql/id.yql"); + AddLibraryFromResource(userData, "/lib/yql/sqr.yql"); + AddLibraryFromResource(userData, "/lib/yql/core.yql"); + + return userData; } TUserDataTable GetYqlModuleResolverImpl( @@ -121,11 +121,11 @@ namespace NYql { { auto ctx = rawCtx ? rawCtx : ownedCtx.Get(); Y_ENSURE(ctx); - TUserDataTable mounts = GetYqlDefaultMounts(); + TUserDataTable mounts = GetYqlDefaultMounts(); TModulesTable modulesTable; if (!CompileLibraries(mounts, *ctx, modulesTable, optimizeLibraries)) { - return {}; + return {}; } for (const auto& item : userData) { diff --git a/ydb/library/yql/core/services/mounts/yql_mounts.h b/ydb/library/yql/core/services/mounts/yql_mounts.h index 27b30c4ccd..ef7e720eea 100644 --- a/ydb/library/yql/core/services/mounts/yql_mounts.h +++ b/ydb/library/yql/core/services/mounts/yql_mounts.h @@ -5,7 +5,7 @@ namespace NYql { -TUserDataTable GetYqlDefaultMounts(); +TUserDataTable GetYqlDefaultMounts(); bool GetYqlDefaultModuleResolver( TExprContext& ctx, @@ -18,7 +18,7 @@ bool GetYqlDefaultModuleResolverWithContext( const THashMap<TString, TString>& clusterMapping = {}, bool optimizeLibraries = true); -TUserDataTable GetYqlModuleResolver( +TUserDataTable GetYqlModuleResolver( TExprContext& ctx, IModuleResolver::TPtr& moduleResolver, const TVector<NUserData::TUserData>& userData, diff --git a/ydb/library/yql/core/services/yql_eval_expr.cpp b/ydb/library/yql/core/services/yql_eval_expr.cpp index 811f22a605..7d0f7efa13 100644 --- a/ydb/library/yql/core/services/yql_eval_expr.cpp +++ b/ydb/library/yql/core/services/yql_eval_expr.cpp @@ -1033,7 +1033,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp TMemoryUsageInfo memInfo("Eval"); THolderFactory holderFactory(alloc.Ref(), memInfo); - auto value = NCommon::ParseYsonNodeInResultFormat(holderFactory, dataNode, mkqlType, &err); + auto value = NCommon::ParseYsonNodeInResultFormat(holderFactory, dataNode, mkqlType, &err); if (!value) { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to parse data: " << err.Str())); return nullptr; diff --git a/ydb/library/yql/core/services/yql_eval_params.cpp b/ydb/library/yql/core/services/yql_eval_params.cpp index dce9df4892..8ec8fbbb18 100644 --- a/ydb/library/yql/core/services/yql_eval_params.cpp +++ b/ydb/library/yql/core/services/yql_eval_params.cpp @@ -12,64 +12,64 @@ namespace NYql { using namespace NKikimr; using namespace NKikimr::NMiniKQL; -namespace { - -bool BuildParameterValuesAsNodes(const THashMap<TStringBuf, const TTypeAnnotationNode*>& paramTypes, - const NYT::TNode& paramData, TExprContext& ctx, const IFunctionRegistry& functionRegistry, - THashMap<TStringBuf, TExprNode::TPtr>& paramValues) { - - if (!paramData.IsMap()) { - ctx.AddError(TIssue({}, TStringBuilder() << "ParamData is not a map")); - return false; - } - - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - TMemoryUsageInfo memInfo("Parameters"); - THolderFactory holderFactory(alloc.Ref(), memInfo); - bool isOk = true; - auto& paramDataMap = paramData.AsMap(); - for (auto& p : paramTypes) { - auto name = p.first; - - TStringStream err; - TProgramBuilder pgmBuilder(env, functionRegistry); - TType* mkqlType = NCommon::BuildType(*p.second, pgmBuilder, err); - if (!mkqlType) { - ctx.AddError(TIssue({}, TStringBuilder() << "Failed to process type for parameter: " << name << ", reason: " << err.Str())); - isOk = false; - continue; - } - - const auto parameterItem = paramDataMap.FindPtr(name); - if (parameterItem && (!parameterItem->IsMap() || !parameterItem->HasKey("Data"))) { - ctx.AddError(TIssue({}, TStringBuilder() << "Parameter '" << name << "' value should be a map with key 'Data'")); - isOk = false; - continue; - } - +namespace { + +bool BuildParameterValuesAsNodes(const THashMap<TStringBuf, const TTypeAnnotationNode*>& paramTypes, + const NYT::TNode& paramData, TExprContext& ctx, const IFunctionRegistry& functionRegistry, + THashMap<TStringBuf, TExprNode::TPtr>& paramValues) { + + if (!paramData.IsMap()) { + ctx.AddError(TIssue({}, TStringBuilder() << "ParamData is not a map")); + return false; + } + + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + TMemoryUsageInfo memInfo("Parameters"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + bool isOk = true; + auto& paramDataMap = paramData.AsMap(); + for (auto& p : paramTypes) { + auto name = p.first; + + TStringStream err; + TProgramBuilder pgmBuilder(env, functionRegistry); + TType* mkqlType = NCommon::BuildType(*p.second, pgmBuilder, err); + if (!mkqlType) { + ctx.AddError(TIssue({}, TStringBuilder() << "Failed to process type for parameter: " << name << ", reason: " << err.Str())); + isOk = false; + continue; + } + + const auto parameterItem = paramDataMap.FindPtr(name); + if (parameterItem && (!parameterItem->IsMap() || !parameterItem->HasKey("Data"))) { + ctx.AddError(TIssue({}, TStringBuilder() << "Parameter '" << name << "' value should be a map with key 'Data'")); + isOk = false; + continue; + } + if (!parameterItem && p.second->GetKind() != ETypeAnnotationKind::Optional && p.second->GetKind() != ETypeAnnotationKind::Null) { - ctx.AddError(TIssue({}, TStringBuilder() << "Missing value for parameter: " << name)); - isOk = false; - continue; - } - - auto value = parameterItem ? NCommon::ParseYsonNodeInResultFormat(holderFactory, (*parameterItem)["Data"], mkqlType, &err) : MakeMaybe(NUdf::TUnboxedValue()); - if (!value) { - ctx.AddError(TIssue({}, TStringBuilder() << "Failed to parse data for parameter: " << name << ", reason: " << err.Str())); - isOk = false; - continue; - } - + ctx.AddError(TIssue({}, TStringBuilder() << "Missing value for parameter: " << name)); + isOk = false; + continue; + } + + auto value = parameterItem ? NCommon::ParseYsonNodeInResultFormat(holderFactory, (*parameterItem)["Data"], mkqlType, &err) : MakeMaybe(NUdf::TUnboxedValue()); + if (!value) { + ctx.AddError(TIssue({}, TStringBuilder() << "Failed to parse data for parameter: " << name << ", reason: " << err.Str())); + isOk = false; + continue; + } + paramValues[name] = NCommon::ValueToExprLiteral(p.second, *value, ctx); } - return isOk; -} - -bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& types, - TExprContext& ctx, THashMap<TStringBuf, const TTypeAnnotationNode*>& paramTypes) { - + return isOk; +} + +bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& types, + TExprContext& ctx, THashMap<TStringBuf, const TTypeAnnotationNode*>& paramTypes) { + auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(types); auto typeTransformer = CreateTypeAnnotationTransformer(callableTransformer, types); TVector<TTransformStage> transformers; @@ -79,14 +79,14 @@ bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& TOptimizeExprSettings settings(nullptr); settings.VisitChanges = true; - TExprNode::TPtr output = input; - auto status1 = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx)->TExprNode::TPtr { + TExprNode::TPtr output = input; + auto status1 = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx)->TExprNode::TPtr { if (!node->IsCallable("Parameter")) { return node; } auto param = node; - fullTransformer->Rewind(); + fullTransformer->Rewind(); auto status = InstantTransform(*fullTransformer, param, ctx); if (status.Level == IGraphTransformer::TStatus::Error) { return nullptr; @@ -110,59 +110,59 @@ bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& return param; }, ctx, settings); - return status1.Level == IGraphTransformer::TStatus::Ok; -} - -} + return status1.Level == IGraphTransformer::TStatus::Ok; +} -bool ExtractParametersMetaAsYson(const TExprNode::TPtr& input, TTypeAnnotationContext& types, - TExprContext& ctx, NYT::TNode& paramsMetaMap) { +} - THashMap<TStringBuf, const TTypeAnnotationNode*> params; - if (!ExtractParameterTypes(input, types, ctx, params)) { - return false; - } +bool ExtractParametersMetaAsYson(const TExprNode::TPtr& input, TTypeAnnotationContext& types, + TExprContext& ctx, NYT::TNode& paramsMetaMap) { - for (auto& p : params) { - paramsMetaMap[p.first] = NCommon::TypeToYsonNode(p.second); - } - return true; -} + THashMap<TStringBuf, const TTypeAnnotationNode*> params; + if (!ExtractParameterTypes(input, types, ctx, params)) { + return false; + } -IGraphTransformer::TStatus EvaluateParameters(const TExprNode::TPtr& input, TExprNode::TPtr& output, - TTypeAnnotationContext& types, TExprContext& ctx, const IFunctionRegistry& functionRegistry) { - output = input; + for (auto& p : params) { + paramsMetaMap[p.first] = NCommon::TypeToYsonNode(p.second); + } + return true; +} - THashMap<TStringBuf, const TTypeAnnotationNode*> paramTypes; - if (!ExtractParameterTypes(input, types, ctx, paramTypes)) { - return IGraphTransformer::TStatus::Error; - } +IGraphTransformer::TStatus EvaluateParameters(const TExprNode::TPtr& input, TExprNode::TPtr& output, + TTypeAnnotationContext& types, TExprContext& ctx, const IFunctionRegistry& functionRegistry) { + output = input; - if (!paramTypes) { - // no params - just exit - return IGraphTransformer::TStatus::Ok; + THashMap<TStringBuf, const TTypeAnnotationNode*> paramTypes; + if (!ExtractParameterTypes(input, types, ctx, paramTypes)) { + return IGraphTransformer::TStatus::Error; } - const auto emptyMapNode = NYT::TNode::CreateMap(); - const auto& paramData = types.OperationOptions.ParametersYson.GetOrElse(emptyMapNode); - THashMap<TStringBuf, TExprNode::TPtr> paramValues; - if (!BuildParameterValuesAsNodes(paramTypes, paramData, ctx, functionRegistry, paramValues)) { - return IGraphTransformer::TStatus::Error; + if (!paramTypes) { + // no params - just exit + return IGraphTransformer::TStatus::Ok; + } + + const auto emptyMapNode = NYT::TNode::CreateMap(); + const auto& paramData = types.OperationOptions.ParametersYson.GetOrElse(emptyMapNode); + THashMap<TStringBuf, TExprNode::TPtr> paramValues; + if (!BuildParameterValuesAsNodes(paramTypes, paramData, ctx, functionRegistry, paramValues)) { + return IGraphTransformer::TStatus::Error; } - // inject param values into graph - TOptimizeExprSettings settings(nullptr); - settings.VisitChanges = true; - auto status = OptimizeExpr(output, output, [&](const TExprNode::TPtr& node, TExprContext& ctx)->TExprNode::TPtr { + // inject param values into graph + TOptimizeExprSettings settings(nullptr); + settings.VisitChanges = true; + auto status = OptimizeExpr(output, output, [&](const TExprNode::TPtr& node, TExprContext& ctx)->TExprNode::TPtr { Y_UNUSED(ctx); if (!node->IsCallable("Parameter")) { return node; } auto name = node->Child(0)->Content(); - auto evaluated = paramValues.FindPtr(name); + auto evaluated = paramValues.FindPtr(name); YQL_ENSURE(evaluated, "Missing parameter value: " << name); - return *evaluated; + return *evaluated; }, ctx, settings); if (status.Level == IGraphTransformer::TStatus::Error) { diff --git a/ydb/library/yql/core/services/yql_eval_params.h b/ydb/library/yql/core/services/yql_eval_params.h index 681f61d2c0..6ec2723a2f 100644 --- a/ydb/library/yql/core/services/yql_eval_params.h +++ b/ydb/library/yql/core/services/yql_eval_params.h @@ -18,6 +18,6 @@ namespace NYql { IGraphTransformer::TStatus EvaluateParameters(const TExprNode::TPtr& input, TExprNode::TPtr& output, TTypeAnnotationContext& types, TExprContext& ctx, const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry); -bool ExtractParametersMetaAsYson(const TExprNode::TPtr& input, TTypeAnnotationContext& types, TExprContext& ctx, NYT::TNode& paramsMetaMap); - +bool ExtractParametersMetaAsYson(const TExprNode::TPtr& input, TTypeAnnotationContext& types, TExprContext& ctx, NYT::TNode& paramsMetaMap); + } diff --git a/ydb/library/yql/core/services/yql_plan.cpp b/ydb/library/yql/core/services/yql_plan.cpp index 4d7cfffaf5..adb949c2e2 100644 --- a/ydb/library/yql/core/services/yql_plan.cpp +++ b/ydb/library/yql/core/services/yql_plan.cpp @@ -66,7 +66,7 @@ struct TPinKey { } struct THash { - size_t operator()(const TPinKey& x) const { + size_t operator()(const TPinKey& x) const { return x.Hash(); } }; diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 5846e6cb10..19fd543347 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -428,35 +428,35 @@ namespace NTypeAnnImpl { : TContext(expr) , Types(types) {} - bool TExtContext::LoadUdfMetadata(const TVector<IUdfResolver::TFunction*>& functions) { - TVector<IUdfResolver::TImport*> imports; - imports.reserve(Types.UdfImports.size()); - for (auto& x : Types.UdfImports) { - imports.push_back(&x.second); - } - - if (!Types.UdfResolver->LoadMetadata(imports, functions, Expr)) { - return false; - } - - for (auto& import : imports) { - RegisterResolvedImport(*import); - } - - return true; - } - - void TExtContext::RegisterResolvedImport(const IUdfResolver::TImport& import) { - YQL_ENSURE(import.Modules); - for (auto& m : *import.Modules) { - auto p = Types.UdfModules.emplace(m, import.FileAlias); - // rework this place when user tries to override another module - if (!p.second && p.first->second != import.FileAlias) { - ythrow yexception() << "Module name duplicated : module = " << m << ", existing alias = " << p.first->second << ", new alis = " << import.FileAlias; - } - } - } - + bool TExtContext::LoadUdfMetadata(const TVector<IUdfResolver::TFunction*>& functions) { + TVector<IUdfResolver::TImport*> imports; + imports.reserve(Types.UdfImports.size()); + for (auto& x : Types.UdfImports) { + imports.push_back(&x.second); + } + + if (!Types.UdfResolver->LoadMetadata(imports, functions, Expr)) { + return false; + } + + for (auto& import : imports) { + RegisterResolvedImport(*import); + } + + return true; + } + + void TExtContext::RegisterResolvedImport(const IUdfResolver::TImport& import) { + YQL_ENSURE(import.Modules); + for (auto& m : *import.Modules) { + auto p = Types.UdfModules.emplace(m, import.FileAlias); + // rework this place when user tries to override another module + if (!p.second && p.first->second != import.FileAlias) { + ythrow yexception() << "Module name duplicated : module = " << m << ", existing alias = " << p.first->second << ", new alis = " << import.FileAlias; + } + } + } + bool EnsureJsonQueryFunction(const NNodes::TCoJsonQueryBase& function, TContext& ctx) { // first argument must be "Json", "Json?", "JsonDocument" or "JsonDocument?" type const auto& jsonArg = function.Json().Ref(); @@ -6579,31 +6579,31 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - bool ValidateFileAlias(const TExprNode& aliasNode, TExtContext& ctx) { + bool ValidateFileAlias(const TExprNode& aliasNode, TExtContext& ctx) { if (!EnsureAtom(aliasNode, ctx.Expr)) { - return false; + return false; } const auto content = aliasNode.Content(); - if (!ctx.Types.UserDataStorage->ContainsUserDataBlock(content)) { + if (!ctx.Types.UserDataStorage->ContainsUserDataBlock(content)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "File not found: " << content)); - return false; + return false; } - return true; + return true; } - bool ValidateFolderAlias(const TExprNode& aliasNode, TExtContext& ctx) { + bool ValidateFolderAlias(const TExprNode& aliasNode, TExtContext& ctx) { if (!EnsureAtom(aliasNode, ctx.Expr)) { - return false; + return false; } const auto content = aliasNode.Content(); - if (!ctx.Types.UserDataStorage->ContainsUserDataFolder(content)) { + if (!ctx.Types.UserDataStorage->ContainsUserDataFolder(content)) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "Folder not found: " << content)); - return false; + return false; } - return true; + return true; } IGraphTransformer::TStatus FilePathWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { @@ -7041,11 +7041,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto type = input->Head().GetTypeAnn()->Cast<TCallableExprType>(); auto autoMapFunction = type->GetReturnType()->GetKind() == ETypeAnnotationKind::Optional ? "FlatMap" : "Map"; auto tmpArg = ctx.Expr.NewArgument(input->Pos(), "tmp"); - if (!EnsureCallableMaxArgsCount(input->Pos(), input->ChildrenSize() - 1, type->GetArgumentsSize(), ctx.Expr)) { + if (!EnsureCallableMaxArgsCount(input->Pos(), input->ChildrenSize() - 1, type->GetArgumentsSize(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (!EnsureCallableMinArgsCount(input->Pos(), input->ChildrenSize() - 1, type->GetArgumentsSize() - type->GetOptionalArgumentsCount(), ctx.Expr)) { + if (!EnsureCallableMinArgsCount(input->Pos(), input->ChildrenSize() - 1, type->GetArgumentsSize() - type->GetOptionalArgumentsCount(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/type_ann/type_ann_impl.h b/ydb/library/yql/core/type_ann/type_ann_impl.h index 3d940ebd73..98cb57dce2 100644 --- a/ydb/library/yql/core/type_ann/type_ann_impl.h +++ b/ydb/library/yql/core/type_ann/type_ann_impl.h @@ -20,8 +20,8 @@ namespace NTypeAnnImpl { TTypeAnnotationContext& Types; TExtContext(TExprContext& expr, TTypeAnnotationContext& types); - bool LoadUdfMetadata(const TVector<IUdfResolver::TFunction*>& functions); - void RegisterResolvedImport(const IUdfResolver::TImport& import); + bool LoadUdfMetadata(const TVector<IUdfResolver::TFunction*>& functions); + void RegisterResolvedImport(const IUdfResolver::TImport& import); }; // Implemented in type_ann_join.cpp diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make index 29169996f5..237a563948 100644 --- a/ydb/library/yql/core/ya.make +++ b/ydb/library/yql/core/ya.make @@ -27,8 +27,8 @@ SRCS( yql_gc_transformer.h yql_graph_transformer.cpp yql_graph_transformer.h - yql_holding_file_storage.cpp - yql_holding_file_storage.h + yql_holding_file_storage.cpp + yql_holding_file_storage.h yql_join.cpp yql_join.h yql_library_compiler.cpp @@ -47,16 +47,16 @@ SRCS( yql_type_annotation.h yql_type_helpers.cpp yql_type_helpers.h - yql_udf_index.cpp - yql_udf_index.h - yql_udf_index_package_set.cpp - yql_udf_index_package_set.h - yql_udf_resolver.cpp - yql_udf_resolver.h - yql_user_data.cpp - yql_user_data.h - yql_user_data_storage.cpp - yql_user_data_storage.h + yql_udf_index.cpp + yql_udf_index.h + yql_udf_index_package_set.cpp + yql_udf_index_package_set.h + yql_udf_resolver.cpp + yql_udf_resolver.h + yql_user_data.cpp + yql_user_data.h + yql_user_data_storage.cpp + yql_user_data_storage.h ) PEERDIR( @@ -82,7 +82,7 @@ PEERDIR( GENERATE_ENUM_SERIALIZATION(yql_data_provider.h) -GENERATE_ENUM_SERIALIZATION(yql_user_data.h) +GENERATE_ENUM_SERIALIZATION(yql_user_data.h) GENERATE_ENUM_SERIALIZATION(yql_atom_enums.h) diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h index 3ffbdc041d..69d9d49c68 100644 --- a/ydb/library/yql/core/yql_data_provider.h +++ b/ydb/library/yql/core/yql_data_provider.h @@ -223,10 +223,10 @@ struct TDataProviderInfo { std::function<bool()> HasActiveProcesses; std::function<void(const TString& sessionId)> CloseSession; - + std::function<void(const TString& sessionId)> CleanupSession; - std::function<TMaybe<TString>(const TString& url)> TokenResolver; + std::function<TMaybe<TString>(const TString& url)> TokenResolver; }; using TDataProviderInitializer = std::function<TDataProviderInfo( diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index f2b793af8d..8f2edb4495 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -1824,23 +1824,23 @@ bool EnsureMinMaxArgsCount(const TExprNode& node, ui32 minArgs, ui32 maxArgs, TE } bool EnsureCallableMinArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { - if (args < expectedArgs) { + if (args < expectedArgs) { ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at least " << expectedArgs << " argument(s), but got " << args)); - return false; - } - - return true; -} - + return false; + } + + return true; +} + bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { - if (args > expectedArgs) { + if (args > expectedArgs) { ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at most " << expectedArgs << " argument(s), but got " << args)); - return false; - } - - return true; -} - + return false; + } + + return true; +} + bool EnsureAtom(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Atom) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected atom, but got: " << node.Type())); @@ -4624,7 +4624,7 @@ TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& return res; } -TExprNode::TPtr ExpandTypeNoCache(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +TExprNode::TPtr ExpandTypeNoCache(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { switch (type.GetKind()) { case ETypeAnnotationKind::Unit: return ctx.NewCallable(position, "UnitType", {}); @@ -4805,14 +4805,14 @@ TExprNode::TPtr ExpandTypeNoCache(TPositionHandle position, const TTypeAnnotatio } } -TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { - TExprNode::TPtr& ret = ctx.TypeAsNodeCache[&type]; - if (!ret) { - ret = ExpandTypeNoCache(position, type, ctx); - } - return ret; -} - +TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { + TExprNode::TPtr& ret = ctx.TypeAsNodeCache[&type]; + if (!ret) { + ret = ExpandTypeNoCache(position, type, ctx); + } + return ret; +} + bool IsSystemMember(const TStringBuf& memberName) { return memberName.StartsWith(TStringBuf("_yql_")); } diff --git a/ydb/library/yql/core/yql_holding_file_storage.cpp b/ydb/library/yql/core/yql_holding_file_storage.cpp index 865dfc5880..ce6c236744 100644 --- a/ydb/library/yql/core/yql_holding_file_storage.cpp +++ b/ydb/library/yql/core/yql_holding_file_storage.cpp @@ -1,113 +1,113 @@ -#include "yql_holding_file_storage.h" +#include "yql_holding_file_storage.h" #include <ydb/library/yql/utils/future_action.h> - -namespace NYql { - -THoldingFileStorage::THoldingFileStorage(TFileStoragePtr fileStorage) - : FileStorage_(std::move(fileStorage)) -{ - // FileStorage_ can be nullptr -} - -TFileStoragePtr THoldingFileStorage::GetRawStorage() const { - return FileStorage_; -} - -bool THoldingFileStorage::CanFreeze() const { - return FileStorage_ != nullptr; -} - -TFileLinkPtr THoldingFileStorage::FreezeFile(const TUserDataBlock& block) { - if (block.FrozenFile) { - return block.FrozenFile; - } - - return FindOrPutData(block); -} - -NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::FreezeFileAsync(const TUserDataBlock& block) { - if (block.FrozenFile) { - return MakeFutureWithConstantAction(block.FrozenFile); - } - - return FindOrPutDataAsync(block); -} - -TFileLinkPtr THoldingFileStorage::FindOrPutData(const TUserDataBlock& block) { - auto it = Links_.find(block); - if (it != Links_.end()) { - return it->second; - } - - return PutDataAndRegister(block); -} - -TFileLinkPtr THoldingFileStorage::PutDataAndRegister(const TUserDataBlock& block) { - // download outside of the lock - TFileLinkPtr link = PutData(block); - return RegisterLink(block, link); -} - -TFileLinkPtr THoldingFileStorage::RegisterLink(const TUserDataBlock& block, TFileLinkPtr link) { - // important: do not overwrite existing value if any and return it - return Links_.emplace(block, link).first->second; -} - -TFileLinkPtr THoldingFileStorage::PutData(const TUserDataBlock& block) { - if (!FileStorage_) { - ythrow yexception() << "FileStorage is not available, unable to load " << block.Data; - } - - switch (block.Type) { - case EUserDataType::PATH: - return FileStorage_->PutFile(block.Data); - - case EUserDataType::URL: - return FileStorage_->PutUrl(block.Data, block.UrlToken); - - case EUserDataType::RAW_INLINE_DATA: - return FileStorage_->PutInline(block.Data); - - default: - ythrow yexception() << "Unknown user data type " << block.Type; - } -} - -NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::FindOrPutDataAsync(const TUserDataBlock& block) { - auto it = Links_.find(block); - if (it != Links_.end()) { - return MakeFutureWithConstantAction(it->second); - } - - return PutDataAndRegisterAsync(block); -} - -NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::PutDataAndRegisterAsync(const TUserDataBlock& block) { - NThreading::TFuture<TFileLinkPtr> future = PutDataAsync(block); - - return AddActionToFuture(future, [this, block](TFileLinkPtr link) { - return this->RegisterLink(block, link); - }); -} - -NThreading::TFuture<TFileLinkPtr> THoldingFileStorage::PutDataAsync(const TUserDataBlock& block) { - if (!FileStorage_) { - ythrow yexception() << "FileStorage is not available, unable to load " << block.Data; - } - - switch (block.Type) { - case EUserDataType::PATH: - return FileStorage_->PutFileAsync(block.Data); - - case EUserDataType::URL: - return FileStorage_->PutUrlAsync(block.Data, block.UrlToken); - - case EUserDataType::RAW_INLINE_DATA: - return FileStorage_->PutInlineAsync(block.Data); - - default: - ythrow yexception() << "Unknown user data type " << block.Type; - } -} - -} + +namespace NYql { + +THoldingFileStorage::THoldingFileStorage(TFileStoragePtr fileStorage) + : FileStorage_(std::move(fileStorage)) +{ + // FileStorage_ can be nullptr +} + +TFileStoragePtr THoldingFileStorage::GetRawStorage() const { + return FileStorage_; +} + +bool THoldingFileStorage::CanFreeze() const { + return FileStorage_ != nullptr; +} + +TFileLinkPtr THoldingFileStorage::FreezeFile(const TUserDataBlock& block) { + if (block.FrozenFile) { + return block.FrozenFile; + } + + return FindOrPutData(block); +} + +NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::FreezeFileAsync(const TUserDataBlock& block) { + if (block.FrozenFile) { + return MakeFutureWithConstantAction(block.FrozenFile); + } + + return FindOrPutDataAsync(block); +} + +TFileLinkPtr THoldingFileStorage::FindOrPutData(const TUserDataBlock& block) { + auto it = Links_.find(block); + if (it != Links_.end()) { + return it->second; + } + + return PutDataAndRegister(block); +} + +TFileLinkPtr THoldingFileStorage::PutDataAndRegister(const TUserDataBlock& block) { + // download outside of the lock + TFileLinkPtr link = PutData(block); + return RegisterLink(block, link); +} + +TFileLinkPtr THoldingFileStorage::RegisterLink(const TUserDataBlock& block, TFileLinkPtr link) { + // important: do not overwrite existing value if any and return it + return Links_.emplace(block, link).first->second; +} + +TFileLinkPtr THoldingFileStorage::PutData(const TUserDataBlock& block) { + if (!FileStorage_) { + ythrow yexception() << "FileStorage is not available, unable to load " << block.Data; + } + + switch (block.Type) { + case EUserDataType::PATH: + return FileStorage_->PutFile(block.Data); + + case EUserDataType::URL: + return FileStorage_->PutUrl(block.Data, block.UrlToken); + + case EUserDataType::RAW_INLINE_DATA: + return FileStorage_->PutInline(block.Data); + + default: + ythrow yexception() << "Unknown user data type " << block.Type; + } +} + +NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::FindOrPutDataAsync(const TUserDataBlock& block) { + auto it = Links_.find(block); + if (it != Links_.end()) { + return MakeFutureWithConstantAction(it->second); + } + + return PutDataAndRegisterAsync(block); +} + +NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::PutDataAndRegisterAsync(const TUserDataBlock& block) { + NThreading::TFuture<TFileLinkPtr> future = PutDataAsync(block); + + return AddActionToFuture(future, [this, block](TFileLinkPtr link) { + return this->RegisterLink(block, link); + }); +} + +NThreading::TFuture<TFileLinkPtr> THoldingFileStorage::PutDataAsync(const TUserDataBlock& block) { + if (!FileStorage_) { + ythrow yexception() << "FileStorage is not available, unable to load " << block.Data; + } + + switch (block.Type) { + case EUserDataType::PATH: + return FileStorage_->PutFileAsync(block.Data); + + case EUserDataType::URL: + return FileStorage_->PutUrlAsync(block.Data, block.UrlToken); + + case EUserDataType::RAW_INLINE_DATA: + return FileStorage_->PutInlineAsync(block.Data); + + default: + ythrow yexception() << "Unknown user data type " << block.Type; + } +} + +} diff --git a/ydb/library/yql/core/yql_holding_file_storage.h b/ydb/library/yql/core/yql_holding_file_storage.h index 39178a36dc..8c388a51bc 100644 --- a/ydb/library/yql/core/yql_holding_file_storage.h +++ b/ydb/library/yql/core/yql_holding_file_storage.h @@ -1,45 +1,45 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_user_data.h> #include <ydb/library/yql/core/file_storage/file_storage.h> -#include <util/generic/map.h> - -namespace NYql { - -class THoldingFileStorage { -private: - struct TDataLess { - bool operator()(const TUserDataBlock& b1, const TUserDataBlock& b2) const { - return std::tie(b1.Type, b1.Data, b1.UrlToken) < std::tie(b2.Type, b2.Data, b2.UrlToken); - } - }; - -public: - explicit THoldingFileStorage(TFileStoragePtr fileStorage); - TFileStoragePtr GetRawStorage() const; - bool CanFreeze() const; - // it is safe to call this method from multiple threads - // method will return the same link for single path even in multi-threaded env - // if we have single url but with different tokens we can potentially download several different versions of url - TFileLinkPtr FreezeFile(const TUserDataBlock& block); - - // downloaded file will be registered in cache after function invocation - NThreading::TFuture<std::function<TFileLinkPtr()>> FreezeFileAsync(const TUserDataBlock& block); - -private: - TFileLinkPtr PutDataAndRegister(const TUserDataBlock& block); - TFileLinkPtr PutData(const TUserDataBlock& block); - TFileLinkPtr FindOrPutData(const TUserDataBlock& block); - - NThreading::TFuture<std::function<TFileLinkPtr()>> PutDataAndRegisterAsync(const TUserDataBlock& block); - NThreading::TFuture<TFileLinkPtr> PutDataAsync(const TUserDataBlock& block); - NThreading::TFuture<std::function<TFileLinkPtr()>> FindOrPutDataAsync(const TUserDataBlock& block); - - TFileLinkPtr RegisterLink(const TUserDataBlock& block, TFileLinkPtr link); - -private: - TFileStoragePtr FileStorage_; - TMap<TUserDataBlock, TFileLinkPtr, TDataLess> Links_; -}; - -} +#include <util/generic/map.h> + +namespace NYql { + +class THoldingFileStorage { +private: + struct TDataLess { + bool operator()(const TUserDataBlock& b1, const TUserDataBlock& b2) const { + return std::tie(b1.Type, b1.Data, b1.UrlToken) < std::tie(b2.Type, b2.Data, b2.UrlToken); + } + }; + +public: + explicit THoldingFileStorage(TFileStoragePtr fileStorage); + TFileStoragePtr GetRawStorage() const; + bool CanFreeze() const; + // it is safe to call this method from multiple threads + // method will return the same link for single path even in multi-threaded env + // if we have single url but with different tokens we can potentially download several different versions of url + TFileLinkPtr FreezeFile(const TUserDataBlock& block); + + // downloaded file will be registered in cache after function invocation + NThreading::TFuture<std::function<TFileLinkPtr()>> FreezeFileAsync(const TUserDataBlock& block); + +private: + TFileLinkPtr PutDataAndRegister(const TUserDataBlock& block); + TFileLinkPtr PutData(const TUserDataBlock& block); + TFileLinkPtr FindOrPutData(const TUserDataBlock& block); + + NThreading::TFuture<std::function<TFileLinkPtr()>> PutDataAndRegisterAsync(const TUserDataBlock& block); + NThreading::TFuture<TFileLinkPtr> PutDataAsync(const TUserDataBlock& block); + NThreading::TFuture<std::function<TFileLinkPtr()>> FindOrPutDataAsync(const TUserDataBlock& block); + + TFileLinkPtr RegisterLink(const TUserDataBlock& block, TFileLinkPtr link); + +private: + TFileStoragePtr FileStorage_; + TMap<TUserDataBlock, TFileLinkPtr, TDataLess> Links_; +}; + +} diff --git a/ydb/library/yql/core/yql_library_compiler.cpp b/ydb/library/yql/core/yql_library_compiler.cpp index 79296792e6..f0de46292b 100644 --- a/ydb/library/yql/core/yql_library_compiler.cpp +++ b/ydb/library/yql/core/yql_library_compiler.cpp @@ -13,25 +13,25 @@ namespace { bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges, TNodeSet& visited, TNodeSet& parents) { if (!node.ChildrenSize()) { - return true; - } + return true; + } const auto pair = parents.emplace(&node); - if (!pair.second) { + if (!pair.second) { return false; - } - - if (!visited.emplace(&node).second) { - parents.erase(pair.first); - return true; - } + } + if (!visited.emplace(&node).second) { + parents.erase(pair.first); + return true; + } + for (ui32 i = 0U; i < node.ChildrenSize(); ++i) { auto& child = node.ChildRef(i); if (const auto it = replaces.find(node.Child(i)); replaces.cend() != it) { child = it->second; hasChanges = true; - } + } if (!ReplaceNodes(*node.Child(i), replaces, hasChanges, visited, parents)) { child.Reset(); @@ -112,17 +112,17 @@ bool CompileLibrary(const TString& alias, const TString& script, TExprContext& c } bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const TModulesTable* loadedModules) { - std::function<const TExportTable*(const TString&)> f = [loadedModules](const TString& normalizedModuleName) -> const TExportTable* { - if (!loadedModules) { - return nullptr; - } - - return loadedModules->FindPtr(normalizedModuleName); - }; - - return LinkLibraries(libs, ctx, ctxToClone, f); -} - + std::function<const TExportTable*(const TString&)> f = [loadedModules](const TString& normalizedModuleName) -> const TExportTable* { + if (!loadedModules) { + return nullptr; + } + + return loadedModules->FindPtr(normalizedModuleName); + }; + + return LinkLibraries(libs, ctx, ctxToClone, f); +} + bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const std::function<const TExportTable*(const TString&)>& module2ExportTable) { TNodeOnNodeOwnedMap clones, replaces; diff --git a/ydb/library/yql/core/yql_library_compiler.h b/ydb/library/yql/core/yql_library_compiler.h index eb6abb83a5..94162177ca 100644 --- a/ydb/library/yql/core/yql_library_compiler.h +++ b/ydb/library/yql/core/yql_library_compiler.h @@ -7,7 +7,7 @@ namespace NYql { bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx); bool CompileLibrary(const TString& alias, const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize = true); - + bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const std::function<const TExportTable*(const TString&)>& module2ExportTable); bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const TModulesTable* loadedModules = nullptr); diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp index 9f50af3b63..d04d6dadb1 100644 --- a/ydb/library/yql/core/yql_type_annotation.cpp +++ b/ydb/library/yql/core/yql_type_annotation.cpp @@ -12,19 +12,19 @@ #include <util/stream/file.h> #include <util/string/join.h> -namespace NYql { +namespace NYql { using namespace NKikimr; -bool TTypeAnnotationContext::Initialize(TExprContext& ctx) { - if (!InitializeResult) { - InitializeResult = DoInitialize(ctx); +bool TTypeAnnotationContext::Initialize(TExprContext& ctx) { + if (!InitializeResult) { + InitializeResult = DoInitialize(ctx); } - + return *InitializeResult; } -bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) { +bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) { for (auto& x : DataSources) { if (!x->Initialize(ctx)) { return false; @@ -37,9 +37,9 @@ bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) { } } - Y_ENSURE(UserDataStorage); - UserDataStorage->FillUserDataTokens(); - + Y_ENSURE(UserDataStorage); + UserDataStorage->FillUserDataTokens(); + // Disable "in progress" constraints //DisableConstraintCheck.emplace(TSortedConstraintNode::Name()); //DisableConstraintCheck.emplace(TEmptyConstraintNode::Name()); @@ -147,18 +147,18 @@ const TCredential* TTypeAnnotationContext::FindCredential(const TStringBuf& name return nullptr; } -TString TTypeAnnotationContext::FindCredentialContent(const TStringBuf& name1, const TStringBuf& name2, const TString& defaultContent) const { - if (auto cred = FindCredential(name1)) { - return cred->Content; - } - - if (auto cred = FindCredential(name2)) { - return cred->Content; - } - - return defaultContent; -} - +TString TTypeAnnotationContext::FindCredentialContent(const TStringBuf& name1, const TStringBuf& name2, const TString& defaultContent) const { + if (auto cred = FindCredential(name1)) { + return cred->Content; + } + + if (auto cred = FindCredential(name2)) { + return cred->Content; + } + + return defaultContent; +} + TString TTypeAnnotationContext::GetDefaultDataSource() const { if (!PureResultDataSource.empty()) { YQL_ENSURE(Find(AvailablePureResultDataSources.begin(), @@ -167,7 +167,7 @@ TString TTypeAnnotationContext::GetDefaultDataSource() const { return PureResultDataSource; } - Y_ENSURE(!AvailablePureResultDataSources.empty()); + Y_ENSURE(!AvailablePureResultDataSources.empty()); return AvailablePureResultDataSources.front(); } @@ -185,28 +185,28 @@ TString TModuleResolver::NormalizeModuleName(const TString& path) { return path; } -void TModuleResolver::RegisterPackage(const TString& package) { - KnownPackages.insert(package); -} - -bool TModuleResolver::SetPackageDefaultVersion(const TString& package, ui32 version) { - if (!KnownPackages.contains(package)) { - return false; - } - PackageVersions[package] = version; - return true; -} - +void TModuleResolver::RegisterPackage(const TString& package) { + KnownPackages.insert(package); +} + +bool TModuleResolver::SetPackageDefaultVersion(const TString& package, ui32 version) { + if (!KnownPackages.contains(package)) { + return false; + } + PackageVersions[package] = version; + return true; +} + const TExportTable* TModuleResolver::GetModule(const TString& module) const { - // ParentModules and Modules should not have common keys - const TString normalizedModuleName = NormalizeModuleName(module); - if (ParentModules) { - if (auto table = ParentModules->FindPtr(normalizedModuleName)) { - return table; - } - } - - return Modules.FindPtr(normalizedModuleName); + // ParentModules and Modules should not have common keys + const TString normalizedModuleName = NormalizeModuleName(module); + if (ParentModules) { + if (auto table = ParentModules->FindPtr(normalizedModuleName)) { + return table; + } + } + + return Modules.FindPtr(normalizedModuleName); } bool TModuleResolver::AddFromUrl(const TStringBuf& file, const TStringBuf& url, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { @@ -224,8 +224,8 @@ bool TModuleResolver::AddFromUrl(const TStringBuf& file, const TStringBuf& url, return AddFromFile(file, ctx, syntaxVersion, packageVersion); } -bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { - if (!UserData) { +bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { + if (!UserData) { ctx.AddError(TIssue(TPosition(), "Loading libraries is prohibited")); return false; } @@ -238,7 +238,7 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1 return false; } - const TUserDataBlock* block = UserData->FindUserDataBlock(fullName); + const TUserDataBlock* block = UserData->FindUserDataBlock(fullName); if (!block) { ctx.AddError(TIssue(TStringBuilder() << "File not found: " << file)); @@ -247,66 +247,66 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1 auto moduleName = TModuleResolver::NormalizeModuleName(TString(file)); if (GetModule(moduleName) || Libs.contains(moduleName)) { - auto it = Libs.find(moduleName); - if (it != Libs.end() && it->second.contains(packageVersion)) { - // TODO (YQL-7170): find better fix - // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); - return true; // false - } + auto it = Libs.find(moduleName); + if (it != Libs.end() && it->second.contains(packageVersion)) { + // TODO (YQL-7170): find better fix + // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); + return true; // false + } } TString body; - switch (block->Type) { - case EUserDataType::RAW_INLINE_DATA: + switch (block->Type) { + case EUserDataType::RAW_INLINE_DATA: body = block->Data; - break; - case EUserDataType::PATH: + break; + case EUserDataType::PATH: body = TFileInput(block->Data).ReadAll(); - break; - case EUserDataType::URL: + break; + case EUserDataType::URL: if (!UrlLoader) { ctx.AddError(TIssue(TStringBuilder() << "Unable to load file \"" << file << "\" from url, because url loader is not available")); return false; } - body = UrlLoader->Load(block->Data, block->UrlToken); - break; - default: - throw yexception() << "Unknown block type " << block->Type; - } - - return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion); -} - -bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { - TString unusedModuleName; - return AddFromMemory(file, body, ctx, syntaxVersion, packageVersion, unusedModuleName); -} - -bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports, std::vector<TString>* imports) { - const auto fullName = TUserDataStorage::MakeFullName(file); - bool isSql = file.EndsWith(".sql"); - bool isYql = file.EndsWith(".yql"); - if (!isSql && !isYql) { + body = UrlLoader->Load(block->Data, block->UrlToken); + break; + default: + throw yexception() << "Unknown block type " << block->Type; + } + + return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion); +} + +bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { + TString unusedModuleName; + return AddFromMemory(file, body, ctx, syntaxVersion, packageVersion, unusedModuleName); +} + +bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports, std::vector<TString>* imports) { + const auto fullName = TUserDataStorage::MakeFullName(file); + bool isSql = file.EndsWith(".sql"); + bool isYql = file.EndsWith(".yql"); + if (!isSql && !isYql) { ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file)); - return false; - } - - moduleName = TModuleResolver::NormalizeModuleName(TString(file)); - if (GetModule(moduleName) || Libs.contains(moduleName)) { - auto it = Libs.find(moduleName); - if (it != Libs.end() && it->second.contains(packageVersion)) { - // TODO (YQL-7170): find better fix - // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); - return true; // false - } - } - - return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion, exports, imports); -} - -bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports, std::vector<TString>* imports) { + return false; + } + + moduleName = TModuleResolver::NormalizeModuleName(TString(file)); + if (GetModule(moduleName) || Libs.contains(moduleName)) { + auto it = Libs.find(moduleName); + if (it != Libs.end() && it->second.contains(packageVersion)) { + // TODO (YQL-7170): find better fix + // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); + return true; // false + } + } + + return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion, exports, imports); +} + +bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports, std::vector<TString>* imports) { TAstParseResult astRes; if (isYql) { astRes = ParseAst(body, nullptr, fullName); @@ -346,90 +346,90 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu } } - if (exports) { - exports->clear(); + if (exports) { + exports->clear(); for (auto p : cohesion.Exports.Symbols()) { - exports->push_back(p.first); - } - } - - if (imports) { - imports->clear(); - for (auto p : cohesion.Imports) { - imports->push_back(p.second.first); - } - } - - Libs[moduleName][packageVersion] = std::move(cohesion); + exports->push_back(p.first); + } + } + + if (imports) { + imports->clear(); + for (auto p : cohesion.Imports) { + imports->push_back(p.second.first); + } + } + + Libs[moduleName][packageVersion] = std::move(cohesion); return true; } bool TModuleResolver::Link(TExprContext& ctx) { - std::function<const TExportTable*(const TString&)> f = [this](const TString& normalizedModuleName) -> const TExportTable* { - return this->GetModule(normalizedModuleName); - }; - - THashMap<TString, TLibraryCohesion> libs = FilterLibsByVersion(); - if (!LinkLibraries(libs, ctx, LibsContext, f)) { + std::function<const TExportTable*(const TString&)> f = [this](const TString& normalizedModuleName) -> const TExportTable* { + return this->GetModule(normalizedModuleName); + }; + + THashMap<TString, TLibraryCohesion> libs = FilterLibsByVersion(); + if (!LinkLibraries(libs, ctx, LibsContext, f)) { return false; } - for (auto& x : libs) { + for (auto& x : libs) { Modules.emplace(x.first, std::move(x.second.Exports)); } Libs.clear(); - PackageVersions.clear(); + PackageVersions.clear(); return true; } -THashMap<TString, TLibraryCohesion> TModuleResolver::FilterLibsByVersion() const { - THashMap<TString, TLibraryCohesion> result; - for (auto p : Libs) { - YQL_ENSURE(!p.second.empty()); - - auto packageName = ExtractPackageNameFromModule(p.first); - if (!packageName) { - YQL_ENSURE(p.second.size() == 1); - result.emplace(p.first, p.second.begin()->second); - continue; - } - - if (!KnownPackages.contains(packageName)) { - ythrow yexception() << "Unknown package " << packageName << " is used in module " << p.first; - } - - auto it = PackageVersions.find(packageName); - const ui32 version = (it != PackageVersions.end()) ? it->second : 0; - auto cohesionIt = p.second.find(version); - if (cohesionIt == p.second.end()) { - ythrow yexception() << "Unable to find library version " << version << " for package " << packageName << " and module " << p.first; - } - result.emplace(p.first, cohesionIt->second); - } - return result; -} - -TString TModuleResolver::ExtractPackageNameFromModule(TStringBuf moduleName) { - // naming convention: pkg.$code_project_name.$code_package_name.$module_name_within_package - // module_name_within_package can contain dots - // function returns $code_project_name.$code_package_name and we call it "package" at worker side - TStringBuf pkg = moduleName.NextTok('/'); - if (pkg != "pkg") { - return ""; - } - - TStringBuf project = moduleName.NextTok('/'); - TStringBuf package = moduleName.NextTok('/'); - if (package.empty()) { - return ""; - } - - return TString(project) + "." + package; -} - +THashMap<TString, TLibraryCohesion> TModuleResolver::FilterLibsByVersion() const { + THashMap<TString, TLibraryCohesion> result; + for (auto p : Libs) { + YQL_ENSURE(!p.second.empty()); + + auto packageName = ExtractPackageNameFromModule(p.first); + if (!packageName) { + YQL_ENSURE(p.second.size() == 1); + result.emplace(p.first, p.second.begin()->second); + continue; + } + + if (!KnownPackages.contains(packageName)) { + ythrow yexception() << "Unknown package " << packageName << " is used in module " << p.first; + } + + auto it = PackageVersions.find(packageName); + const ui32 version = (it != PackageVersions.end()) ? it->second : 0; + auto cohesionIt = p.second.find(version); + if (cohesionIt == p.second.end()) { + ythrow yexception() << "Unable to find library version " << version << " for package " << packageName << " and module " << p.first; + } + result.emplace(p.first, cohesionIt->second); + } + return result; +} + +TString TModuleResolver::ExtractPackageNameFromModule(TStringBuf moduleName) { + // naming convention: pkg.$code_project_name.$code_package_name.$module_name_within_package + // module_name_within_package can contain dots + // function returns $code_project_name.$code_package_name and we call it "package" at worker side + TStringBuf pkg = moduleName.NextTok('/'); + if (pkg != "pkg") { + return ""; + } + + TStringBuf project = moduleName.NextTok('/'); + TStringBuf package = moduleName.NextTok('/'); + if (package.empty()) { + return ""; + } + + return TString(project) + "." + package; +} + void TModuleResolver::UpdateNextUniqueId(TExprContext& ctx) const { - if (UserData && ctx.NextUniqueId < LibsContext.NextUniqueId) { + if (UserData && ctx.NextUniqueId < LibsContext.NextUniqueId) { ctx.NextUniqueId = LibsContext.NextUniqueId; } } @@ -438,14 +438,14 @@ ui64 TModuleResolver::GetNextUniqueId() const { return LibsContext.NextUniqueId; } -IModuleResolver::TPtr TModuleResolver::CreateMutableChild() const { - if (UserData || UrlLoader) { - throw yexception() << "Module resolver should not contain user data and URL loader"; - } - - return std::make_shared<TModuleResolver>(&Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs); -} - +IModuleResolver::TPtr TModuleResolver::CreateMutableChild() const { + if (UserData || UrlLoader) { + throw yexception() << "Module resolver should not contain user data and URL loader"; + } + + return std::make_shared<TModuleResolver>(&Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs); +} + TString TModuleResolver::SubstParameters(const TString& str) { if (!Parameters) { return str; diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h index 1a1f1fb476..23202089bc 100644 --- a/ydb/library/yql/core/yql_type_annotation.h +++ b/ydb/library/yql/core/yql_type_annotation.h @@ -1,9 +1,9 @@ #pragma once #include "yql_data_provider.h" -#include "yql_udf_index_package_set.h" -#include "yql_udf_resolver.h" -#include "yql_user_data_storage.h" +#include "yql_udf_index_package_set.h" +#include "yql_udf_resolver.h" +#include "yql_user_data_storage.h" #include <ydb/library/yql/public/udf/udf_validate.h> @@ -28,7 +28,7 @@ class IUrlLoader : public TThrRefBase { public: ~IUrlLoader() = default; - virtual TString Load(const TString& url, const TString& token) = 0; + virtual TString Load(const TString& url, const TString& token) = 0; using TPtr = TIntrusivePtr<IUrlLoader>; }; @@ -50,21 +50,21 @@ public: } TModuleResolver(const TModulesTable* parentModules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping, - const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString, THashMap<int, TLibraryCohesion>>& libs) - : ParentModules(parentModules) - , LibsContext(nextUniqueId) - , KnownPackages(knownPackages) - , Libs(libs) - , ClusterMapping(clusterMapping) + const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString, THashMap<int, TLibraryCohesion>>& libs) + : ParentModules(parentModules) + , LibsContext(nextUniqueId) + , KnownPackages(knownPackages) + , Libs(libs) + , ClusterMapping(clusterMapping) , SqlFlags(sqlFlags) - , OptimizeLibraries(optimizeLibraries) - { - } - + , OptimizeLibraries(optimizeLibraries) + { + } + static TString NormalizeModuleName(const TString& path); - void AttachUserData(TUserDataStorage::TPtr userData) { - UserData = userData; + void AttachUserData(TUserDataStorage::TPtr userData) { + UserData = userData; } void SetUrlLoader(IUrlLoader::TPtr loader) { @@ -75,34 +75,34 @@ public: Parameters = node; } - void RegisterPackage(const TString& package) override; - bool SetPackageDefaultVersion(const TString& package, ui32 version) override; + void RegisterPackage(const TString& package) override; + bool SetPackageDefaultVersion(const TString& package, ui32 version) override; const TExportTable* GetModule(const TString& module) const override; - bool AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override; + bool AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override; bool AddFromUrl(const TStringBuf& file, const TStringBuf& url, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override; - bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override; - bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) override; + bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override; + bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) override; bool Link(TExprContext& ctx) override; void UpdateNextUniqueId(TExprContext& ctx) const override; ui64 GetNextUniqueId() const override; - IModuleResolver::TPtr CreateMutableChild() const override; + IModuleResolver::TPtr CreateMutableChild() const override; -private: - bool AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr); - THashMap<TString, TLibraryCohesion> FilterLibsByVersion() const; - static TString ExtractPackageNameFromModule(TStringBuf moduleName); +private: + bool AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr); + THashMap<TString, TLibraryCohesion> FilterLibsByVersion() const; + static TString ExtractPackageNameFromModule(TStringBuf moduleName); TString SubstParameters(const TString& str); - -private: + +private: THolder<TExprContext> OwnedCtx; - const TModulesTable* ParentModules = nullptr; - TUserDataStorage::TPtr UserData; + const TModulesTable* ParentModules = nullptr; + TUserDataStorage::TPtr UserData; IUrlLoader::TPtr UrlLoader; TMaybe<NYT::TNode> Parameters; TExprContext LibsContext; - TSet<TString> KnownPackages; - THashMap<TString, ui32> PackageVersions; - THashMap<TString, THashMap<int, TLibraryCohesion>> Libs; + TSet<TString> KnownPackages; + THashMap<TString, ui32> PackageVersions; + THashMap<TString, THashMap<int, TLibraryCohesion>> Libs; TModulesTable Modules; const THashMap<TString, TString> ClusterMapping; const THashSet<TString> SqlFlags; @@ -179,22 +179,22 @@ struct TTypeAnnotationContext: public TThrRefBase { THashMap<TString, TIntrusivePtr<IDataProvider>> DataSinkMap; TVector<TIntrusivePtr<IDataProvider>> DataSources; TVector<TIntrusivePtr<IDataProvider>> DataSinks; - TUdfIndex::TPtr UdfIndex; - TUdfIndexPackageSet::TPtr UdfIndexPackageSet; + TUdfIndex::TPtr UdfIndex; + TUdfIndexPackageSet::TPtr UdfIndexPackageSet; IUdfResolver::TPtr UdfResolver; THashMap<TString, IUdfResolver::TImport> UdfImports; // aliases of files that was imported => list of module names TUdfModulesTable UdfModules; TString PureResultDataSource; TVector<TString> AvailablePureResultDataSources; TString FullResultDataSink; - TUserDataStorage::TPtr UserDataStorage; + TUserDataStorage::TPtr UserDataStorage; TUserDataTable UserDataStorageCrutches; TYqlOperationOptions OperationOptions; TVector<TCredentialTablePtr> Credentials; TUserCredentials UserCredentials; IModuleResolver::TPtr Modules; NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::None; - bool DisableNativeUdfSupport = false; + bool DisableNativeUdfSupport = false; TMaybe<TString> OptLLVM; bool IsReadOnly = false; TAutoPtr<IGraphTransformer> CustomInstantTypeTransformer; @@ -234,7 +234,7 @@ struct TTypeAnnotationContext: public TThrRefBase { std::tuple<std::optional<ui64>, std::optional<double>, std::optional<TGUID>> CachedRandom; std::optional<bool> InitializeResult; - + template <typename T> T GetRandom() const noexcept; @@ -279,10 +279,10 @@ struct TTypeAnnotationContext: public TThrRefBase { } bool Initialize(TExprContext& ctx); - bool DoInitialize(TExprContext& ctx); - + bool DoInitialize(TExprContext& ctx); + const TCredential* FindCredential(const TStringBuf& name) const; - TString FindCredentialContent(const TStringBuf& name1, const TStringBuf& name2, const TString& defaultContent) const; + TString FindCredentialContent(const TStringBuf& name1, const TStringBuf& name2, const TString& defaultContent) const; TString GetDefaultDataSource() const; TMaybe<ui32> TranslateOperationId(ui64 id) const { diff --git a/ydb/library/yql/core/yql_udf_index.cpp b/ydb/library/yql/core/yql_udf_index.cpp index bbd6b30781..2aef17bbee 100644 --- a/ydb/library/yql/core/yql_udf_index.cpp +++ b/ydb/library/yql/core/yql_udf_index.cpp @@ -1,229 +1,229 @@ -#include "yql_udf_index.h" +#include "yql_udf_index.h" #include <ydb/library/yql/minikql/mkql_function_registry.h> - -namespace NYql { -namespace { - -TVector<TResourceInfo::TPtr> ConvertResolveResultToResources(const TResolveResult& resolveResult, const TMap<TString, TString>& pathsWithMd5, bool isTrusted) { - THashMap<TString, size_t> importIndex; // module => Imports index - THashMap<TString, size_t> packageIndex; // package => Imports index - THashMap<TString, TVector<TFunctionInfo>> functionIndex; // package => vector of functions - for (size_t i = 0; i < resolveResult.ImportsSize(); ++i) { - auto& import = resolveResult.GetImports(i); - if (!import.ModulesSize()) { - continue; - } - - for (auto& m : import.GetModules()) { - importIndex.emplace(m, i); - } - - const TString package = import.GetModules(0); - packageIndex.emplace(package, i); - functionIndex.emplace(package, TVector<TFunctionInfo>()); - } - - for (auto& udf : resolveResult.GetUdfs()) { - const TString module = TString(NKikimr::NMiniKQL::ModuleName(TStringBuf(udf.GetName()))); - const auto& import = resolveResult.GetImports(importIndex.at(module)); - const TString package = import.GetModules(0); - - TFunctionInfo newFunction; - newFunction.Name = udf.GetName(); - newFunction.IsTypeAwareness = udf.GetIsTypeAwareness(); - newFunction.ArgCount = udf.GetArgCount(); - newFunction.OptionalArgCount = udf.GetOptionalArgCount(); - if (udf.HasCallableType()) { - newFunction.CallableType = udf.GetCallableType(); - } - - if (udf.HasRunConfigType()) { - newFunction.RunConfigType = udf.GetRunConfigType(); - } - - functionIndex[package].push_back(newFunction); - } - - TVector<TResourceInfo::TPtr> result; - result.reserve(functionIndex.size()); - for (auto& p : functionIndex) { - const auto& import = resolveResult.GetImports(packageIndex.at(p.first)); - - auto info = MakeIntrusive<TResourceInfo>(); - info->IsTrusted = isTrusted; - auto md5 = pathsWithMd5.FindPtr(import.GetFileAlias()); - info->Link = TDownloadLink::File(import.GetFileAlias(), md5 ? *md5 : ""); - info->Modules.insert(import.GetModules().begin(), import.GetModules().end()); - info->SetFunctions(p.second); - - result.push_back(info); - } - - return result; -} - -void AddResolveResultToRegistry(const TResolveResult& resolveResult, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { - auto resources = ConvertResolveResultToResources(resolveResult, pathsWithMd5, isTrusted); - registry.RegisterResources(resources, mode); -} - -} - -TUdfIndex::TUdfIndex() { -} - -TUdfIndex::TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources) - : Resources_(resources) -{ - -} - -bool TUdfIndex::ContainsModule(const TString& moduleName) const { + +namespace NYql { +namespace { + +TVector<TResourceInfo::TPtr> ConvertResolveResultToResources(const TResolveResult& resolveResult, const TMap<TString, TString>& pathsWithMd5, bool isTrusted) { + THashMap<TString, size_t> importIndex; // module => Imports index + THashMap<TString, size_t> packageIndex; // package => Imports index + THashMap<TString, TVector<TFunctionInfo>> functionIndex; // package => vector of functions + for (size_t i = 0; i < resolveResult.ImportsSize(); ++i) { + auto& import = resolveResult.GetImports(i); + if (!import.ModulesSize()) { + continue; + } + + for (auto& m : import.GetModules()) { + importIndex.emplace(m, i); + } + + const TString package = import.GetModules(0); + packageIndex.emplace(package, i); + functionIndex.emplace(package, TVector<TFunctionInfo>()); + } + + for (auto& udf : resolveResult.GetUdfs()) { + const TString module = TString(NKikimr::NMiniKQL::ModuleName(TStringBuf(udf.GetName()))); + const auto& import = resolveResult.GetImports(importIndex.at(module)); + const TString package = import.GetModules(0); + + TFunctionInfo newFunction; + newFunction.Name = udf.GetName(); + newFunction.IsTypeAwareness = udf.GetIsTypeAwareness(); + newFunction.ArgCount = udf.GetArgCount(); + newFunction.OptionalArgCount = udf.GetOptionalArgCount(); + if (udf.HasCallableType()) { + newFunction.CallableType = udf.GetCallableType(); + } + + if (udf.HasRunConfigType()) { + newFunction.RunConfigType = udf.GetRunConfigType(); + } + + functionIndex[package].push_back(newFunction); + } + + TVector<TResourceInfo::TPtr> result; + result.reserve(functionIndex.size()); + for (auto& p : functionIndex) { + const auto& import = resolveResult.GetImports(packageIndex.at(p.first)); + + auto info = MakeIntrusive<TResourceInfo>(); + info->IsTrusted = isTrusted; + auto md5 = pathsWithMd5.FindPtr(import.GetFileAlias()); + info->Link = TDownloadLink::File(import.GetFileAlias(), md5 ? *md5 : ""); + info->Modules.insert(import.GetModules().begin(), import.GetModules().end()); + info->SetFunctions(p.second); + + result.push_back(info); + } + + return result; +} + +void AddResolveResultToRegistry(const TResolveResult& resolveResult, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { + auto resources = ConvertResolveResultToResources(resolveResult, pathsWithMd5, isTrusted); + registry.RegisterResources(resources, mode); +} + +} + +TUdfIndex::TUdfIndex() { +} + +TUdfIndex::TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources) + : Resources_(resources) +{ + +} + +bool TUdfIndex::ContainsModule(const TString& moduleName) const { return Resources_.contains(moduleName); -} - -bool TUdfIndex::ContainsAnyModule(const TSet<TString>& modules) const { - return AnyOf(modules, [this](auto& m) { - return this->ContainsModule(m); - }); -} - -bool TUdfIndex::FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const { - auto r = FindResourceByModule(moduleName); - if (!r) { - return false; - } - - auto f = r->Functions.FindPtr(functionName); - if (!f) { - return false; - } - - function = *f; - return true; -} - -TResourceInfo::TPtr TUdfIndex::FindResourceByModule(const TString& moduleName) const { - auto p = Resources_.FindPtr(moduleName); - return p ? *p : nullptr; -} - -TSet<TResourceInfo::TPtr> TUdfIndex::FindResourcesByModules(const TSet<TString>& modules) const { - TSet<TResourceInfo::TPtr> result; - for (auto& m : modules) { - auto r = FindResourceByModule(m); - if (r) { - result.insert(r); - } - } - return result; -} - -void TUdfIndex::UnregisterResource(TResourceInfo::TPtr resource) { - for (auto& m : resource->Modules) { - Resources_.erase(m); - } - // resource pointer should be alive here to avoid problems with erase -} - -void TUdfIndex::RegisterResource(const TResourceInfo::TPtr& resource, EOverrideMode mode) { - Y_ENSURE(resource); - if (resource->Modules.empty()) { - // quite strange, but let's ignore - return; - } - - // detect conflict first - if (ContainsAnyModule(resource->Modules)) { - switch (mode) { - case EOverrideMode::PreserveExisting: - return; - - case EOverrideMode::RaiseError: - // todo: specify module name(s) in intersection - ythrow yexception() << "Conflict during resource " << resource->Link.Path << " registration"; - - case EOverrideMode::ReplaceWithNew: { - // we have to find resources and remove all related modules: - // 1. find resources by newModules - // 2. remove all functions related to found resources - - auto existingResources = FindResourcesByModules(resource->Modules); - Y_ENSURE(!existingResources.empty()); - - for (auto& r : existingResources) { - UnregisterResource(r); - } - - break; - } - } // switch - } - - for (auto& m : resource->Modules) { - Resources_.emplace(m, resource); - } -} - -TIntrusivePtr<TUdfIndex> TUdfIndex::Clone() const { - return new TUdfIndex(Resources_); -} - -void TUdfIndex::RegisterResources(const TVector<TResourceInfo::TPtr>& resources, EOverrideMode mode) { - for (auto& r : resources) { - RegisterResource(r, mode); - } -} - -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { - TMap<TString, TString> pathsWithMd5; - for (const auto& path : paths) { - pathsWithMd5[path] = ""; - } - LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry); -} - -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { - TVector<TString> paths; - paths.reserve(pathsWithMd5.size()); - for (const auto& p : pathsWithMd5) { - paths.push_back(p.first); - } - const TResolveResult resolveResult = LoadRichMetadata(resolver, paths); - AddResolveResultToRegistry(resolveResult, pathsWithMd5, isTrusted, mode, registry); -} - -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { - // we can work with file path only - TMap<TString, TString> pathsWithMd5; - for (auto& b : blocks) { - switch (b.Type) { - case EUserDataType::URL: - if (!b.FrozenFile) { - ythrow yexception() << "DataBlock for " << b.Data << " is not frozen"; - } - pathsWithMd5.emplace(b.FrozenFile->GetPath().GetPath(), b.FrozenFile->GetMd5()); - break; - case EUserDataType::PATH: - { - TString md5; - if (b.FrozenFile) { - md5 = b.FrozenFile->GetMd5(); - } - pathsWithMd5.emplace(b.Data, md5); - break; - } - default: - ythrow yexception() << "Unsupport data block type for " << b.Data; - } - } - LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry); -} - -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { - TVector<TUserDataBlock> blocks({ block }); - const bool isTrusted = false; - LoadRichMetadataToUdfIndex(resolver, blocks, isTrusted, mode, registry); -} - -} // namespace NYql +} + +bool TUdfIndex::ContainsAnyModule(const TSet<TString>& modules) const { + return AnyOf(modules, [this](auto& m) { + return this->ContainsModule(m); + }); +} + +bool TUdfIndex::FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const { + auto r = FindResourceByModule(moduleName); + if (!r) { + return false; + } + + auto f = r->Functions.FindPtr(functionName); + if (!f) { + return false; + } + + function = *f; + return true; +} + +TResourceInfo::TPtr TUdfIndex::FindResourceByModule(const TString& moduleName) const { + auto p = Resources_.FindPtr(moduleName); + return p ? *p : nullptr; +} + +TSet<TResourceInfo::TPtr> TUdfIndex::FindResourcesByModules(const TSet<TString>& modules) const { + TSet<TResourceInfo::TPtr> result; + for (auto& m : modules) { + auto r = FindResourceByModule(m); + if (r) { + result.insert(r); + } + } + return result; +} + +void TUdfIndex::UnregisterResource(TResourceInfo::TPtr resource) { + for (auto& m : resource->Modules) { + Resources_.erase(m); + } + // resource pointer should be alive here to avoid problems with erase +} + +void TUdfIndex::RegisterResource(const TResourceInfo::TPtr& resource, EOverrideMode mode) { + Y_ENSURE(resource); + if (resource->Modules.empty()) { + // quite strange, but let's ignore + return; + } + + // detect conflict first + if (ContainsAnyModule(resource->Modules)) { + switch (mode) { + case EOverrideMode::PreserveExisting: + return; + + case EOverrideMode::RaiseError: + // todo: specify module name(s) in intersection + ythrow yexception() << "Conflict during resource " << resource->Link.Path << " registration"; + + case EOverrideMode::ReplaceWithNew: { + // we have to find resources and remove all related modules: + // 1. find resources by newModules + // 2. remove all functions related to found resources + + auto existingResources = FindResourcesByModules(resource->Modules); + Y_ENSURE(!existingResources.empty()); + + for (auto& r : existingResources) { + UnregisterResource(r); + } + + break; + } + } // switch + } + + for (auto& m : resource->Modules) { + Resources_.emplace(m, resource); + } +} + +TIntrusivePtr<TUdfIndex> TUdfIndex::Clone() const { + return new TUdfIndex(Resources_); +} + +void TUdfIndex::RegisterResources(const TVector<TResourceInfo::TPtr>& resources, EOverrideMode mode) { + for (auto& r : resources) { + RegisterResource(r, mode); + } +} + +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { + TMap<TString, TString> pathsWithMd5; + for (const auto& path : paths) { + pathsWithMd5[path] = ""; + } + LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry); +} + +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { + TVector<TString> paths; + paths.reserve(pathsWithMd5.size()); + for (const auto& p : pathsWithMd5) { + paths.push_back(p.first); + } + const TResolveResult resolveResult = LoadRichMetadata(resolver, paths); + AddResolveResultToRegistry(resolveResult, pathsWithMd5, isTrusted, mode, registry); +} + +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { + // we can work with file path only + TMap<TString, TString> pathsWithMd5; + for (auto& b : blocks) { + switch (b.Type) { + case EUserDataType::URL: + if (!b.FrozenFile) { + ythrow yexception() << "DataBlock for " << b.Data << " is not frozen"; + } + pathsWithMd5.emplace(b.FrozenFile->GetPath().GetPath(), b.FrozenFile->GetMd5()); + break; + case EUserDataType::PATH: + { + TString md5; + if (b.FrozenFile) { + md5 = b.FrozenFile->GetMd5(); + } + pathsWithMd5.emplace(b.Data, md5); + break; + } + default: + ythrow yexception() << "Unsupport data block type for " << b.Data; + } + } + LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry); +} + +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry) { + TVector<TUserDataBlock> blocks({ block }); + const bool isTrusted = false; + LoadRichMetadataToUdfIndex(resolver, blocks, isTrusted, mode, registry); +} + +} // namespace NYql diff --git a/ydb/library/yql/core/yql_udf_index.h b/ydb/library/yql/core/yql_udf_index.h index 51f768d9de..de3bf6572b 100644 --- a/ydb/library/yql/core/yql_udf_index.h +++ b/ydb/library/yql/core/yql_udf_index.h @@ -1,131 +1,131 @@ -#pragma once - -#include "yql_udf_resolver.h" +#pragma once + +#include "yql_udf_resolver.h" #include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/set.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NYql { - -struct TFunctionInfo { - TString Name; - int ArgCount = 0; - int OptionalArgCount = 0; - bool IsTypeAwareness = false; - TString CallableType; - TString RunConfigType; -}; - -// todo: specify whether path is frozen -struct TDownloadLink { - bool IsUrl = false; - TString Path; - TString Md5; - - TDownloadLink() { - - } - - TDownloadLink(bool isUrl, const TString& path, const TString& md5) - : IsUrl(isUrl) - , Path(path) - , Md5(md5) - { - } - - TDownloadLink(const TDownloadLink&) = default; - TDownloadLink& operator=(const TDownloadLink&) = default; - - static TDownloadLink Url(const TString& path, const TString& md5 = "") { - return { true, path, md5 }; - } - - static TDownloadLink File(const TString& path, const TString& md5 = "") { - return { false, path, md5 }; - } - - bool operator==(const TDownloadLink& other) const { - return std::tie(IsUrl, Path, Md5) == std::tie(other.IsUrl, other.Path, Md5); - } - - bool operator!=(const TDownloadLink& other) const { - return !(*this == other); - } - - size_t Hash() const { - return CombineHashes( +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/set.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NYql { + +struct TFunctionInfo { + TString Name; + int ArgCount = 0; + int OptionalArgCount = 0; + bool IsTypeAwareness = false; + TString CallableType; + TString RunConfigType; +}; + +// todo: specify whether path is frozen +struct TDownloadLink { + bool IsUrl = false; + TString Path; + TString Md5; + + TDownloadLink() { + + } + + TDownloadLink(bool isUrl, const TString& path, const TString& md5) + : IsUrl(isUrl) + , Path(path) + , Md5(md5) + { + } + + TDownloadLink(const TDownloadLink&) = default; + TDownloadLink& operator=(const TDownloadLink&) = default; + + static TDownloadLink Url(const TString& path, const TString& md5 = "") { + return { true, path, md5 }; + } + + static TDownloadLink File(const TString& path, const TString& md5 = "") { + return { false, path, md5 }; + } + + bool operator==(const TDownloadLink& other) const { + return std::tie(IsUrl, Path, Md5) == std::tie(other.IsUrl, other.Path, Md5); + } + + bool operator!=(const TDownloadLink& other) const { + return !(*this == other); + } + + size_t Hash() const { + return CombineHashes( CombineHashes((size_t)IsUrl, ComputeHash(Path)), ComputeHash(Md5) - ); - } -}; - -struct TResourceInfo : public TThrRefBase { - typedef TIntrusiveConstPtr<TResourceInfo> TPtr; - - bool IsTrusted = false; - TDownloadLink Link; - TSet<TString> Modules; - TMap<TString, TFunctionInfo> Functions; - - void SetFunctions(const TVector<TFunctionInfo>& functions) { - for (auto& f : functions) { - Functions.emplace(f.Name, f); - } - } -}; - -inline bool operator<(const TResourceInfo::TPtr& p1, const TResourceInfo::TPtr& p2) { - return p1.Get() < p2.Get(); -} - -class TUdfIndex : public TThrRefBase { -public: - typedef TIntrusivePtr<TUdfIndex> TPtr; - -public: - // todo: trusted resources should not be replaceble regardless of specified mode - enum class EOverrideMode { - PreserveExisting, - ReplaceWithNew, - RaiseError - }; - -public: - TUdfIndex(); - bool ContainsModule(const TString& moduleName) const; - bool FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const; - TResourceInfo::TPtr FindResourceByModule(const TString& moduleName) const; - - /* - New resource can contain already registered module. - In this case 'mode' will be used to resolve conflicts. - For instance, if mode == ReplaceWithNew all functions from old resource will be removed and new functions will be registered. - It is important to do it atomically because two .so cannot have intersecting module lists - */ - void RegisterResource(const TResourceInfo::TPtr& resource, EOverrideMode mode); - void RegisterResources(const TVector<TResourceInfo::TPtr>& resources, EOverrideMode mode); - - TIntrusivePtr<TUdfIndex> Clone() const; - -private: - explicit TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources); - - bool ContainsAnyModule(const TSet<TString>& modules) const; - TSet<TResourceInfo::TPtr> FindResourcesByModules(const TSet<TString>& modules) const; - void UnregisterResource(TResourceInfo::TPtr resource); - -private: - // module => Resource - TMap<TString, TResourceInfo::TPtr> Resources_; -}; - -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); -void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); - -} + ); + } +}; + +struct TResourceInfo : public TThrRefBase { + typedef TIntrusiveConstPtr<TResourceInfo> TPtr; + + bool IsTrusted = false; + TDownloadLink Link; + TSet<TString> Modules; + TMap<TString, TFunctionInfo> Functions; + + void SetFunctions(const TVector<TFunctionInfo>& functions) { + for (auto& f : functions) { + Functions.emplace(f.Name, f); + } + } +}; + +inline bool operator<(const TResourceInfo::TPtr& p1, const TResourceInfo::TPtr& p2) { + return p1.Get() < p2.Get(); +} + +class TUdfIndex : public TThrRefBase { +public: + typedef TIntrusivePtr<TUdfIndex> TPtr; + +public: + // todo: trusted resources should not be replaceble regardless of specified mode + enum class EOverrideMode { + PreserveExisting, + ReplaceWithNew, + RaiseError + }; + +public: + TUdfIndex(); + bool ContainsModule(const TString& moduleName) const; + bool FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const; + TResourceInfo::TPtr FindResourceByModule(const TString& moduleName) const; + + /* + New resource can contain already registered module. + In this case 'mode' will be used to resolve conflicts. + For instance, if mode == ReplaceWithNew all functions from old resource will be removed and new functions will be registered. + It is important to do it atomically because two .so cannot have intersecting module lists + */ + void RegisterResource(const TResourceInfo::TPtr& resource, EOverrideMode mode); + void RegisterResources(const TVector<TResourceInfo::TPtr>& resources, EOverrideMode mode); + + TIntrusivePtr<TUdfIndex> Clone() const; + +private: + explicit TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources); + + bool ContainsAnyModule(const TSet<TString>& modules) const; + TSet<TResourceInfo::TPtr> FindResourcesByModules(const TSet<TString>& modules) const; + void UnregisterResource(TResourceInfo::TPtr resource); + +private: + // module => Resource + TMap<TString, TResourceInfo::TPtr> Resources_; +}; + +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); +void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); + +} diff --git a/ydb/library/yql/core/yql_udf_index_package_set.cpp b/ydb/library/yql/core/yql_udf_index_package_set.cpp index e262d08f92..b2da644cff 100644 --- a/ydb/library/yql/core/yql_udf_index_package_set.cpp +++ b/ydb/library/yql/core/yql_udf_index_package_set.cpp @@ -1,61 +1,61 @@ -#include "yql_udf_index_package_set.h" - +#include "yql_udf_index_package_set.h" + #include <ydb/library/yql/utils/yql_panic.h> - -using namespace NYql; - -TUdfIndexPackageSet::TUdfIndexPackageSet() { -} - -TUdfIndexPackageSet::TUdfIndexPackageSet(const TSet<TString>& knownPackages, const THashMap<TString, ui32>& packageDefaultVersions, const TMap<std::pair<TString, ui32>, TResourceInfo::TPtr>& resources) - : KnownPackages_(knownPackages) - , PackageDefaultVersions_(packageDefaultVersions) - , Resources_(resources) -{ -} - -void TUdfIndexPackageSet::RegisterPackage(const TString& package) { - KnownPackages_.insert(package); -} - -bool TUdfIndexPackageSet::SetPackageDefaultVersion(const TString& package, ui32 version) { - if (!KnownPackages_.contains(package)) { - return false; - } - - PackageDefaultVersions_[package] = version; - return true; -} - -void TUdfIndexPackageSet::RegisterResource(const TString& package, ui32 version, const TResourceInfo::TPtr& resource) { - YQL_ENSURE(resource != nullptr); - YQL_ENSURE(KnownPackages_.contains(package), "Unknown package " << package); - Resources_.emplace(std::make_pair(package, version), resource); -} - -bool TUdfIndexPackageSet::AddResourceTo(const TString& package, ui32 version, const TUdfIndex::TPtr& target) const { - auto it = Resources_.find(std::make_pair(package, version)); - if (it == Resources_.end()) { - return KnownPackages_.contains(package); - } - target->RegisterResource(it->second, TUdfIndex::EOverrideMode::ReplaceWithNew); - return true; -} - -void TUdfIndexPackageSet::AddResourcesTo(const TUdfIndex::TPtr& target) const { - YQL_ENSURE(target != nullptr); - for (auto& package : KnownPackages_) { - auto vIt = PackageDefaultVersions_.find(package); - const ui32 version = (vIt != PackageDefaultVersions_.end()) ? vIt->second : 0; - auto it = Resources_.find(std::make_pair(package, version)); - if (it == Resources_.end()) { - // todo: consider error? - continue; - } - target->RegisterResource(it->second, TUdfIndex::EOverrideMode::ReplaceWithNew); - } -} - -TUdfIndexPackageSet::TPtr TUdfIndexPackageSet::Clone() const { - return new TUdfIndexPackageSet(KnownPackages_, PackageDefaultVersions_, Resources_); -} + +using namespace NYql; + +TUdfIndexPackageSet::TUdfIndexPackageSet() { +} + +TUdfIndexPackageSet::TUdfIndexPackageSet(const TSet<TString>& knownPackages, const THashMap<TString, ui32>& packageDefaultVersions, const TMap<std::pair<TString, ui32>, TResourceInfo::TPtr>& resources) + : KnownPackages_(knownPackages) + , PackageDefaultVersions_(packageDefaultVersions) + , Resources_(resources) +{ +} + +void TUdfIndexPackageSet::RegisterPackage(const TString& package) { + KnownPackages_.insert(package); +} + +bool TUdfIndexPackageSet::SetPackageDefaultVersion(const TString& package, ui32 version) { + if (!KnownPackages_.contains(package)) { + return false; + } + + PackageDefaultVersions_[package] = version; + return true; +} + +void TUdfIndexPackageSet::RegisterResource(const TString& package, ui32 version, const TResourceInfo::TPtr& resource) { + YQL_ENSURE(resource != nullptr); + YQL_ENSURE(KnownPackages_.contains(package), "Unknown package " << package); + Resources_.emplace(std::make_pair(package, version), resource); +} + +bool TUdfIndexPackageSet::AddResourceTo(const TString& package, ui32 version, const TUdfIndex::TPtr& target) const { + auto it = Resources_.find(std::make_pair(package, version)); + if (it == Resources_.end()) { + return KnownPackages_.contains(package); + } + target->RegisterResource(it->second, TUdfIndex::EOverrideMode::ReplaceWithNew); + return true; +} + +void TUdfIndexPackageSet::AddResourcesTo(const TUdfIndex::TPtr& target) const { + YQL_ENSURE(target != nullptr); + for (auto& package : KnownPackages_) { + auto vIt = PackageDefaultVersions_.find(package); + const ui32 version = (vIt != PackageDefaultVersions_.end()) ? vIt->second : 0; + auto it = Resources_.find(std::make_pair(package, version)); + if (it == Resources_.end()) { + // todo: consider error? + continue; + } + target->RegisterResource(it->second, TUdfIndex::EOverrideMode::ReplaceWithNew); + } +} + +TUdfIndexPackageSet::TPtr TUdfIndexPackageSet::Clone() const { + return new TUdfIndexPackageSet(KnownPackages_, PackageDefaultVersions_, Resources_); +} diff --git a/ydb/library/yql/core/yql_udf_index_package_set.h b/ydb/library/yql/core/yql_udf_index_package_set.h index 4d39904795..71f59aa6eb 100644 --- a/ydb/library/yql/core/yql_udf_index_package_set.h +++ b/ydb/library/yql/core/yql_udf_index_package_set.h @@ -1,29 +1,29 @@ -#pragma once - -#include "yql_udf_index.h" - -namespace NYql { - -class TUdfIndexPackageSet : public TThrRefBase { -public: - typedef TIntrusivePtr<TUdfIndexPackageSet> TPtr; - -private: - TUdfIndexPackageSet(const TSet<TString>& knownPackages, const THashMap<TString, ui32>& packageDefaultVersions, const TMap<std::pair<TString, ui32>, TResourceInfo::TPtr>& resources); - -public: - TUdfIndexPackageSet(); - void RegisterPackage(const TString& package); - bool SetPackageDefaultVersion(const TString& package, ui32 version); - void RegisterResource(const TString& package, ui32 version, const TResourceInfo::TPtr& resource); - bool AddResourceTo(const TString& package, ui32 version, const TUdfIndex::TPtr& target) const; - void AddResourcesTo(const TUdfIndex::TPtr& target) const; - TPtr Clone() const; - -private: - TSet<TString> KnownPackages_; - THashMap<TString, ui32> PackageDefaultVersions_; - TMap<std::pair<TString, ui32>, TResourceInfo::TPtr> Resources_; -}; - -} +#pragma once + +#include "yql_udf_index.h" + +namespace NYql { + +class TUdfIndexPackageSet : public TThrRefBase { +public: + typedef TIntrusivePtr<TUdfIndexPackageSet> TPtr; + +private: + TUdfIndexPackageSet(const TSet<TString>& knownPackages, const THashMap<TString, ui32>& packageDefaultVersions, const TMap<std::pair<TString, ui32>, TResourceInfo::TPtr>& resources); + +public: + TUdfIndexPackageSet(); + void RegisterPackage(const TString& package); + bool SetPackageDefaultVersion(const TString& package, ui32 version); + void RegisterResource(const TString& package, ui32 version, const TResourceInfo::TPtr& resource); + bool AddResourceTo(const TString& package, ui32 version, const TUdfIndex::TPtr& target) const; + void AddResourcesTo(const TUdfIndex::TPtr& target) const; + TPtr Clone() const; + +private: + TSet<TString> KnownPackages_; + THashMap<TString, ui32> PackageDefaultVersions_; + TMap<std::pair<TString, ui32>, TResourceInfo::TPtr> Resources_; +}; + +} diff --git a/ydb/library/yql/core/yql_udf_resolver.cpp b/ydb/library/yql/core/yql_udf_resolver.cpp index b4d29015cd..be12f11d35 100644 --- a/ydb/library/yql/core/yql_udf_resolver.cpp +++ b/ydb/library/yql/core/yql_udf_resolver.cpp @@ -1,33 +1,33 @@ -#include "yql_udf_resolver.h" - -namespace NYql { - -TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks) { - TVector<IUdfResolver::TImport> imports; - imports.reserve(blocks.size()); - std::transform(blocks.begin(), blocks.end(), std::back_inserter(imports), [](auto& b) { - IUdfResolver::TImport import; - import.Block = &b; - // this field is not used later, but allows us to map importResult to import - import.FileAlias = b.Data; - return import; - }); - - return resolver.LoadRichMetadata(imports); -} - -TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths) { - TVector<TUserDataBlock> blocks; - blocks.reserve(paths.size()); - std::transform(paths.begin(), paths.end(), std::back_inserter(blocks), [](auto& p) { - TUserDataBlock b; - b.Type = EUserDataType::PATH; - b.Data = p; - b.Usage.Set(EUserDataBlockUsage::Udf); - return b; - }); - - return LoadRichMetadata(resolver, blocks); -} - -} +#include "yql_udf_resolver.h" + +namespace NYql { + +TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks) { + TVector<IUdfResolver::TImport> imports; + imports.reserve(blocks.size()); + std::transform(blocks.begin(), blocks.end(), std::back_inserter(imports), [](auto& b) { + IUdfResolver::TImport import; + import.Block = &b; + // this field is not used later, but allows us to map importResult to import + import.FileAlias = b.Data; + return import; + }); + + return resolver.LoadRichMetadata(imports); +} + +TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths) { + TVector<TUserDataBlock> blocks; + blocks.reserve(paths.size()); + std::transform(paths.begin(), paths.end(), std::back_inserter(blocks), [](auto& p) { + TUserDataBlock b; + b.Type = EUserDataType::PATH; + b.Data = p; + b.Usage.Set(EUserDataBlockUsage::Udf); + return b; + }); + + return LoadRichMetadata(resolver, blocks); +} + +} diff --git a/ydb/library/yql/core/yql_udf_resolver.h b/ydb/library/yql/core/yql_udf_resolver.h index 235987ecab..ab9e8e3adf 100644 --- a/ydb/library/yql/core/yql_udf_resolver.h +++ b/ydb/library/yql/core/yql_udf_resolver.h @@ -1,74 +1,74 @@ -#pragma once - -#include "yql_user_data.h" +#pragma once + +#include "yql_user_data.h" #include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> - -#include <util/generic/maybe.h> -#include <util/generic/vector.h> - -namespace NYql { - -class TTypeAnnotationNode; -struct TUserDataBlock; -struct TExprContext; - -struct TFilePathWithMd5 { - TString Path; - TString Md5; - - explicit TFilePathWithMd5(const TString& path = "", const TString& md5 = "") - : Path(path) - , Md5(md5) - { - } - - TFilePathWithMd5& operator=(const TFilePathWithMd5& other) = default; -}; - -class IUdfResolver : public TThrRefBase { -public: - using TPtr = TIntrusiveConstPtr<IUdfResolver>; - - virtual ~IUdfResolver() = default; - - struct TFunction { - // input - TPosition Pos; - TString Name; - TString TypeConfig; - const TTypeAnnotationNode* UserType = nullptr; + +#include <util/generic/maybe.h> +#include <util/generic/vector.h> + +namespace NYql { + +class TTypeAnnotationNode; +struct TUserDataBlock; +struct TExprContext; + +struct TFilePathWithMd5 { + TString Path; + TString Md5; + + explicit TFilePathWithMd5(const TString& path = "", const TString& md5 = "") + : Path(path) + , Md5(md5) + { + } + + TFilePathWithMd5& operator=(const TFilePathWithMd5& other) = default; +}; + +class IUdfResolver : public TThrRefBase { +public: + using TPtr = TIntrusiveConstPtr<IUdfResolver>; + + virtual ~IUdfResolver() = default; + + struct TFunction { + // input + TPosition Pos; + TString Name; + TString TypeConfig; + const TTypeAnnotationNode* UserType = nullptr; THashMap<TString, TString> SecureParams; - - // output - const TTypeAnnotationNode* NormalizedUserType = nullptr; - const TTypeAnnotationNode* RunConfigType = nullptr; - const TTypeAnnotationNode* CallableType = nullptr; - }; - - struct TImport { - // input - TPosition Pos; - TString FileAlias; - const TUserDataBlock* Block = nullptr; - - // output - TMaybe<TVector<TString>> Modules; - }; - - /* - Returns nothing if module is not a system one - Always returns frozen path - */ - virtual TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const = 0; - virtual bool LoadMetadata(const TVector<TImport*>& imports, - const TVector<TFunction*>& functions, TExprContext& ctx) const = 0; - - virtual TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const = 0; -}; - -TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks); -TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths); - -} + + // output + const TTypeAnnotationNode* NormalizedUserType = nullptr; + const TTypeAnnotationNode* RunConfigType = nullptr; + const TTypeAnnotationNode* CallableType = nullptr; + }; + + struct TImport { + // input + TPosition Pos; + TString FileAlias; + const TUserDataBlock* Block = nullptr; + + // output + TMaybe<TVector<TString>> Modules; + }; + + /* + Returns nothing if module is not a system one + Always returns frozen path + */ + virtual TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const = 0; + virtual bool LoadMetadata(const TVector<TImport*>& imports, + const TVector<TFunction*>& functions, TExprContext& ctx) const = 0; + + virtual TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const = 0; +}; + +TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks); +TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths); + +} diff --git a/ydb/library/yql/core/yql_user_data.cpp b/ydb/library/yql/core/yql_user_data.cpp index 5640a10f15..05762596a1 100644 --- a/ydb/library/yql/core/yql_user_data.cpp +++ b/ydb/library/yql/core/yql_user_data.cpp @@ -1 +1 @@ -#include "yql_user_data.h" +#include "yql_user_data.h" diff --git a/ydb/library/yql/core/yql_user_data.h b/ydb/library/yql/core/yql_user_data.h index fadc8f79cf..29e89f4884 100644 --- a/ydb/library/yql/core/yql_user_data.h +++ b/ydb/library/yql/core/yql_user_data.h @@ -1,103 +1,103 @@ -#pragma once - +#pragma once + #include <library/cpp/enumbitset/enumbitset.h> - + #include <ydb/library/yql/core/file_storage/file_storage.h> - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> - -namespace NYql { - -// -- user files -- - -enum class EUserDataType { - URL = 1, // indicates that provided user data represents URL which can be used to obtain table data - PATH = 2, // indicates that provided user data represents file path which can be used to obtain table data - RAW_INLINE_DATA = 3, // table data is inside the provided data block -}; - -enum class EUserDataBlockUsage { - Begin, - Path = Begin, - Content, - Udf, - Library, - End, -}; -typedef TEnumBitSet<EUserDataBlockUsage, static_cast<int>(EUserDataBlockUsage::Begin), - static_cast<int>(EUserDataBlockUsage::End)> TUserDataBlockUsage; - -struct TUserDataBlock { - EUserDataType Type = EUserDataType::PATH; - TString UrlToken; - TString Data; - TUserDataBlockUsage Usage; - - TFileLinkPtr FrozenFile; -}; - -class TUserDataKey { -public: - enum class EDataType { - FILE, UDF - }; - - static TUserDataKey File(const TString& alias) { - return { EDataType::FILE, alias }; - } - - static TUserDataKey File(const TStringBuf& alias) { - return { EDataType::FILE, TString(alias) }; - } - - static TUserDataKey Udf(const TString& alias) { - return { EDataType::UDF, alias }; - } - - static TUserDataKey Udf(const TStringBuf& alias) { - return { EDataType::UDF, TString(alias) }; - } - - inline bool IsFile() const { return Type_ == EDataType::FILE; } - inline bool IsUdf() const { return Type_ == EDataType::UDF; } - inline const TString& Alias() const { return Alias_; } - inline EDataType Type() const { return Type_; } - - bool operator<(const TUserDataKey& other) const { - return std::tie(Type_, Alias_) < std::tie(other.Type_, other.Alias_); - } - - struct THash { - size_t operator()(const TUserDataKey& key) { - auto type = static_cast<size_t>(key.Type_); + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> + +namespace NYql { + +// -- user files -- + +enum class EUserDataType { + URL = 1, // indicates that provided user data represents URL which can be used to obtain table data + PATH = 2, // indicates that provided user data represents file path which can be used to obtain table data + RAW_INLINE_DATA = 3, // table data is inside the provided data block +}; + +enum class EUserDataBlockUsage { + Begin, + Path = Begin, + Content, + Udf, + Library, + End, +}; +typedef TEnumBitSet<EUserDataBlockUsage, static_cast<int>(EUserDataBlockUsage::Begin), + static_cast<int>(EUserDataBlockUsage::End)> TUserDataBlockUsage; + +struct TUserDataBlock { + EUserDataType Type = EUserDataType::PATH; + TString UrlToken; + TString Data; + TUserDataBlockUsage Usage; + + TFileLinkPtr FrozenFile; +}; + +class TUserDataKey { +public: + enum class EDataType { + FILE, UDF + }; + + static TUserDataKey File(const TString& alias) { + return { EDataType::FILE, alias }; + } + + static TUserDataKey File(const TStringBuf& alias) { + return { EDataType::FILE, TString(alias) }; + } + + static TUserDataKey Udf(const TString& alias) { + return { EDataType::UDF, alias }; + } + + static TUserDataKey Udf(const TStringBuf& alias) { + return { EDataType::UDF, TString(alias) }; + } + + inline bool IsFile() const { return Type_ == EDataType::FILE; } + inline bool IsUdf() const { return Type_ == EDataType::UDF; } + inline const TString& Alias() const { return Alias_; } + inline EDataType Type() const { return Type_; } + + bool operator<(const TUserDataKey& other) const { + return std::tie(Type_, Alias_) < std::tie(other.Type_, other.Alias_); + } + + struct THash { + size_t operator()(const TUserDataKey& key) { + auto type = static_cast<size_t>(key.Type_); return CombineHashes(type, ComputeHash(key.Alias_)); - } - }; - - struct TEqualTo { - bool operator()(const TUserDataKey& k1, const TUserDataKey& k2) const { - return k1.Type_ == k2.Type_ && k1.Alias_ == k2.Alias_; - } - }; - -private: - inline TUserDataKey(EDataType type, TString alias) - : Type_(type) - , Alias_(std::move(alias)) - { - } - -private: - const EDataType Type_; - const TString Alias_; -}; - -inline IOutputStream& operator<<(IOutputStream& os, const TUserDataKey& key) { - os << "Type: " << key.Type() << ", alias: " << key.Alias(); - return os; -} - -using TUserDataTable = THashMap<TUserDataKey, TUserDataBlock, TUserDataKey::THash, TUserDataKey::TEqualTo>; - -} // namespace NYql + } + }; + + struct TEqualTo { + bool operator()(const TUserDataKey& k1, const TUserDataKey& k2) const { + return k1.Type_ == k2.Type_ && k1.Alias_ == k2.Alias_; + } + }; + +private: + inline TUserDataKey(EDataType type, TString alias) + : Type_(type) + , Alias_(std::move(alias)) + { + } + +private: + const EDataType Type_; + const TString Alias_; +}; + +inline IOutputStream& operator<<(IOutputStream& os, const TUserDataKey& key) { + os << "Type: " << key.Type() << ", alias: " << key.Alias(); + return os; +} + +using TUserDataTable = THashMap<TUserDataKey, TUserDataBlock, TUserDataKey::THash, TUserDataKey::TEqualTo>; + +} // namespace NYql diff --git a/ydb/library/yql/core/yql_user_data_storage.cpp b/ydb/library/yql/core/yql_user_data_storage.cpp index 1a9d725d27..a01b1ef42f 100644 --- a/ydb/library/yql/core/yql_user_data_storage.cpp +++ b/ydb/library/yql/core/yql_user_data_storage.cpp @@ -1,127 +1,127 @@ -#include "yql_user_data_storage.h" - +#include "yql_user_data_storage.h" + #include <ydb/library/yql/utils/log/profile.h> - + #include <ydb/library/yql/utils/future_action.h> - -#include <util/folder/iterator.h> -#include <util/string/builder.h> -#include <util/string/strip.h> -#include <util/system/fs.h> -#include <util/system/fstat.h> -#include <util/system/file.h> - -namespace { -const auto Sep = '/'; -const TString Root(1, Sep); -const TString Home = "home"; -const TString HomePath = Root + Home + Sep; - -template <typename DataMapType, typename F> -void EnumFolderContent(DataMapType& data, const TString& folderPath, ui32 maxFileCount, F&& f) { - ui32 count = 0; - for (auto& p : data) { - if (!p.first.IsFile() || !p.first.Alias().StartsWith(folderPath)) { - continue; - } - - f(p.first, p.second); - - if (count++ >= maxFileCount) { - break; - } - } -} - -} - -namespace NYql { - -TUserDataStorage::TUserDataStorage(TFileStoragePtr fileStorage, TUserDataTable data, IUdfResolver::TPtr udfResolver, TUdfIndex::TPtr udfIndex) - : FileStorage_(std::move(fileStorage)) - , UserData_(std::move(data)) - , UdfResolver(std::move(udfResolver)) - , UdfIndex(std::move(udfIndex)) -{ -} - -void TUserDataStorage::SetTokenResolver(TTokenResolver tokenResolver) { - TokenResolver_ = std::move(tokenResolver); -} - -void TUserDataStorage::AddUserDataBlock(const TStringBuf& name, const TUserDataBlock& block) { - const auto key = ComposeUserDataKey(name); - AddUserDataBlock(key, block); -} - -void TUserDataStorage::AddUserDataBlock(const TUserDataKey& key, const TUserDataBlock& block) { - if (!UserData_.emplace(key, block).second) { - throw yexception() << "Failed to add user data block, key " << key << " already registered"; - } -} - -bool TUserDataStorage::ContainsUserDataBlock(const TStringBuf& name) const { - const auto key = ComposeUserDataKey(name); - return ContainsUserDataBlock(key); -} - -bool TUserDataStorage::ContainsUserDataBlock(const TUserDataKey& key) const { + +#include <util/folder/iterator.h> +#include <util/string/builder.h> +#include <util/string/strip.h> +#include <util/system/fs.h> +#include <util/system/fstat.h> +#include <util/system/file.h> + +namespace { +const auto Sep = '/'; +const TString Root(1, Sep); +const TString Home = "home"; +const TString HomePath = Root + Home + Sep; + +template <typename DataMapType, typename F> +void EnumFolderContent(DataMapType& data, const TString& folderPath, ui32 maxFileCount, F&& f) { + ui32 count = 0; + for (auto& p : data) { + if (!p.first.IsFile() || !p.first.Alias().StartsWith(folderPath)) { + continue; + } + + f(p.first, p.second); + + if (count++ >= maxFileCount) { + break; + } + } +} + +} + +namespace NYql { + +TUserDataStorage::TUserDataStorage(TFileStoragePtr fileStorage, TUserDataTable data, IUdfResolver::TPtr udfResolver, TUdfIndex::TPtr udfIndex) + : FileStorage_(std::move(fileStorage)) + , UserData_(std::move(data)) + , UdfResolver(std::move(udfResolver)) + , UdfIndex(std::move(udfIndex)) +{ +} + +void TUserDataStorage::SetTokenResolver(TTokenResolver tokenResolver) { + TokenResolver_ = std::move(tokenResolver); +} + +void TUserDataStorage::AddUserDataBlock(const TStringBuf& name, const TUserDataBlock& block) { + const auto key = ComposeUserDataKey(name); + AddUserDataBlock(key, block); +} + +void TUserDataStorage::AddUserDataBlock(const TUserDataKey& key, const TUserDataBlock& block) { + if (!UserData_.emplace(key, block).second) { + throw yexception() << "Failed to add user data block, key " << key << " already registered"; + } +} + +bool TUserDataStorage::ContainsUserDataBlock(const TStringBuf& name) const { + const auto key = ComposeUserDataKey(name); + return ContainsUserDataBlock(key); +} + +bool TUserDataStorage::ContainsUserDataBlock(const TUserDataKey& key) const { return UserData_.contains(key); -} - -const TUserDataBlock& TUserDataStorage::GetUserDataBlock(const TUserDataKey& key) const { - auto block = FindUserDataBlock(key); - if (!block) { - ythrow yexception() << "Failed to find user data block by key " << key; - } - - return *block; -} - -const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TStringBuf& name) const { - const auto key = ComposeUserDataKey(name); - return FindUserDataBlock(key); -} - -const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataKey& key) const { - return FindUserDataBlock(UserData_, key); -} - -TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataKey& key) { - return FindUserDataBlock(UserData_, key); -} - -const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataTable& userData, const TStringBuf& name) { - const auto key = ComposeUserDataKey(name); - return FindUserDataBlock(userData, key); -} - -const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataTable& userData, const TUserDataKey& key) { - return userData.FindPtr(key); -} - -TUserDataBlock* TUserDataStorage::FindUserDataBlock(TUserDataTable& userData, const TStringBuf& name) { - const auto key = ComposeUserDataKey(name); - return FindUserDataBlock(userData, key); -} - -TUserDataBlock* TUserDataStorage::FindUserDataBlock(TUserDataTable& userData, const TUserDataKey& key) { - return userData.FindPtr(key); -} - -TString TUserDataStorage::MakeFullName(const TStringBuf& name) { - return name.StartsWith(Sep) ? TString(name) : HomePath + name; -} - -TString TUserDataStorage::MakeFolderName(const TStringBuf& name) { - auto fullName = MakeFullName(name); - if (!fullName.EndsWith(Sep)) { - fullName += Sep; - } - - return fullName; -} - +} + +const TUserDataBlock& TUserDataStorage::GetUserDataBlock(const TUserDataKey& key) const { + auto block = FindUserDataBlock(key); + if (!block) { + ythrow yexception() << "Failed to find user data block by key " << key; + } + + return *block; +} + +const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TStringBuf& name) const { + const auto key = ComposeUserDataKey(name); + return FindUserDataBlock(key); +} + +const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataKey& key) const { + return FindUserDataBlock(UserData_, key); +} + +TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataKey& key) { + return FindUserDataBlock(UserData_, key); +} + +const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataTable& userData, const TStringBuf& name) { + const auto key = ComposeUserDataKey(name); + return FindUserDataBlock(userData, key); +} + +const TUserDataBlock* TUserDataStorage::FindUserDataBlock(const TUserDataTable& userData, const TUserDataKey& key) { + return userData.FindPtr(key); +} + +TUserDataBlock* TUserDataStorage::FindUserDataBlock(TUserDataTable& userData, const TStringBuf& name) { + const auto key = ComposeUserDataKey(name); + return FindUserDataBlock(userData, key); +} + +TUserDataBlock* TUserDataStorage::FindUserDataBlock(TUserDataTable& userData, const TUserDataKey& key) { + return userData.FindPtr(key); +} + +TString TUserDataStorage::MakeFullName(const TStringBuf& name) { + return name.StartsWith(Sep) ? TString(name) : HomePath + name; +} + +TString TUserDataStorage::MakeFolderName(const TStringBuf& name) { + auto fullName = MakeFullName(name); + if (!fullName.EndsWith(Sep)) { + fullName += Sep; + } + + return fullName; +} + TString TUserDataStorage::MakeRelativeName(const TStringBuf& name) { if (name.StartsWith(HomePath)) { return TString(name.substr(HomePath.size())); @@ -129,245 +129,245 @@ TString TUserDataStorage::MakeRelativeName(const TStringBuf& name) { return name.StartsWith(Sep) ? TString(name.substr(1)) : TString(name); } -TUserDataKey TUserDataStorage::ComposeUserDataKey(const TStringBuf& name) { - auto fullName = MakeFullName(name); - return TUserDataKey::File(fullName); -} - -bool TUserDataStorage::ContainsUserDataFolder(const TStringBuf& name) const { - return FindUserDataFolder(name, 1).Defined(); -} - -TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> TUserDataStorage::FindUserDataFolder(const TStringBuf& name, ui32 maxFileCount) const { +TUserDataKey TUserDataStorage::ComposeUserDataKey(const TStringBuf& name) { + auto fullName = MakeFullName(name); + return TUserDataKey::File(fullName); +} + +bool TUserDataStorage::ContainsUserDataFolder(const TStringBuf& name) const { + return FindUserDataFolder(name, 1).Defined(); +} + +TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> TUserDataStorage::FindUserDataFolder(const TStringBuf& name, ui32 maxFileCount) const { return FindUserDataFolder(UserData_,name,maxFileCount); } TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> TUserDataStorage::FindUserDataFolder(const TUserDataTable& userData, const TStringBuf& name, ui32 maxFileCount) { - auto fullName = MakeFolderName(name); - TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> res; + auto fullName = MakeFolderName(name); + TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> res; EnumFolderContent(userData, fullName, maxFileCount, [&](auto& key, auto& block) { - if (!res) { - res.ConstructInPlace(); - } - - res->emplace(key, &block); - }); - - return res; -} - -void TUserDataStorage::FillUserDataTokens() { - for (auto& p : UserData_) { - TryFillUserDataToken(p.second); - } -} - -void TUserDataStorage::TryFillUserDataToken(TUserDataBlock& block) const { - if (block.Type != EUserDataType::URL) { - return; - } - - // no need to overwrite existing value - if (block.UrlToken) { - return; - } - - if (!TokenResolver_) { - return; - } - - block.UrlToken = TokenResolver_(block.Data).GetOrElse({}); -} - -std::map<TString, const TUserDataBlock*> TUserDataStorage::GetDirectoryContent(const TStringBuf& path, ui32 maxFileCount) const { - const auto fullPath = MakeFolderName(path); - - std::map<TString, const TUserDataBlock*> result; - EnumFolderContent(UserData_, fullPath, maxFileCount, [&](auto& key, auto& block) { - const auto name = key.Alias().substr(fullPath.size()); - const auto pos = name.find(Sep); - if (TString::npos == pos) - result.emplace(name, &block); - else - result.emplace(name.substr(0U, pos), nullptr); - }); - - return result; -} - -const TUserDataBlock& TUserDataStorage::Freeze(const TUserDataKey& key) { - const TUserDataBlock& block = GetUserDataBlock(key); - if (block.FrozenFile) { - return block; - } - - // do it outside of the lock - auto link = FileStorage_.FreezeFile(block); - return RegisterLink(key, link); -} - -const TUserDataBlock* TUserDataStorage::FreezeNoThrow(const TUserDataKey& key, TString& errorMessage) { - try { - return &Freeze(key); - } catch (const std::exception& e) { - errorMessage = TStringBuilder() << "Failed to freeze file with key " << key << ", details: " << e.what(); - return nullptr; - } -} - -const TUserDataBlock* TUserDataStorage::FreezeUdfNoThrow(const TUserDataKey& key, TString& errorMessage) { - const TUserDataBlock* block = FreezeNoThrow(key, errorMessage); - if (!block) { - return nullptr; - } - - if (!ScannedUdfs.insert(key).second) { - // already scanned - return block; - } - - if (!UdfIndex) { - return block; - } - - try { - TString scope = "ScanUdfStrategy " + key.Alias(); - YQL_PROFILE_SCOPE(DEBUG, scope.c_str()); - Y_ENSURE(UdfResolver); - Y_ENSURE(UdfIndex); - LoadRichMetadataToUdfIndex(*UdfResolver, *block, TUdfIndex::EOverrideMode::ReplaceWithNew, *UdfIndex); - } catch (const std::exception& e) { - errorMessage = TStringBuilder() << "Failed to scan udf with key " << key << ", details: " << e.what(); - return nullptr; - } - - return block; -} - -NThreading::TFuture<std::function<TUserDataBlock()>> TUserDataStorage::FreezeAsync(const TUserDataKey& key) { - auto block = GetUserDataBlock(key); - if (block.FrozenFile) { - return MakeFutureWithConstantAction(block); - } - - return MapFutureAction(FileStorage_.FreezeFileAsync(block), [this, key](TFileLinkPtr link) { - return this->RegisterLink(key, link); - }); -} - -TUserDataBlock& TUserDataStorage::RegisterLink(const TUserDataKey& key, TFileLinkPtr link) { - auto block = FindUserDataBlock(key); - Y_ENSURE(block); - - if (!block->FrozenFile) { - block->FrozenFile = link; - } - return *block; -} - -const TString& GetDefaultFilePrefix() { - return HomePath; -} - -void FillUserDataTableFromFileSystem(const TString& aliasPrefix, const TString& path, bool isLibrary, TUserDataTable& userData) { - if (!NFs::Exists(path) || !TFileStat(path).IsDir()) { - return; - } - - TDirIterator dir(path); - + if (!res) { + res.ConstructInPlace(); + } + + res->emplace(key, &block); + }); + + return res; +} + +void TUserDataStorage::FillUserDataTokens() { + for (auto& p : UserData_) { + TryFillUserDataToken(p.second); + } +} + +void TUserDataStorage::TryFillUserDataToken(TUserDataBlock& block) const { + if (block.Type != EUserDataType::URL) { + return; + } + + // no need to overwrite existing value + if (block.UrlToken) { + return; + } + + if (!TokenResolver_) { + return; + } + + block.UrlToken = TokenResolver_(block.Data).GetOrElse({}); +} + +std::map<TString, const TUserDataBlock*> TUserDataStorage::GetDirectoryContent(const TStringBuf& path, ui32 maxFileCount) const { + const auto fullPath = MakeFolderName(path); + + std::map<TString, const TUserDataBlock*> result; + EnumFolderContent(UserData_, fullPath, maxFileCount, [&](auto& key, auto& block) { + const auto name = key.Alias().substr(fullPath.size()); + const auto pos = name.find(Sep); + if (TString::npos == pos) + result.emplace(name, &block); + else + result.emplace(name.substr(0U, pos), nullptr); + }); + + return result; +} + +const TUserDataBlock& TUserDataStorage::Freeze(const TUserDataKey& key) { + const TUserDataBlock& block = GetUserDataBlock(key); + if (block.FrozenFile) { + return block; + } + + // do it outside of the lock + auto link = FileStorage_.FreezeFile(block); + return RegisterLink(key, link); +} + +const TUserDataBlock* TUserDataStorage::FreezeNoThrow(const TUserDataKey& key, TString& errorMessage) { + try { + return &Freeze(key); + } catch (const std::exception& e) { + errorMessage = TStringBuilder() << "Failed to freeze file with key " << key << ", details: " << e.what(); + return nullptr; + } +} + +const TUserDataBlock* TUserDataStorage::FreezeUdfNoThrow(const TUserDataKey& key, TString& errorMessage) { + const TUserDataBlock* block = FreezeNoThrow(key, errorMessage); + if (!block) { + return nullptr; + } + + if (!ScannedUdfs.insert(key).second) { + // already scanned + return block; + } + + if (!UdfIndex) { + return block; + } + + try { + TString scope = "ScanUdfStrategy " + key.Alias(); + YQL_PROFILE_SCOPE(DEBUG, scope.c_str()); + Y_ENSURE(UdfResolver); + Y_ENSURE(UdfIndex); + LoadRichMetadataToUdfIndex(*UdfResolver, *block, TUdfIndex::EOverrideMode::ReplaceWithNew, *UdfIndex); + } catch (const std::exception& e) { + errorMessage = TStringBuilder() << "Failed to scan udf with key " << key << ", details: " << e.what(); + return nullptr; + } + + return block; +} + +NThreading::TFuture<std::function<TUserDataBlock()>> TUserDataStorage::FreezeAsync(const TUserDataKey& key) { + auto block = GetUserDataBlock(key); + if (block.FrozenFile) { + return MakeFutureWithConstantAction(block); + } + + return MapFutureAction(FileStorage_.FreezeFileAsync(block), [this, key](TFileLinkPtr link) { + return this->RegisterLink(key, link); + }); +} + +TUserDataBlock& TUserDataStorage::RegisterLink(const TUserDataKey& key, TFileLinkPtr link) { + auto block = FindUserDataBlock(key); + Y_ENSURE(block); + + if (!block->FrozenFile) { + block->FrozenFile = link; + } + return *block; +} + +const TString& GetDefaultFilePrefix() { + return HomePath; +} + +void FillUserDataTableFromFileSystem(const TString& aliasPrefix, const TString& path, bool isLibrary, TUserDataTable& userData) { + if (!NFs::Exists(path) || !TFileStat(path).IsDir()) { + return; + } + + TDirIterator dir(path); + for (auto it = dir.begin(); it != dir.end(); ++it) { - if (FTS_F == it->fts_info) { - TString filePath = it->fts_path; + if (FTS_F == it->fts_info) { + TString filePath = it->fts_path; if (!filePath.empty()) { - auto ptr = &*filePath.begin(); + auto ptr = &*filePath.begin(); for (size_t i = 0; i < filePath.size(); ++i) { - if (ptr[i] == '\\') { - ptr[i] = Sep; - } - } - } - - const TString alias = aliasPrefix + filePath.substr(path.size()); - - if (filePath.EndsWith(".url")) { - TFile file(filePath, EOpenModeFlag::RdOnly); - if (file.GetLength() > 0) { - auto& entry = userData[TUserDataKey::File(alias)]; - entry.Type = EUserDataType::URL; - entry.Usage.Set(EUserDataBlockUsage::Library, isLibrary); - std::vector<TString::value_type> buffer(file.GetLength()); - file.Load(buffer.data(), buffer.size()); - entry.Data = StripStringRight(TString(buffer.data(), buffer.size())); - } - } else { - auto& entry = userData[TUserDataKey::File(alias)]; - entry.Usage.Set(EUserDataBlockUsage::Library, isLibrary); - entry.Type = EUserDataType::PATH; - entry.Data = filePath; - } - } - } -} - -void FillUserDataTableFromFileSystem(const NYqlMountConfig::TMountConfig& mount, TUserDataTable& userData) { - for (const auto& mp : mount.GetMountPoints()) { - FillUserDataTableFromFileSystem(mp.GetRootAlias(), mp.GetMountPoint(), mp.GetLibrary(), userData); - } -} - -NThreading::TFuture<std::function<TUserDataTable()>> FreezeUserDataTableIfNeeded(TUserDataStorage::TPtr userDataStorage, const TUserDataTable& files, const std::function<bool(const TString&)>& urlDownloadFilter) { - if (files.empty()) { - return MakeFutureWithConstantAction(TUserDataTable()); - } - - TVector<TUserDataKey> keysForDownloading; - for (auto& p : files) { - if (p.second.FrozenFile) { - continue; - } - - if (p.second.Type != EUserDataType::URL || urlDownloadFilter(p.second.Data)) { - keysForDownloading.push_back(p.first); - } - } - - // fast exit - if (keysForDownloading.empty()) { - return MakeFutureWithConstantAction(files); - } - - TVector<NThreading::TFuture<std::function<std::pair<TUserDataKey, TUserDataBlock>()>>> futures; - TVector<NThreading::TFuture<void>> voidFutures; - futures.reserve(keysForDownloading.size()); - voidFutures.reserve(keysForDownloading.size()); - for (auto& k : keysForDownloading) { - auto f = MapFutureAction(userDataStorage->FreezeAsync(k), [k](const TUserDataBlock& block) { - return std::make_pair(k, block); - }); - - futures.push_back(f); - voidFutures.push_back(f.IgnoreResult()); - } - + if (ptr[i] == '\\') { + ptr[i] = Sep; + } + } + } + + const TString alias = aliasPrefix + filePath.substr(path.size()); + + if (filePath.EndsWith(".url")) { + TFile file(filePath, EOpenModeFlag::RdOnly); + if (file.GetLength() > 0) { + auto& entry = userData[TUserDataKey::File(alias)]; + entry.Type = EUserDataType::URL; + entry.Usage.Set(EUserDataBlockUsage::Library, isLibrary); + std::vector<TString::value_type> buffer(file.GetLength()); + file.Load(buffer.data(), buffer.size()); + entry.Data = StripStringRight(TString(buffer.data(), buffer.size())); + } + } else { + auto& entry = userData[TUserDataKey::File(alias)]; + entry.Usage.Set(EUserDataBlockUsage::Library, isLibrary); + entry.Type = EUserDataType::PATH; + entry.Data = filePath; + } + } + } +} + +void FillUserDataTableFromFileSystem(const NYqlMountConfig::TMountConfig& mount, TUserDataTable& userData) { + for (const auto& mp : mount.GetMountPoints()) { + FillUserDataTableFromFileSystem(mp.GetRootAlias(), mp.GetMountPoint(), mp.GetLibrary(), userData); + } +} + +NThreading::TFuture<std::function<TUserDataTable()>> FreezeUserDataTableIfNeeded(TUserDataStorage::TPtr userDataStorage, const TUserDataTable& files, const std::function<bool(const TString&)>& urlDownloadFilter) { + if (files.empty()) { + return MakeFutureWithConstantAction(TUserDataTable()); + } + + TVector<TUserDataKey> keysForDownloading; + for (auto& p : files) { + if (p.second.FrozenFile) { + continue; + } + + if (p.second.Type != EUserDataType::URL || urlDownloadFilter(p.second.Data)) { + keysForDownloading.push_back(p.first); + } + } + + // fast exit + if (keysForDownloading.empty()) { + return MakeFutureWithConstantAction(files); + } + + TVector<NThreading::TFuture<std::function<std::pair<TUserDataKey, TUserDataBlock>()>>> futures; + TVector<NThreading::TFuture<void>> voidFutures; + futures.reserve(keysForDownloading.size()); + voidFutures.reserve(keysForDownloading.size()); + for (auto& k : keysForDownloading) { + auto f = MapFutureAction(userDataStorage->FreezeAsync(k), [k](const TUserDataBlock& block) { + return std::make_pair(k, block); + }); + + futures.push_back(f); + voidFutures.push_back(f.IgnoreResult()); + } + return NThreading::WaitExceptionOrAll(voidFutures).Apply([files = files, futures = std::move(futures)](NThreading::TFuture<void> f) mutable { - std::function<TUserDataTable()> result = [f, files, futures]() mutable { - // rethrow exception if any - f.GetValue(); - - for (auto func : futures) { - std::pair<TUserDataKey, TUserDataBlock> p = func.GetValue()(); + std::function<TUserDataTable()> result = [f, files, futures]() mutable { + // rethrow exception if any + f.GetValue(); + + for (auto func : futures) { + std::pair<TUserDataKey, TUserDataBlock> p = func.GetValue()(); Y_ENSURE(files.contains(p.first)); - files[p.first].FrozenFile = p.second.FrozenFile; - } - - return files; - }; - - return result; - }); -} - + files[p.first].FrozenFile = p.second.FrozenFile; + } + + return files; + }; + + return result; + }); +} + TVector<TString> TUserDataStorage::GetLibraries() const { TVector<TString> result; for (const auto& x : UserData_) { @@ -378,6 +378,6 @@ TVector<TString> TUserDataStorage::GetLibraries() const { } return result; -} +} } diff --git a/ydb/library/yql/core/yql_user_data_storage.h b/ydb/library/yql/core/yql_user_data_storage.h index f1575cfffe..813e6e5ff0 100644 --- a/ydb/library/yql/core/yql_user_data_storage.h +++ b/ydb/library/yql/core/yql_user_data_storage.h @@ -1,92 +1,92 @@ -#pragma once - -#include "yql_holding_file_storage.h" -#include "yql_udf_index.h" -#include "yql_udf_resolver.h" -#include "yql_user_data.h" - +#pragma once + +#include "yql_holding_file_storage.h" +#include "yql_udf_index.h" +#include "yql_udf_resolver.h" +#include "yql_user_data.h" + #include <ydb/library/yql/protos/yql_mount.pb.h> - -#include <util/generic/maybe.h> - -namespace NYql { - -class TUserDataStorage : public TThrRefBase { -public: - typedef TIntrusivePtr<TUserDataStorage> TPtr; - typedef std::function<TMaybe<TString>(const TString& url)> TTokenResolver; - -public: - TUserDataStorage(TFileStoragePtr fileStorage, TUserDataTable data, IUdfResolver::TPtr udfResolver, TUdfIndex::TPtr udfIndex); - void SetTokenResolver(TTokenResolver tokenResolver); - - void AddUserDataBlock(const TStringBuf& name, const TUserDataBlock& block); - void AddUserDataBlock(const TUserDataKey& key, const TUserDataBlock& block); - - bool ContainsUserDataBlock(const TStringBuf& name) const; - bool ContainsUserDataBlock(const TUserDataKey& key) const; - const TUserDataBlock& GetUserDataBlock(const TUserDataKey& key) const; - const TUserDataBlock* FindUserDataBlock(const TStringBuf& name) const; - const TUserDataBlock* FindUserDataBlock(const TUserDataKey& key) const; - TUserDataBlock* FindUserDataBlock(const TUserDataKey& key); - - static const TUserDataBlock* FindUserDataBlock(const TUserDataTable& userData, const TStringBuf& name); - static const TUserDataBlock* FindUserDataBlock(const TUserDataTable& userData, const TUserDataKey& key); - - static TUserDataBlock* FindUserDataBlock(TUserDataTable& userData, const TStringBuf& name); - static TUserDataBlock* FindUserDataBlock(TUserDataTable& userData, const TUserDataKey& key); - - bool ContainsUserDataFolder(const TStringBuf& name) const; - TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> FindUserDataFolder(const TStringBuf& name, ui32 maxFileCount = ~0u) const; + +#include <util/generic/maybe.h> + +namespace NYql { + +class TUserDataStorage : public TThrRefBase { +public: + typedef TIntrusivePtr<TUserDataStorage> TPtr; + typedef std::function<TMaybe<TString>(const TString& url)> TTokenResolver; + +public: + TUserDataStorage(TFileStoragePtr fileStorage, TUserDataTable data, IUdfResolver::TPtr udfResolver, TUdfIndex::TPtr udfIndex); + void SetTokenResolver(TTokenResolver tokenResolver); + + void AddUserDataBlock(const TStringBuf& name, const TUserDataBlock& block); + void AddUserDataBlock(const TUserDataKey& key, const TUserDataBlock& block); + + bool ContainsUserDataBlock(const TStringBuf& name) const; + bool ContainsUserDataBlock(const TUserDataKey& key) const; + const TUserDataBlock& GetUserDataBlock(const TUserDataKey& key) const; + const TUserDataBlock* FindUserDataBlock(const TStringBuf& name) const; + const TUserDataBlock* FindUserDataBlock(const TUserDataKey& key) const; + TUserDataBlock* FindUserDataBlock(const TUserDataKey& key); + + static const TUserDataBlock* FindUserDataBlock(const TUserDataTable& userData, const TStringBuf& name); + static const TUserDataBlock* FindUserDataBlock(const TUserDataTable& userData, const TUserDataKey& key); + + static TUserDataBlock* FindUserDataBlock(TUserDataTable& userData, const TStringBuf& name); + static TUserDataBlock* FindUserDataBlock(TUserDataTable& userData, const TUserDataKey& key); + + bool ContainsUserDataFolder(const TStringBuf& name) const; + TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> FindUserDataFolder(const TStringBuf& name, ui32 maxFileCount = ~0u) const; static TMaybe<std::map<TUserDataKey, const TUserDataBlock*>> FindUserDataFolder(const TUserDataTable& userData, const TStringBuf& name, ui32 maxFileCount = ~0u); - void FillUserDataTokens(); - void TryFillUserDataToken(TUserDataBlock& block) const; - std::map<TString, const TUserDataBlock*> GetDirectoryContent(const TStringBuf& path, ui32 maxFileCount = ~0u) const; - static TString MakeFullName(const TStringBuf& name); - static TString MakeFolderName(const TStringBuf& name); - static TUserDataKey ComposeUserDataKey(const TStringBuf& name); + void FillUserDataTokens(); + void TryFillUserDataToken(TUserDataBlock& block) const; + std::map<TString, const TUserDataBlock*> GetDirectoryContent(const TStringBuf& path, ui32 maxFileCount = ~0u) const; + static TString MakeFullName(const TStringBuf& name); + static TString MakeFolderName(const TStringBuf& name); + static TUserDataKey ComposeUserDataKey(const TStringBuf& name); static TString MakeRelativeName(const TStringBuf& name); TVector<TString> GetLibraries() const; - - // working with frozen files - // download file and fill FrozenFile property of data block - // returned pointer is valid until storage destruction, but keep in mind storage is used in multi-threading env and data could be changed - // todo: consider more robust approach for multi-threading - // there is a guarantee we return the same TFileLinkPtr for provided key if this method get called multiple times from different threads - // program should work with single version of downloaded url when block type is URL - // method should support parallel file downloading for different keys - const TUserDataBlock& Freeze(const TUserDataKey& key); - const TUserDataBlock* FreezeNoThrow(const TUserDataKey& key, TString& errorMessage); - - // as above + udf will be scanned and meta info put into UdfIndex - const TUserDataBlock* FreezeUdfNoThrow(const TUserDataKey& key, TString& errorMessage); - - // returns function which will register value in cache after invocation - NThreading::TFuture<std::function<TUserDataBlock()>> FreezeAsync(const TUserDataKey& key); - -private: - TUserDataBlock& RegisterLink(const TUserDataKey& key, TFileLinkPtr link); - -private: - THoldingFileStorage FileStorage_; - TUserDataTable UserData_; - IUdfResolver::TPtr UdfResolver; - TUdfIndex::TPtr UdfIndex; - TTokenResolver TokenResolver_; - - THashSet<TUserDataKey, TUserDataKey::THash, TUserDataKey::TEqualTo> ScannedUdfs; - std::function<void(const TUserDataBlock& block)> ScanUdfStrategy_; -}; - -const TString& GetDefaultFilePrefix(); -void FillUserDataTableFromFileSystem(const NYqlMountConfig::TMountConfig& mount, TUserDataTable& userData); - -/* -Start async freeze procedure for every not frozen element of the table (if FrozenFile is nullptr or if urlDownloadFilter returns true for url) -Frozen file will be cached under data storage. -Every key of not frozen block in input table should exist in data storage. No such requirements for other keys. -Usage of input blocks will be preserved. -*/ -NThreading::TFuture<std::function<TUserDataTable()>> FreezeUserDataTableIfNeeded(TUserDataStorage::TPtr userDataStorage, const TUserDataTable& files, const std::function<bool(const TString&)>& urlDownloadFilter); -} // namespace NYql + + // working with frozen files + // download file and fill FrozenFile property of data block + // returned pointer is valid until storage destruction, but keep in mind storage is used in multi-threading env and data could be changed + // todo: consider more robust approach for multi-threading + // there is a guarantee we return the same TFileLinkPtr for provided key if this method get called multiple times from different threads + // program should work with single version of downloaded url when block type is URL + // method should support parallel file downloading for different keys + const TUserDataBlock& Freeze(const TUserDataKey& key); + const TUserDataBlock* FreezeNoThrow(const TUserDataKey& key, TString& errorMessage); + + // as above + udf will be scanned and meta info put into UdfIndex + const TUserDataBlock* FreezeUdfNoThrow(const TUserDataKey& key, TString& errorMessage); + + // returns function which will register value in cache after invocation + NThreading::TFuture<std::function<TUserDataBlock()>> FreezeAsync(const TUserDataKey& key); + +private: + TUserDataBlock& RegisterLink(const TUserDataKey& key, TFileLinkPtr link); + +private: + THoldingFileStorage FileStorage_; + TUserDataTable UserData_; + IUdfResolver::TPtr UdfResolver; + TUdfIndex::TPtr UdfIndex; + TTokenResolver TokenResolver_; + + THashSet<TUserDataKey, TUserDataKey::THash, TUserDataKey::TEqualTo> ScannedUdfs; + std::function<void(const TUserDataBlock& block)> ScanUdfStrategy_; +}; + +const TString& GetDefaultFilePrefix(); +void FillUserDataTableFromFileSystem(const NYqlMountConfig::TMountConfig& mount, TUserDataTable& userData); + +/* +Start async freeze procedure for every not frozen element of the table (if FrozenFile is nullptr or if urlDownloadFilter returns true for url) +Frozen file will be cached under data storage. +Every key of not frozen block in input table should exist in data storage. No such requirements for other keys. +Usage of input blocks will be preserved. +*/ +NThreading::TFuture<std::function<TUserDataTable()>> FreezeUserDataTableIfNeeded(TUserDataStorage::TPtr userDataStorage, const TUserDataTable& files, const std::function<bool(const TString&)>& urlDownloadFilter); +} // namespace NYql diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index deaadefb42..12d6aafb56 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -1166,7 +1166,7 @@ protected: } for (auto& [inputIndex, source] : SourcesMap) { if (TaskRunner) { source.Source = TaskRunner->GetSource(inputIndex); Y_VERIFY(source.Source);} - Y_VERIFY(SourceActorFactory); + Y_VERIFY(SourceActorFactory); const auto& inputDesc = Task.GetInputs(inputIndex); const ui64 i = inputIndex; // Crutch for clang CA_LOG_D("Create source actor for input " << i << " " << inputDesc); @@ -1222,9 +1222,9 @@ protected: continue; } const i64 freeSpace = SourceFreeSpace(source); - if (freeSpace > 0) { + if (freeSpace > 0) { NKikimr::NMiniKQL::TUnboxedValueVector batch; - Y_VERIFY(source.SourceActor); + Y_VERIFY(source.SourceActor); bool finished = false; const i64 space = source.SourceActor->GetSourceData(batch, finished, freeSpace); const ui64 index = inputIndex; diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto index 0e38ab3bbc..b6af2ca9b8 100644 --- a/ydb/library/yql/dq/actors/protos/dq_events.proto +++ b/ydb/library/yql/dq/actors/protos/dq_events.proto @@ -9,22 +9,22 @@ import "ydb/library/yql/dq/proto/dq_transport.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; -message TCheckpoint { - optional uint64 Id = 1; +message TCheckpoint { + optional uint64 Id = 1; optional uint64 Generation = 2; -}; - -/* - Data and control messages will be processed in the following order: - 1) Data - 2) Checkpoint - 3) Finished +}; + +/* + Data and control messages will be processed in the following order: + 1) Data + 2) Checkpoint + 3) Finished */ message TChannelData { optional uint64 ChannelId = 1; optional NYql.NDqProto.TData Data = 2; optional bool Finished = 3; - optional TCheckpoint Checkpoint = 4; + optional TCheckpoint Checkpoint = 4; }; message TEvRun { diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h index 5b3146d0f4..0d6db327b0 100644 --- a/ydb/library/yql/dq/opt/dq_opt.h +++ b/ydb/library/yql/dq/opt/dq_opt.h @@ -22,7 +22,7 @@ struct TDqStageSettings { bool SinglePartition = false; bool IsExternalFunction = false; - NDqProto::ETransformType TransformType = NDqProto::TRANSFORM_YANDEX_CLOUD; + NDqProto::ETransformType TransformType = NDqProto::TRANSFORM_YANDEX_CLOUD; TString TransformName; ui32 TransformConcurrency = 0; diff --git a/ydb/library/yql/minikql/codegen/codegen.cpp b/ydb/library/yql/minikql/codegen/codegen.cpp index 6e4096af22..b9c6b2ce82 100644 --- a/ydb/library/yql/minikql/codegen/codegen.cpp +++ b/ydb/library/yql/minikql/codegen/codegen.cpp @@ -309,11 +309,11 @@ public: Triple_ = llvm::Triple::normalize(triple); Module_->setTargetTriple(Triple_); - llvm::TargetOptions targetOptions; + llvm::TargetOptions targetOptions; targetOptions.EnableFastISel = true; - // init manually, this field was lost in llvm::TargetOptions ctor :/ - // make coverity happy - targetOptions.StackProtectorGuardOffset = 0; + // init manually, this field was lost in llvm::TargetOptions ctor :/ + // make coverity happy + targetOptions.StackProtectorGuardOffset = 0; std::string what; auto&& engineBuilder = llvm::EngineBuilder(std::move(module)); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp index 4e13f63a14..e6b7df4a35 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp @@ -8,8 +8,8 @@ namespace NKikimr { namespace NMiniKQL { -TStatKey Combine_FlushesCount("Combine_FlushesCount", true); -TStatKey Combine_MaxRowsCount("Combine_MaxRowsCount", false); +TStatKey Combine_FlushesCount("Combine_FlushesCount", true); +TStatKey Combine_MaxRowsCount("Combine_MaxRowsCount", false); namespace { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp index d2182bbc23..b9ee46ff47 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp @@ -18,9 +18,9 @@ namespace { const ui64 DEFAULT_STACK_ITEMS = 16; -static const TStatKey Join_Spill_Count("Join_Spill_Count", true); -static const TStatKey Join_Spill_MaxFileSize("Join_Spill_MaxFileSize", false); -static const TStatKey Join_Spill_MaxRowsCount("Join_Spill_MaxRowsCount", false); +static const TStatKey Join_Spill_Count("Join_Spill_Count", true); +static const TStatKey Join_Spill_MaxFileSize("Join_Spill_MaxFileSize", false); +static const TStatKey Join_Spill_MaxRowsCount("Join_Spill_MaxRowsCount", false); enum class EOutputMode { Unknown, diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp index cb2b13f86d..6ed3304178 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp @@ -17,7 +17,7 @@ namespace { const TStatKey Hop_NewHopsCount("MultiHop_NewHopsCount", true); const TStatKey Hop_ThrownEventsCount("MultiHop_ThrownEventsCount", true); const TStatKey Hop_EmptyTimeCount("MultiHop_EmptyTimeCount", true); -const TStatKey Hop_KeysCount("MultiHop_KeysCount", true); +const TStatKey Hop_KeysCount("MultiHop_KeysCount", true); constexpr ui32 StateVersion = 1; @@ -231,7 +231,7 @@ public: CloseOldBuckets(*newWatermark, newHopsStat); } } - MKQL_SET_STAT(Ctx.Stats, Hop_KeysCount, StatesMap.size()); + MKQL_SET_STAT(Ctx.Stats, Hop_KeysCount, StatesMap.size()); } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp index 4571141d80..65f641d13f 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp @@ -13,8 +13,8 @@ using NYql::EnsureDynamicCast; namespace { -static const TStatKey Switch_FlushesCount("Switch_FlushesCount", true); -static const TStatKey Switch_MaxRowsCount("Switch_MaxRowsCount", false); +static const TStatKey Switch_FlushesCount("Switch_FlushesCount", true); +static const TStatKey Switch_MaxRowsCount("Switch_MaxRowsCount", false); using TPagedUnboxedValueList = TPagedList<NUdf::TUnboxedValue>; diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index 55838c4012..f324e2e5c7 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -217,8 +217,8 @@ PEERDIR( ydb/library/yql/utils ) -NO_COMPILER_WARNINGS() - +NO_COMPILER_WARNINGS() + IF (NOT MKQL_DISABLE_CODEGEN) PEERDIR( ydb/library/yql/minikql/codegen diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index f2b188ae47..7bbc28af70 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -21,34 +21,34 @@ using namespace NDetail; namespace { -#ifndef MKQL_DISABLE_CODEGEN +#ifndef MKQL_DISABLE_CODEGEN constexpr ui64 TotalFunctionsLimit = 1000; constexpr ui64 TotalInstructionsLimit = 100000; constexpr ui64 MaxFunctionInstructionsLimit = 50000; -#endif +#endif const ui64 IS_NODE_REACHABLE = 1; -const static TStatKey PagePool_PeakAllocated("PagePool_PeakAllocated", false); -const static TStatKey PagePool_PeakUsed("PagePool_PeakUsed", false); -const static TStatKey PagePool_AllocCount("PagePool_AllocCount", true); -const static TStatKey PagePool_PageAllocCount("PagePool_PageAllocCount", true); -const static TStatKey PagePool_PageHitCount("PagePool_PageHitCount", true); -const static TStatKey PagePool_PageMissCount("PagePool_PageMissCount", true); -const static TStatKey PagePool_OffloadedAllocCount("PagePool_OffloadedAllocCount", true); -const static TStatKey PagePool_OffloadedBytes("PagePool_OffloadedBytes", true); - -const static TStatKey CodeGen_FullTime("CodeGen_FullTime", true); -const static TStatKey CodeGen_GenerateTime("CodeGen_GenerateTime", true); -const static TStatKey CodeGen_CompileTime("CodeGen_CompileTime", true); -const static TStatKey CodeGen_TotalFunctions("CodeGen_TotalFunctions", true); -const static TStatKey CodeGen_TotalInstructions("CodeGen_TotalInstructions", true); -const static TStatKey CodeGen_MaxFunctionInstructions("CodeGen_MaxFunctionInstructions", false); -const static TStatKey CodeGen_ModulePassTime("CodeGen_ModulePassTime", true); -const static TStatKey CodeGen_FinalizeTime("CodeGen_FinalizeTime", true); - -const static TStatKey Mkql_TotalNodes("Mkql_TotalNodes", true); -const static TStatKey Mkql_CodegenFunctions("Mkql_CodegenFunctions", true); +const static TStatKey PagePool_PeakAllocated("PagePool_PeakAllocated", false); +const static TStatKey PagePool_PeakUsed("PagePool_PeakUsed", false); +const static TStatKey PagePool_AllocCount("PagePool_AllocCount", true); +const static TStatKey PagePool_PageAllocCount("PagePool_PageAllocCount", true); +const static TStatKey PagePool_PageHitCount("PagePool_PageHitCount", true); +const static TStatKey PagePool_PageMissCount("PagePool_PageMissCount", true); +const static TStatKey PagePool_OffloadedAllocCount("PagePool_OffloadedAllocCount", true); +const static TStatKey PagePool_OffloadedBytes("PagePool_OffloadedBytes", true); + +const static TStatKey CodeGen_FullTime("CodeGen_FullTime", true); +const static TStatKey CodeGen_GenerateTime("CodeGen_GenerateTime", true); +const static TStatKey CodeGen_CompileTime("CodeGen_CompileTime", true); +const static TStatKey CodeGen_TotalFunctions("CodeGen_TotalFunctions", true); +const static TStatKey CodeGen_TotalInstructions("CodeGen_TotalInstructions", true); +const static TStatKey CodeGen_MaxFunctionInstructions("CodeGen_MaxFunctionInstructions", false); +const static TStatKey CodeGen_ModulePassTime("CodeGen_ModulePassTime", true); +const static TStatKey CodeGen_FinalizeTime("CodeGen_FinalizeTime", true); + +const static TStatKey Mkql_TotalNodes("Mkql_TotalNodes", true); +const static TStatKey Mkql_CodegenFunctions("Mkql_CodegenFunctions", true); class TDependencyScanVisitor : public TEmptyNodeVisitor { public: @@ -436,7 +436,7 @@ private: const auto computationNode = Factory(node, ctx); if (!computationNode) { - THROW yexception() + THROW yexception() << "Computation graph builder, unsupported function: " << node.GetType()->GetName() << " type: " << Factory.target_type().name() ; } diff --git a/ydb/library/yql/minikql/computation/mkql_validate_impl.h b/ydb/library/yql/minikql/computation/mkql_validate_impl.h index c4e10ac4b1..3763b833fb 100644 --- a/ydb/library/yql/minikql/computation/mkql_validate_impl.h +++ b/ydb/library/yql/minikql/computation/mkql_validate_impl.h @@ -18,7 +18,7 @@ struct TValidateErrorPolicyThrow { template<class TException> static void GenerateExc(const TException& exc) { static_assert(std::is_base_of<yexception, TException>::value, "Must be derived from yexception"); - ythrow TException() << exc.AsStrBuf(); + ythrow TException() << exc.AsStrBuf(); } }; diff --git a/ydb/library/yql/minikql/mkql_function_registry.cpp b/ydb/library/yql/minikql/mkql_function_registry.cpp index d3545ad138..699049bd44 100644 --- a/ydb/library/yql/minikql/mkql_function_registry.cpp +++ b/ydb/library/yql/minikql/mkql_function_registry.cpp @@ -99,7 +99,7 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry }; public: - TMutableFunctionRegistry(IBuiltinFunctionRegistry::TPtr builtins) + TMutableFunctionRegistry(IBuiltinFunctionRegistry::TPtr builtins) : Builtins_(std::move(builtins)) { } @@ -127,7 +127,7 @@ public: auto libIt = LoadedLibraries_.find(libraryPath); if (libIt != LoadedLibraries_.end()) { - return; + return; } else { lib = MakeIntrusive<TUdfLibrary>(); #ifdef _win32_ @@ -352,7 +352,7 @@ public: return new TMutableFunctionRegistry(*this); } - void SetBackTraceCallback(NUdf::TBackTraceCallback callback) override { + void SetBackTraceCallback(NUdf::TBackTraceCallback callback) override { BackTraceCallback_ = callback; } @@ -440,7 +440,7 @@ public: } TIntrusivePtr<IMutableFunctionRegistry> Clone() const override { - return new TMutableFunctionRegistry(Builtins_); + return new TMutableFunctionRegistry(Builtins_); } private: @@ -514,11 +514,11 @@ TIntrusivePtr<IFunctionRegistry> CreateFunctionRegistry(IBuiltinFunctionRegistry } TIntrusivePtr<IFunctionRegistry> CreateFunctionRegistry( - NKikimr::NUdf::TBackTraceCallback backtraceCallback, - IBuiltinFunctionRegistry::TPtr&& builtins, - bool allowUdfPatch, - const TVector<TString>& udfsPaths, - ui32 flags /* = 0 */) + NKikimr::NUdf::TBackTraceCallback backtraceCallback, + IBuiltinFunctionRegistry::TPtr&& builtins, + bool allowUdfPatch, + const TVector<TString>& udfsPaths, + ui32 flags /* = 0 */) { auto registry = MakeHolder<TMutableFunctionRegistry>(std::move(builtins)); if (allowUdfPatch) { diff --git a/ydb/library/yql/minikql/mkql_node.cpp b/ydb/library/yql/minikql/mkql_node.cpp index 4c6f3e45ad..14f98daba4 100644 --- a/ydb/library/yql/minikql/mkql_node.cpp +++ b/ydb/library/yql/minikql/mkql_node.cpp @@ -596,7 +596,7 @@ ui32 TStructType::GetMemberIndex(const TStringBuf& name) const { for (ui32 i = 0; i < MembersCount; ++i) { ss << " " << Members[i].first.Str(); } - THROW yexception() << "Member with name '" << name << "' not found; " + THROW yexception() << "Member with name '" << name << "' not found; " << " known members: " << ss.Str() << "."; } diff --git a/ydb/library/yql/minikql/mkql_node_serialization.cpp b/ydb/library/yql/minikql/mkql_node_serialization.cpp index e314f38cd6..e079965bc6 100644 --- a/ydb/library/yql/minikql/mkql_node_serialization.cpp +++ b/ydb/library/yql/minikql/mkql_node_serialization.cpp @@ -1045,11 +1045,11 @@ namespace { private: [[noreturn]] static void ThrowNoData() { - THROW yexception() << "No more data in buffer"; + THROW yexception() << "No more data in buffer"; } [[noreturn]] static void ThrowCorrupted() { - THROW yexception() << "Serialized data is corrupted"; + THROW yexception() << "Serialized data is corrupted"; } Y_FORCE_INLINE char Read() { diff --git a/ydb/library/yql/minikql/mkql_node_visitor.cpp b/ydb/library/yql/minikql/mkql_node_visitor.cpp index 5a4ac8e479..9605e425cc 100644 --- a/ydb/library/yql/minikql/mkql_node_visitor.cpp +++ b/ydb/library/yql/minikql/mkql_node_visitor.cpp @@ -172,7 +172,7 @@ void TThrowingNodeVisitor::Visit(TBlockType& node) { } void TThrowingNodeVisitor::ThrowUnexpectedNodeType() { - THROW yexception() << "Unexpected node type"; + THROW yexception() << "Unexpected node type"; } void TEmptyNodeVisitor::Visit(TTypeType& node) { diff --git a/ydb/library/yql/minikql/mkql_stats_registry.cpp b/ydb/library/yql/minikql/mkql_stats_registry.cpp index 431737731a..56b6070119 100644 --- a/ydb/library/yql/minikql/mkql_stats_registry.cpp +++ b/ydb/library/yql/minikql/mkql_stats_registry.cpp @@ -69,9 +69,9 @@ ui32 TStatKey::IdSequence_ = 0; TStatKey* TStatKey::KeysChain_ = nullptr; -TStatKey::TStatKey(TStringBuf name, bool deriv) +TStatKey::TStatKey(TStringBuf name, bool deriv) : Name_(name) - , Deriv_(deriv) + , Deriv_(deriv) , Id_(IdSequence_++) { bool newOne = Singleton<TKeyNames>()->AddIfNotExists(name); diff --git a/ydb/library/yql/minikql/mkql_stats_registry.h b/ydb/library/yql/minikql/mkql_stats_registry.h index 1b888153e4..a6d85dc56e 100644 --- a/ydb/library/yql/minikql/mkql_stats_registry.h +++ b/ydb/library/yql/minikql/mkql_stats_registry.h @@ -24,25 +24,25 @@ class TStatKey: public TIntrusiveSListItem<TStatKey> { // -- instance part -- public: - TStatKey(TStringBuf name, bool deriv); + TStatKey(TStringBuf name, bool deriv); TStringBuf GetName() const noexcept { return Name_; } - bool IsDeriv() const noexcept { return Deriv_; } + bool IsDeriv() const noexcept { return Deriv_; } ui32 GetId() const noexcept { return Id_; } - static ui32 GetMaxId() { return IdSequence_; } - - template <typename F> - static void ForEach(F f) { - for (TStatKey* p = KeysChain_; p;) { - f(*p); - auto n = p->Next(); - p = n ? n->Node() : nullptr; - } - } - + static ui32 GetMaxId() { return IdSequence_; } + + template <typename F> + static void ForEach(F f) { + for (TStatKey* p = KeysChain_; p;) { + f(*p); + auto n = p->Next(); + p = n ? n->Node() : nullptr; + } + } + private: TStringBuf Name_; - bool Deriv_; + bool Deriv_; ui32 Id_; }; @@ -71,9 +71,9 @@ struct IStatsRegistry { // TConsumer = void(const TStatKey& key, i64 value) template <typename TConsumer> void ForEachStat(TConsumer c) const { - TStatKey::ForEach([this, &c](const auto& key) { - c(key, GetStat(key)); - }); + TStatKey::ForEach([this, &c](const auto& key) { + c(key, GetStat(key)); + }); } }; diff --git a/ydb/library/yql/minikql/mkql_stats_registry_ut.cpp b/ydb/library/yql/minikql/mkql_stats_registry_ut.cpp index 0504fb2986..c94959649b 100644 --- a/ydb/library/yql/minikql/mkql_stats_registry_ut.cpp +++ b/ydb/library/yql/minikql/mkql_stats_registry_ut.cpp @@ -9,7 +9,7 @@ using namespace NMiniKQL; Y_UNIT_TEST_SUITE(TStatsRegistryTest) { Y_UNIT_TEST(Stats) { - static TStatKey key("key", false); + static TStatKey key("key", false); auto stats = CreateDefaultStatsRegistry(); stats->SetStat(key, 37); @@ -38,9 +38,9 @@ Y_UNIT_TEST_SUITE(TStatsRegistryTest) { } Y_UNIT_TEST(ForEach) { - static TStatKey key1("key1", false), key2("key2", true); - UNIT_ASSERT(!key1.IsDeriv()); - UNIT_ASSERT(key2.IsDeriv()); + static TStatKey key1("key1", false), key2("key2", true); + UNIT_ASSERT(!key1.IsDeriv()); + UNIT_ASSERT(key2.IsDeriv()); auto stats = CreateDefaultStatsRegistry(); stats->SetStat(key1, 42); @@ -61,7 +61,7 @@ Y_UNIT_TEST_SUITE(TStatsRegistryTest) { Y_UNIT_TEST(DuplicateKeys) { TString error; try { - static TStatKey key("my_key", false), sameKey("my_key", false); + static TStatKey key("my_key", false), sameKey("my_key", false); // avoid variables elimitation Cerr << key.GetId() << ": " << key.GetName() << '\n' << sameKey.GetId() << ": " << sameKey.GetName() << Endl; diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp index 4aecb564bf..5812baa0a6 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.cpp +++ b/ydb/library/yql/minikql/mkql_type_builder.cpp @@ -1396,9 +1396,9 @@ NUdf::IFunctionTypeInfoBuilder1& TFunctionTypeInfoBuilder::UserTypeImpl( void TFunctionTypeInfoBuilder::SetError(const NUdf::TStringRef& error) { - if (!Error_) { - Error_ = error; - } + if (!Error_) { + Error_ = error; + } } void TFunctionTypeInfoBuilder::Build(TFunctionTypeInfo* funcInfo) diff --git a/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h b/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h index e9da86f7e7..9cea53e895 100644 --- a/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h +++ b/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h @@ -1,27 +1,27 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/parser/proto_ast/proto_ast.h> #include <ydb/library/yql/public/issue/yql_issue.h> - + namespace NSQLTranslation { - -class TErrorCollectorOverIssues : public NProtoAST::IErrorCollector { -public: + +class TErrorCollectorOverIssues : public NProtoAST::IErrorCollector { +public: TErrorCollectorOverIssues(NYql::TIssues& issues, size_t maxErrors, const TString& file) - : IErrorCollector(maxErrors) - , Issues_(issues) + : IErrorCollector(maxErrors) + , Issues_(issues) , File_(file) - { - } - -private: - void AddError(ui32 line, ui32 col, const TString& message) override { + { + } + +private: + void AddError(ui32 line, ui32 col, const TString& message) override { Issues_.AddIssue(NYql::TPosition(col, line, File_), message); - } - -private: - NYql::TIssues& Issues_; + } + +private: + NYql::TIssues& Issues_; const TString File_; -}; - +}; + } // namespace NSQLTranslation diff --git a/ydb/library/yql/parser/proto_ast/proto_ast.cpp b/ydb/library/yql/parser/proto_ast/proto_ast.cpp index 097ba08c85..faeb30373e 100644 --- a/ydb/library/yql/parser/proto_ast/proto_ast.cpp +++ b/ydb/library/yql/parser/proto_ast/proto_ast.cpp @@ -13,16 +13,16 @@ IErrorCollector::~IErrorCollector() } void IErrorCollector::Error(ui32 line, ui32 col, const TString& message) { - if (NumErrors + 1 == MaxErrors) { - AddError(0, 0, "Too many errors"); - ++NumErrors; - } - - if (NumErrors >= MaxErrors) { - ythrow TTooManyErrors() << "Too many errors"; - } - - AddError(line, col, message); + if (NumErrors + 1 == MaxErrors) { + AddError(0, 0, "Too many errors"); + ++NumErrors; + } + + if (NumErrors >= MaxErrors) { + ythrow TTooManyErrors() << "Too many errors"; + } + + AddError(line, col, message); ++NumErrors; } diff --git a/ydb/library/yql/parser/proto_ast/proto_ast.h b/ydb/library/yql/parser/proto_ast/proto_ast.h index 5bb8a9f5f9..757d422203 100644 --- a/ydb/library/yql/parser/proto_ast/proto_ast.h +++ b/ydb/library/yql/parser/proto_ast/proto_ast.h @@ -32,15 +32,15 @@ namespace NProtoAST { } } - class TTooManyErrors : public yexception { - }; - + class TTooManyErrors : public yexception { + }; + class IErrorCollector { public: - explicit IErrorCollector(size_t maxErrors); + explicit IErrorCollector(size_t maxErrors); virtual ~IErrorCollector(); - // throws TTooManyErrors + // throws TTooManyErrors void Error(ui32 line, ui32 col, const TString& message); private: @@ -57,7 +57,7 @@ namespace NProtoAST { virtual ~TErrorOutput(); private: - void AddError(ui32 line, ui32 col, const TString& message) override; + void AddError(ui32 line, ui32 col, const TString& message) override; public: IOutputStream& Err; @@ -83,9 +83,9 @@ namespace NProtoAST { try { Lexer.ReportErrors(&errors); return Parser.Parse(Lexer, &errors); - } catch (const TTooManyErrors&) { - return nullptr; - } catch (const yexception& e) { + } catch (const TTooManyErrors&) { + return nullptr; + } catch (const yexception& e) { errors.Error(0, 0, e.what()); return nullptr; } diff --git a/ydb/library/yql/providers/common/codec/ya.make b/ydb/library/yql/providers/common/codec/ya.make index b3f3a92db5..aaa20c85c3 100644 --- a/ydb/library/yql/providers/common/codec/ya.make +++ b/ydb/library/yql/providers/common/codec/ya.make @@ -7,10 +7,10 @@ SRCS( yql_codec.h yql_codec_buf.cpp yql_codec_buf.h - yql_codec_results.cpp - yql_codec_results.h - yql_restricted_yson.cpp - yql_restricted_yson.h + yql_codec_results.cpp + yql_codec_results.h + yql_restricted_yson.cpp + yql_restricted_yson.h yql_codec_type_flags.cpp yql_codec_type_flags.h ) diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 7587e188fa..1c74c9960f 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -1,5 +1,5 @@ #include "yql_codec.h" -#include "yql_restricted_yson.h" +#include "yql_restricted_yson.h" #include "yql_codec_type_flags.h" #include <ydb/library/yql/core/yql_expr_type_annotation.h> @@ -30,9 +30,9 @@ using namespace NKikimr; using namespace NKikimr::NMiniKQL; using namespace NYson::NDetail; -void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& value, TType* type, - const TVector<ui32>* structPositions) { - // Result format +void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& value, TType* type, + const TVector<ui32>* structPositions) { + // Result format switch (type->GetKind()) { case TType::EKind::Void: writer.OnVoid(); @@ -52,83 +52,83 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& switch (dataType->GetSchemeType()) { case NUdf::TDataType<bool>::Id: writer.OnBooleanScalar(value.Get<bool>()); - return; + return; case NUdf::TDataType<i32>::Id: writer.OnInt64Scalar(value.Get<i32>()); - return; + return; case NUdf::TDataType<ui32>::Id: writer.OnUint64Scalar(value.Get<ui32>()); - return; + return; case NUdf::TDataType<i64>::Id: writer.OnInt64Scalar(value.Get<i64>()); - return; + return; case NUdf::TDataType<ui64>::Id: writer.OnUint64Scalar(value.Get<ui64>()); - return; + return; case NUdf::TDataType<ui8>::Id: writer.OnUint64Scalar(value.Get<ui8>()); - return; + return; case NUdf::TDataType<i8>::Id: writer.OnInt64Scalar(value.Get<i8>()); - return; + return; case NUdf::TDataType<ui16>::Id: writer.OnUint64Scalar(value.Get<ui16>()); - return; + return; case NUdf::TDataType<i16>::Id: writer.OnInt64Scalar(value.Get<i16>()); - return; + return; case NUdf::TDataType<float>::Id: - writer.OnFloatScalar(value.Get<float>()); - return; + writer.OnFloatScalar(value.Get<float>()); + return; case NUdf::TDataType<double>::Id: writer.OnDoubleScalar(value.Get<double>()); - return; - - case NUdf::TDataType<NUdf::TJson>::Id: + return; + + case NUdf::TDataType<NUdf::TJson>::Id: case NUdf::TDataType<NUdf::TUtf8>::Id: - // assume underlying string is utf8 - writer.OnUtf8StringScalar(value.AsStringRef()); - return; - + // assume underlying string is utf8 + writer.OnUtf8StringScalar(value.AsStringRef()); + return; + case NUdf::TDataType<char*>::Id: case NUdf::TDataType<NUdf::TUuid>::Id: case NUdf::TDataType<NUdf::TDyNumber>::Id: writer.OnStringScalar(value.AsStringRef()); - return; - + return; + case NUdf::TDataType<NUdf::TDecimal>::Id: { const auto params = static_cast<TDataDecimalType*>(type)->GetParams(); const auto str = NDecimal::ToString(value.GetInt128(), params.first, params.second); const auto size = str ? std::strlen(str) : 0; - writer.OnUtf8StringScalar(TStringBuf(str, size)); - return; + writer.OnUtf8StringScalar(TStringBuf(str, size)); + return; } case NUdf::TDataType<NUdf::TYson>::Id: - EncodeRestrictedYson(writer, value.AsStringRef()); - return; + EncodeRestrictedYson(writer, value.AsStringRef()); + return; case NUdf::TDataType<NUdf::TDate>::Id: writer.OnUint64Scalar(value.Get<ui16>()); - return; + return; case NUdf::TDataType<NUdf::TDatetime>::Id: writer.OnUint64Scalar(value.Get<ui32>()); - return; + return; case NUdf::TDataType<NUdf::TTimestamp>::Id: writer.OnUint64Scalar(value.Get<ui64>()); - return; + return; case NUdf::TDataType<NUdf::TInterval>::Id: writer.OnInt64Scalar(value.Get<i64>()); - return; + return; case NUdf::TDataType<NUdf::TTzDate>::Id: case NUdf::TDataType<NUdf::TTzDatetime>::Id: case NUdf::TDataType<NUdf::TTzTimestamp>::Id: case NUdf::TDataType<NUdf::TJsonDocument>::Id: { const NUdf::TUnboxedValue out(ValueToString(*dataType->GetDataSlot(), value)); writer.OnUtf8StringScalar(out.AsStringRef()); - return; + return; } default: - throw yexception() << "Unknown data type: " << dataType->GetSchemeType(); + throw yexception() << "Unknown data type: " << dataType->GetSchemeType(); } } break; @@ -143,12 +143,12 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& const ui32 pos = structPositions ? (*structPositions)[i] : i; if (pos < e) { writer.OnListItem(); - WriteYsonValueImpl(writer, value.GetElement(pos), structType->GetMemberType(pos), nullptr); + WriteYsonValueImpl(writer, value.GetElement(pos), structType->GetMemberType(pos), nullptr); } } writer.OnEndList(); - return; + return; } case TType::EKind::List: { @@ -157,24 +157,24 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& const auto it = value.GetListIterator(); for (NUdf::TUnboxedValue item; it.Next(item);) { writer.OnListItem(); - WriteYsonValueImpl(writer, item, listType->GetItemType(), nullptr); + WriteYsonValueImpl(writer, item, listType->GetItemType(), nullptr); } writer.OnEndList(); - return; + return; } case TType::EKind::Optional: { if (!value) { - writer.OnEntity(); + writer.OnEntity(); } else { writer.OnBeginList(); auto optionalType = AS_TYPE(TOptionalType, type); writer.OnListItem(); - WriteYsonValueImpl(writer, value.GetOptionalValue(), optionalType->GetItemType(), nullptr); + WriteYsonValueImpl(writer, value.GetOptionalValue(), optionalType->GetItemType(), nullptr); writer.OnEndList(); } - return; + return; } case TType::EKind::Dict: { @@ -184,29 +184,29 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) { writer.OnListItem(); writer.OnBeginList(); - { + { writer.OnListItem(); - WriteYsonValueImpl(writer, key, dictType->GetKeyType(), nullptr); + WriteYsonValueImpl(writer, key, dictType->GetKeyType(), nullptr); writer.OnListItem(); - WriteYsonValueImpl(writer, payload, dictType->GetPayloadType(), nullptr); - } + WriteYsonValueImpl(writer, payload, dictType->GetPayloadType(), nullptr); + } writer.OnEndList(); } writer.OnEndList(); } - return; + return; case TType::EKind::Tuple: { writer.OnBeginList(); auto tupleType = AS_TYPE(TTupleType, type); for (ui32 i = 0, e = tupleType->GetElementsCount(); i < e; ++i) { writer.OnListItem(); - WriteYsonValueImpl(writer, value.GetElement(i), tupleType->GetElementType(i), nullptr); + WriteYsonValueImpl(writer, value.GetElement(i), tupleType->GetElementType(i), nullptr); } writer.OnEndList(); - return; + return; } case TType::EKind::Variant: { @@ -217,13 +217,13 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& writer.OnUint64Scalar(index); writer.OnListItem(); if (underlyingType->IsTuple()) { - WriteYsonValueImpl(writer, value.GetVariantItem(), AS_TYPE(TTupleType, underlyingType)->GetElementType(index), nullptr); + WriteYsonValueImpl(writer, value.GetVariantItem(), AS_TYPE(TTupleType, underlyingType)->GetElementType(index), nullptr); } else { - WriteYsonValueImpl(writer, value.GetVariantItem(), AS_TYPE(TStructType, underlyingType)->GetMemberType(index), nullptr); + WriteYsonValueImpl(writer, value.GetVariantItem(), AS_TYPE(TStructType, underlyingType)->GetMemberType(index), nullptr); } writer.OnEndList(); - return; + return; } case TType::EKind::Tagged: @@ -234,22 +234,22 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& } default: - YQL_ENSURE(false, "unknown type " << type->GetKindAsStr()); + YQL_ENSURE(false, "unknown type " << type->GetKindAsStr()); } } void WriteYsonValue(NYson::TYsonConsumerBase& writer, const NUdf::TUnboxedValuePod& value, TType* type, - const TVector<ui32>* structPositions) -{ - TYsonResultWriter resultWriter(writer); - WriteYsonValueImpl(resultWriter, value, type, structPositions); -} - + const TVector<ui32>* structPositions) +{ + TYsonResultWriter resultWriter(writer); + WriteYsonValueImpl(resultWriter, value, type, structPositions); +} + TString WriteYsonValue(const NUdf::TUnboxedValuePod& value, TType* type, const TVector<ui32>* structPositions, NYson::EYsonFormat format) { TStringStream str; NYson::TYsonWriter writer(&str, format); - WriteYsonValue(writer, value, type, structPositions); + WriteYsonValue(writer, value, type, structPositions); return str.Str(); } @@ -407,31 +407,31 @@ TExprNode::TPtr DataNodeToExprLiteral(TPositionHandle pos, const TTypeAnnotation TString DataValueToString(const NKikimr::NUdf::TUnboxedValuePod& value, const TDataExprType* type) { switch (type->GetSlot()) { case NUdf::EDataSlot::Int32: - return ToString(value.Get<i32>()); + return ToString(value.Get<i32>()); case NUdf::EDataSlot::Int64: case NUdf::EDataSlot::Interval: - return ToString(value.Get<i64>()); + return ToString(value.Get<i64>()); case NUdf::EDataSlot::Uint32: case NUdf::EDataSlot::Datetime: - return ToString(value.Get<ui32>()); + return ToString(value.Get<ui32>()); case NUdf::EDataSlot::Uint64: case NUdf::EDataSlot::Timestamp: - return ToString(value.Get<ui64>()); + return ToString(value.Get<ui64>()); case NUdf::EDataSlot::Float: - return ::FloatToString(value.Get<float>()); + return ::FloatToString(value.Get<float>()); case NUdf::EDataSlot::Double: - return ::FloatToString(value.Get<double>()); + return ::FloatToString(value.Get<double>()); case NUdf::EDataSlot::Bool: - return ToString(value.Get<bool>()); + return ToString(value.Get<bool>()); case NUdf::EDataSlot::Uint8: - return ToString(static_cast<unsigned int>(value.Get<ui8>())); + return ToString(static_cast<unsigned int>(value.Get<ui8>())); case NUdf::EDataSlot::Int8: - return ToString(static_cast<int>(value.Get<i8>())); + return ToString(static_cast<int>(value.Get<i8>())); case NUdf::EDataSlot::Uint16: case NUdf::EDataSlot::Date: - return ToString(static_cast<unsigned int>(value.Get<ui16>())); + return ToString(static_cast<unsigned int>(value.Get<ui16>())); case NUdf::EDataSlot::Int16: - return ToString(static_cast<int>(value.Get<i16>())); + return ToString(static_cast<int>(value.Get<i16>())); case NUdf::EDataSlot::String: case NUdf::EDataSlot::Utf8: case NUdf::EDataSlot::Json: @@ -440,31 +440,31 @@ TString DataValueToString(const NKikimr::NUdf::TUnboxedValuePod& value, const TD case NUdf::EDataSlot::DyNumber: return ToString((TStringBuf)value.AsStringRef()); case NUdf::EDataSlot::Decimal: - { + { const auto params = dynamic_cast<const TDataExprParamsType*>(type); return NDecimal::ToString(value.GetInt128(), FromString<ui8>(params->GetParamOne()), FromString<ui8>(params->GetParamTwo())); } case NUdf::EDataSlot::TzDate: { - TStringStream out; - out << value.Get<ui16>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); - return out.Str(); - } + TStringStream out; + out << value.Get<ui16>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); + return out.Str(); + } case NUdf::EDataSlot::TzDatetime: { - TStringStream out; - out << value.Get<ui32>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); - return out.Str(); - } + TStringStream out; + out << value.Get<ui32>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); + return out.Str(); + } case NUdf::EDataSlot::TzTimestamp: { - TStringStream out; - out << value.Get<ui64>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); - return out.Str(); - } + TStringStream out; + out << value.Get<ui64>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId()); + return out.Str(); + } case NUdf::EDataSlot::JsonDocument: { NUdf::TUnboxedValue json = ValueToString(EDataSlot::JsonDocument, value); return ToString(TStringBuf(value.AsStringRef())); } } - + Y_FAIL("Unexpected"); } } //namespace @@ -501,7 +501,7 @@ NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NM } } return result; -} +} TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) { TExprNode::TPtr result; @@ -664,46 +664,46 @@ void SkipYson(char cmd, TInputBuf& buf) { CopyYsonWithAttrs(cmd, buf, yson); } -NUdf::TUnboxedValue ReadYsonStringInResultFormat(char cmd, TInputBuf& buf) { - NUdf::TUnboxedValue result; - const bool needDecode = (cmd == BeginListSymbol); - - if (needDecode) { - cmd = buf.Read(); - } - - CHECK_EXPECTED(cmd, StringMarker); - const i32 length = buf.ReadVarI32(); - CHECK_STRING_LENGTH(length); - TTempBuf tmpBuf(length); - buf.ReadMany(tmpBuf.Data(), length); - - if (needDecode) { - TString decoded = Base64Decode(TStringBuf(tmpBuf.Data(), length)); - result = NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(decoded))); - } else { - result = NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(tmpBuf.Data(), length))); - } - - if (needDecode) { - cmd = buf.Read(); - if (cmd == ListItemSeparatorSymbol) { - cmd = buf.Read(); - } - - CHECK_EXPECTED(cmd, EndListSymbol); - } - return result; -} - -TStringBuf ReadNextString(char cmd, TInputBuf& buf) { - CHECK_EXPECTED(cmd, StringMarker); - return buf.ReadYtString(); -} - -template <typename T> -T ReadNextSerializedNumber(char cmd, TInputBuf& buf) { - auto nextString = ReadNextString(cmd, buf); +NUdf::TUnboxedValue ReadYsonStringInResultFormat(char cmd, TInputBuf& buf) { + NUdf::TUnboxedValue result; + const bool needDecode = (cmd == BeginListSymbol); + + if (needDecode) { + cmd = buf.Read(); + } + + CHECK_EXPECTED(cmd, StringMarker); + const i32 length = buf.ReadVarI32(); + CHECK_STRING_LENGTH(length); + TTempBuf tmpBuf(length); + buf.ReadMany(tmpBuf.Data(), length); + + if (needDecode) { + TString decoded = Base64Decode(TStringBuf(tmpBuf.Data(), length)); + result = NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(decoded))); + } else { + result = NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(tmpBuf.Data(), length))); + } + + if (needDecode) { + cmd = buf.Read(); + if (cmd == ListItemSeparatorSymbol) { + cmd = buf.Read(); + } + + CHECK_EXPECTED(cmd, EndListSymbol); + } + return result; +} + +TStringBuf ReadNextString(char cmd, TInputBuf& buf) { + CHECK_EXPECTED(cmd, StringMarker); + return buf.ReadYtString(); +} + +template <typename T> +T ReadNextSerializedNumber(char cmd, TInputBuf& buf) { + auto nextString = ReadNextString(cmd, buf); if constexpr (!std::numeric_limits<T>::is_integer) { if (nextString == "inf" || nextString == "+inf") { return std::numeric_limits<T>::infinity(); @@ -713,23 +713,23 @@ T ReadNextSerializedNumber(char cmd, TInputBuf& buf) { return std::numeric_limits<T>::quiet_NaN(); } } - return FromString<T>(nextString); -} - -template <typename T> -T ReadYsonFloatNumber(char cmd, TInputBuf& buf, bool isTableFormat) { - if (isTableFormat) { - CHECK_EXPECTED(cmd, DoubleMarker); - double dbl; - buf.ReadMany((char*)&dbl, sizeof(dbl)); - return dbl; - } - - return ReadNextSerializedNumber<T>(cmd, buf); -} - + return FromString<T>(nextString); +} + +template <typename T> +T ReadYsonFloatNumber(char cmd, TInputBuf& buf, bool isTableFormat) { + if (isTableFormat) { + CHECK_EXPECTED(cmd, DoubleMarker); + double dbl; + buf.ReadMany((char*)&dbl, sizeof(dbl)); + return dbl; + } + + return ReadNextSerializedNumber<T>(cmd, buf); +} + NUdf::TUnboxedValue ReadYsonValue(TType* type, - const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf, bool isTableFormat) { + const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf, bool isTableFormat) { switch (type->GetKind()) { case TType::EKind::Variant: { auto varType = static_cast<TVariantType*>(type); @@ -744,15 +744,15 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, } CHECK_EXPECTED(cmd, BeginListSymbol); - cmd = buf.Read(); - ui64 index = 0; - if (isTableFormat) { + cmd = buf.Read(); + ui64 index = 0; + if (isTableFormat) { CHECK_EXPECTED(cmd, Uint64Marker); index = buf.ReadVarUI64(); - } else { - index = ReadNextSerializedNumber<ui64>(cmd, buf); - } - + } else { + index = ReadNextSerializedNumber<ui64>(cmd, buf); + } + YQL_ENSURE(index < varType->GetAlternativesCount(), "Bad variant alternative: " << index << ", only " << varType->GetAlternativesCount() << " are available"); YQL_ENSURE(underlyingType->IsTuple() || underlyingType->IsStruct(), "Wrong underlying type"); @@ -766,7 +766,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, EXPECTED(buf, ListItemSeparatorSymbol); cmd = buf.Read(); - auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); + auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { cmd = buf.Read(); @@ -779,87 +779,87 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, case TType::EKind::Data: { auto schemeType = static_cast<TDataType*>(type)->GetSchemeType(); switch (schemeType) { - case NUdf::TDataType<bool>::Id: - YQL_ENSURE(cmd == FalseMarker || cmd == TrueMarker, "Expected either true or false, but got: " << TString(cmd).Quote()); - return NUdf::TUnboxedValuePod(cmd == TrueMarker); + case NUdf::TDataType<bool>::Id: + YQL_ENSURE(cmd == FalseMarker || cmd == TrueMarker, "Expected either true or false, but got: " << TString(cmd).Quote()); + return NUdf::TUnboxedValuePod(cmd == TrueMarker); case NUdf::TDataType<ui8>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod(ui8(buf.ReadVarUI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui8>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod(ui8(buf.ReadVarUI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui8>(cmd, buf)); case NUdf::TDataType<i8>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Int64Marker); - return NUdf::TUnboxedValuePod(i8(buf.ReadVarI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i8>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Int64Marker); + return NUdf::TUnboxedValuePod(i8(buf.ReadVarI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i8>(cmd, buf)); case NUdf::TDataType<ui16>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod(ui16(buf.ReadVarUI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui16>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod(ui16(buf.ReadVarUI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui16>(cmd, buf)); case NUdf::TDataType<i16>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Int64Marker); - return NUdf::TUnboxedValuePod(i16(buf.ReadVarI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i16>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Int64Marker); + return NUdf::TUnboxedValuePod(i16(buf.ReadVarI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i16>(cmd, buf)); case NUdf::TDataType<i32>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Int64Marker); - return NUdf::TUnboxedValuePod(i32(buf.ReadVarI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i32>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Int64Marker); + return NUdf::TUnboxedValuePod(i32(buf.ReadVarI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i32>(cmd, buf)); case NUdf::TDataType<ui32>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod(ui32(buf.ReadVarUI64())); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui32>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod(ui32(buf.ReadVarUI64())); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui32>(cmd, buf)); case NUdf::TDataType<i64>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Int64Marker); - return NUdf::TUnboxedValuePod(buf.ReadVarI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i64>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Int64Marker); + return NUdf::TUnboxedValuePod(buf.ReadVarI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i64>(cmd, buf)); case NUdf::TDataType<ui64>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod(buf.ReadVarUI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui64>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod(buf.ReadVarUI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui64>(cmd, buf)); - case NUdf::TDataType<float>::Id: - return NUdf::TUnboxedValuePod(ReadYsonFloatNumber<float>(cmd, buf, isTableFormat)); + case NUdf::TDataType<float>::Id: + return NUdf::TUnboxedValuePod(ReadYsonFloatNumber<float>(cmd, buf, isTableFormat)); - case NUdf::TDataType<double>::Id: - return NUdf::TUnboxedValuePod(ReadYsonFloatNumber<double>(cmd, buf, isTableFormat)); + case NUdf::TDataType<double>::Id: + return NUdf::TUnboxedValuePod(ReadYsonFloatNumber<double>(cmd, buf, isTableFormat)); case NUdf::TDataType<NUdf::TUtf8>::Id: case NUdf::TDataType<char*>::Id: case NUdf::TDataType<NUdf::TJson>::Id: case NUdf::TDataType<NUdf::TDyNumber>::Id: case NUdf::TDataType<NUdf::TUuid>::Id: { - if (isTableFormat) { - auto nextString = ReadNextString(cmd, buf); - return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(nextString))); - } - - return ReadYsonStringInResultFormat(cmd, buf); + if (isTableFormat) { + auto nextString = ReadNextString(cmd, buf); + return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(nextString))); + } + + return ReadYsonStringInResultFormat(cmd, buf); } case NUdf::TDataType<NUdf::TDecimal>::Id: { - auto nextString = ReadNextString(cmd, buf); + auto nextString = ReadNextString(cmd, buf); if (isTableFormat) { const auto& des = NDecimal::Deserialize(nextString.data()); YQL_ENSURE(nextString.size() == des.second); @@ -877,91 +877,91 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, auto& yson = buf.YsonBuffer(); yson.clear(); CopyYsonWithAttrs(cmd, buf, yson); - - if (isTableFormat) { - return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(yson))); - } - + + if (isTableFormat) { + return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(yson))); + } + TString decodedYson = DecodeRestrictedYson(TStringBuf(yson.data(), yson.size()), NYson::EYsonFormat::Text); - return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(decodedYson))); + return NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(decodedYson))); } case NUdf::TDataType<NUdf::TDate>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod((ui16)buf.ReadVarUI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui16>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod((ui16)buf.ReadVarUI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui16>(cmd, buf)); case NUdf::TDataType<NUdf::TDatetime>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod((ui32)buf.ReadVarUI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui32>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod((ui32)buf.ReadVarUI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui32>(cmd, buf)); case NUdf::TDataType<NUdf::TTimestamp>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Uint64Marker); - return NUdf::TUnboxedValuePod(buf.ReadVarUI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui64>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Uint64Marker); + return NUdf::TUnboxedValuePod(buf.ReadVarUI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<ui64>(cmd, buf)); case NUdf::TDataType<NUdf::TInterval>::Id: - if (isTableFormat) { - CHECK_EXPECTED(cmd, Int64Marker); - return NUdf::TUnboxedValuePod(buf.ReadVarI64()); - } - return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i64>(cmd, buf)); + if (isTableFormat) { + CHECK_EXPECTED(cmd, Int64Marker); + return NUdf::TUnboxedValuePod(buf.ReadVarI64()); + } + return NUdf::TUnboxedValuePod(ReadNextSerializedNumber<i64>(cmd, buf)); case NUdf::TDataType<NUdf::TTzDate>::Id: { - auto nextString = ReadNextString(cmd, buf); - NUdf::TUnboxedValuePod data; - if (isTableFormat) { - ui16 value; + auto nextString = ReadNextString(cmd, buf); + NUdf::TUnboxedValuePod data; + if (isTableFormat) { + ui16 value; ui16 tzId = 0; - YQL_ENSURE(DeserializeTzDate(nextString, value, tzId)); - data = NUdf::TUnboxedValuePod(value); + YQL_ENSURE(DeserializeTzDate(nextString, value, tzId)); + data = NUdf::TUnboxedValuePod(value); data.SetTimezoneId(tzId); - } else { + } else { data = ValueFromString(NUdf::EDataSlot::TzDate, nextString); YQL_ENSURE(data, "incorrect tz date format for value " << nextString); - } - + } + return data; } case NUdf::TDataType<NUdf::TTzDatetime>::Id: { - auto nextString = ReadNextString(cmd, buf); - NUdf::TUnboxedValuePod data; - if (isTableFormat) { - ui32 value; + auto nextString = ReadNextString(cmd, buf); + NUdf::TUnboxedValuePod data; + if (isTableFormat) { + ui32 value; ui16 tzId = 0; - YQL_ENSURE(DeserializeTzDatetime(nextString, value, tzId)); - data = NUdf::TUnboxedValuePod(value); + YQL_ENSURE(DeserializeTzDatetime(nextString, value, tzId)); + data = NUdf::TUnboxedValuePod(value); data.SetTimezoneId(tzId); - } else { + } else { data = ValueFromString(NUdf::EDataSlot::TzDatetime, nextString); YQL_ENSURE(data, "incorrect tz datetime format for value " << nextString); - } - + } + return data; } case NUdf::TDataType<NUdf::TTzTimestamp>::Id: { - auto nextString = ReadNextString(cmd, buf); - NUdf::TUnboxedValuePod data; - if (isTableFormat) { - ui64 value; + auto nextString = ReadNextString(cmd, buf); + NUdf::TUnboxedValuePod data; + if (isTableFormat) { + ui64 value; ui16 tzId = 0; - YQL_ENSURE(DeserializeTzTimestamp(nextString, value, tzId)); - data = NUdf::TUnboxedValuePod(value); + YQL_ENSURE(DeserializeTzTimestamp(nextString, value, tzId)); + data = NUdf::TUnboxedValuePod(value); data.SetTimezoneId(tzId); - } else { + } else { data = ValueFromString(NUdf::EDataSlot::TzTimestamp, nextString); YQL_ENSURE(data, "incorrect tz timestamp format for value " << nextString); - } - + } + return data; } @@ -988,7 +988,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, cmd = buf.Read(); for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { - items[i] = ReadYsonValue(structType->GetMemberType(i), holderFactory, cmd, buf, isTableFormat); + items[i] = ReadYsonValue(structType->GetMemberType(i), holderFactory, cmd, buf, isTableFormat); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { @@ -1007,7 +1007,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, break; } - auto keyBuffer = ReadNextString(cmd, buf); + auto keyBuffer = ReadNextString(cmd, buf); auto pos = structType->FindMemberIndex(keyBuffer); EXPECTED(buf, KeyValueSeparatorSymbol); cmd = buf.Read(); @@ -1052,7 +1052,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, break; } - items = items.Append(ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat)); + items = items.Append(ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat)); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { @@ -1070,7 +1070,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, auto itemType = static_cast<TOptionalType*>(type)->GetItemType(); if (cmd != BeginListSymbol) { - auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); + auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); return value.Release().MakeOptional(); } @@ -1079,7 +1079,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, return NUdf::TUnboxedValuePod(); } - auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); + auto value = ReadYsonValue(itemType, holderFactory, cmd, buf, isTableFormat); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { cmd = buf.Read(); @@ -1105,8 +1105,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, YQL_ENSURE(cmd == BeginListSymbol || cmd == BeginMapSymbol, "Expected '{' or '[', but read: " << TString(cmd).Quote()); if (cmd == BeginMapSymbol) { - bool unusedIsOptional; - auto unpackedType = UnpackOptional(keyType, unusedIsOptional); + bool unusedIsOptional; + auto unpackedType = UnpackOptional(keyType, unusedIsOptional); YQL_ENSURE(unpackedType->IsData() && static_cast<TDataType*>(unpackedType)->GetSchemeType() == NUdf::TDataType<char*>::Id, "Expected String type as dictionary key type"); @@ -1119,11 +1119,11 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, break; } - auto keyBuffer = ReadNextString(cmd, buf); + auto keyBuffer = ReadNextString(cmd, buf); auto keyStr = NUdf::TUnboxedValue(MakeString(keyBuffer)); EXPECTED(buf, KeyValueSeparatorSymbol); cmd = buf.Read(); - auto payload = ReadYsonValue(payloadType, holderFactory, cmd, buf, isTableFormat); + auto payload = ReadYsonValue(payloadType, holderFactory, cmd, buf, isTableFormat); map.emplace(std::move(keyStr), std::move(payload)); cmd = buf.Read(); @@ -1146,10 +1146,10 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, CHECK_EXPECTED(cmd, BeginListSymbol); cmd = buf.Read(); - auto key = ReadYsonValue(keyType, holderFactory, cmd, buf, isTableFormat); + auto key = ReadYsonValue(keyType, holderFactory, cmd, buf, isTableFormat); EXPECTED(buf, ListItemSeparatorSymbol); cmd = buf.Read(); - auto payload = ReadYsonValue(payloadType, holderFactory, cmd, buf, isTableFormat); + auto payload = ReadYsonValue(payloadType, holderFactory, cmd, buf, isTableFormat); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { cmd = buf.Read(); @@ -1181,7 +1181,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, cmd = buf.Read(); for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { - items[i] = ReadYsonValue(tupleType->GetElementType(i), holderFactory, cmd, buf, isTableFormat); + items[i] = ReadYsonValue(tupleType->GetElementType(i), holderFactory, cmd, buf, isTableFormat); cmd = buf.Read(); if (cmd == ListItemSeparatorSymbol) { @@ -1199,8 +1199,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, return NUdf::TUnboxedValuePod::Void(); } - auto nextString = ReadNextString(cmd, buf); - YQL_ENSURE(nextString == TYsonResultWriter::VoidString, "Expected Void"); + auto nextString = ReadNextString(cmd, buf); + YQL_ENSURE(nextString == TYsonResultWriter::VoidString, "Expected Void"); return NUdf::TUnboxedValuePod::Void(); } @@ -1235,7 +1235,7 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type, } TMaybe<NUdf::TUnboxedValue> ParseYsonValue(const THolderFactory& holderFactory, - const TStringBuf& yson, TType* type, IOutputStream* err, bool isTableFormat) { + const TStringBuf& yson, TType* type, IOutputStream* err, bool isTableFormat) { try { class TReader : public IBlockReader { public: @@ -1274,7 +1274,7 @@ TMaybe<NUdf::TUnboxedValue> ParseYsonValue(const THolderFactory& holderFactory, TReader reader(yson); TInputBuf buf(reader, nullptr); char cmd = buf.Read(); - return ReadYsonValue(type, holderFactory, cmd, buf, isTableFormat); + return ReadYsonValue(type, holderFactory, cmd, buf, isTableFormat); } catch (const yexception& e) { if (err) { @@ -1289,11 +1289,11 @@ TMaybe<NUdf::TUnboxedValue> ParseYsonNode(const THolderFactory& holderFactory, return ParseYsonValue(holderFactory, NYT::NodeToYsonString(node, NYson::EYsonFormat::Binary), type, err, true); } -TMaybe<NUdf::TUnboxedValue> ParseYsonNodeInResultFormat(const THolderFactory& holderFactory, - const NYT::TNode& node, TType* type, IOutputStream* err) { +TMaybe<NUdf::TUnboxedValue> ParseYsonNodeInResultFormat(const THolderFactory& holderFactory, + const NYT::TNode& node, TType* type, IOutputStream* err) { return ParseYsonValue(holderFactory, NYT::NodeToYsonString(node, NYson::EYsonFormat::Binary), type, err, false); -} - +} + extern "C" void ReadYsonContainerValue(TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value, TInputBuf& buf, bool wrapOptional) { // yson content @@ -1303,7 +1303,7 @@ extern "C" void ReadYsonContainerValue(TType* type, const NKikimr::NMiniKQL::THo // parse binary yson... YQL_ENSURE(size > 0); char cmd = buf.Read(); - auto tmp = ReadYsonValue(type, holderFactory, cmd, buf, true); + auto tmp = ReadYsonValue(type, holderFactory, cmd, buf, true); if (!wrapOptional) { value = std::move(tmp); } @@ -1687,8 +1687,8 @@ private: TPageHeader Dummy_; }; -void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnboxedValuePod& value, bool topLevel) { - // Table format, very compact +void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnboxedValuePod& value, bool topLevel) { + // Table format, very compact switch (type->GetKind()) { case TType::EKind::Variant: { buf.Write(BeginListSymbol); @@ -1709,7 +1709,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo itemType = static_cast<TStructType*>(underlyingType)->GetMemberType(index); } - WriteYsonValueInTableFormat(buf, itemType, value.GetVariantItem(), false); + WriteYsonValueInTableFormat(buf, itemType, value.GetVariantItem(), false); buf.Write(ListItemSeparatorSymbol); buf.Write(EndListSymbol); break; @@ -1877,7 +1877,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo auto structType = static_cast<TStructType*>(type); buf.Write(BeginListSymbol); for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { - WriteYsonValueInTableFormat(buf, structType->GetMemberType(i), value.GetElement(i), false); + WriteYsonValueInTableFormat(buf, structType->GetMemberType(i), value.GetElement(i), false); buf.Write(ListItemSeparatorSymbol); } @@ -1890,7 +1890,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo const auto iter = value.GetListIterator(); buf.Write(BeginListSymbol); for (NUdf::TUnboxedValue item; iter.Next(item); buf.Write(ListItemSeparatorSymbol)) { - WriteYsonValueInTableFormat(buf, itemType, item, false); + WriteYsonValueInTableFormat(buf, itemType, item, false); } buf.Write(EndListSymbol); @@ -1910,7 +1910,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo } else { buf.Write(BeginListSymbol); - WriteYsonValueInTableFormat(buf, itemType, value.GetOptionalValue(), false); + WriteYsonValueInTableFormat(buf, itemType, value.GetOptionalValue(), false); buf.Write(ListItemSeparatorSymbol); buf.Write(EndListSymbol); } @@ -1924,9 +1924,9 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo buf.Write(BeginListSymbol); for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) { buf.Write(BeginListSymbol); - WriteYsonValueInTableFormat(buf, dictType->GetKeyType(), key, false); + WriteYsonValueInTableFormat(buf, dictType->GetKeyType(), key, false); buf.Write(ListItemSeparatorSymbol); - WriteYsonValueInTableFormat(buf, dictType->GetPayloadType(), payload, false); + WriteYsonValueInTableFormat(buf, dictType->GetPayloadType(), payload, false); buf.Write(ListItemSeparatorSymbol); buf.Write(EndListSymbol); buf.Write(ListItemSeparatorSymbol); @@ -1940,7 +1940,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo auto tupleType = static_cast<TTupleType*>(type); buf.Write(BeginListSymbol); for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { - WriteYsonValueInTableFormat(buf, tupleType->GetElementType(i), value.GetElement(i), false); + WriteYsonValueInTableFormat(buf, tupleType->GetElementType(i), value.GetElement(i), false); buf.Write(ListItemSeparatorSymbol); } @@ -1978,7 +1978,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, const NUdf::TUnbo extern "C" void WriteYsonContainerValue(TType* type, const NUdf::TUnboxedValuePod& value, TOutputBuf& buf) { TTempBlockWriter blockWriter; TOutputBuf ysonBuf(blockWriter, nullptr); - WriteYsonValueInTableFormat(ysonBuf, type, value, true); + WriteYsonValueInTableFormat(ysonBuf, type, value, true); ysonBuf.Flush(); ui32 size = ysonBuf.GetWrittenBytes(); buf.WriteMany((const char*)&size, sizeof(size)); @@ -2284,11 +2284,11 @@ TExprNode::TPtr ValueToExprLiteral(const TTypeAnnotationNode* type, const NKikim auto dataType = type->Cast<TDataExprType>(); TVector<TExprNode::TPtr> args({ ctx.NewAtom(pos, DataValueToString(value, dataType)) }); if (auto params = dynamic_cast<const TDataExprParamsType*>(dataType)) { - args.reserve(3); + args.reserve(3); args.push_back(ctx.NewAtom(pos, params->GetParamOne())); args.push_back(ctx.NewAtom(pos, params->GetParamTwo())); - } - + } + return ctx.NewCallable(pos, dataType->GetName(), std::move(args)); } diff --git a/ydb/library/yql/providers/common/codec/yql_codec.h b/ydb/library/yql/providers/common/codec/yql_codec.h index 512e01224c..c6e93ac9fc 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.h +++ b/ydb/library/yql/providers/common/codec/yql_codec.h @@ -28,7 +28,7 @@ void WriteYsonValue( NYson::TYsonConsumerBase& writer, const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type, - const TVector<ui32>* structPositions = nullptr + const TVector<ui32>* structPositions = nullptr ); TString WriteYsonValue( @@ -62,16 +62,16 @@ struct TCodecContext { void SkipYson(char cmd, TInputBuf& buf); void CopyYson(char cmd, TInputBuf& buf, TVector<char>& yson); void CopyYsonWithAttrs(char cmd, TInputBuf& buf, TVector<char>& yson); -NKikimr::NUdf::TUnboxedValue ReadYsonValue(NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf, bool isTableFormat); +NKikimr::NUdf::TUnboxedValue ReadYsonValue(NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf, bool isTableFormat); TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonValue(const NKikimr::NMiniKQL::THolderFactory& holderFactory, - const TStringBuf& yson, NKikimr::NMiniKQL::TType* type, IOutputStream* err, bool isTableFormat); + const TStringBuf& yson, NKikimr::NMiniKQL::TType* type, IOutputStream* err, bool isTableFormat); TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonNode(const NKikimr::NMiniKQL::THolderFactory& holderFactory, const NYT::TNode& node, NKikimr::NMiniKQL::TType* type, IOutputStream* err); -TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonNodeInResultFormat(const NKikimr::NMiniKQL::THolderFactory& holderFactory, - const NYT::TNode& node, NKikimr::NMiniKQL::TType* type, IOutputStream* err); - +TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonNodeInResultFormat(const NKikimr::NMiniKQL::THolderFactory& holderFactory, + const NYT::TNode& node, NKikimr::NMiniKQL::TType* type, IOutputStream* err); + extern "C" void ReadYsonContainerValue(NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value, NCommon::TInputBuf& buf, bool wrapOptional); @@ -95,7 +95,7 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl extern "C" void WriteContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf); -void WriteYsonValueInTableFormat(TOutputBuf& buf, NKikimr::NMiniKQL::TType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel); +void WriteYsonValueInTableFormat(TOutputBuf& buf, NKikimr::NMiniKQL::TType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel); TExprNode::TPtr ValueToExprLiteral(const TTypeAnnotationNode* type, const NKikimr::NUdf::TUnboxedValuePod& value, TExprContext& ctx, TPositionHandle pos = {}); diff --git a/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp b/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp index ddaaeab6b7..10a611a43e 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp @@ -3,8 +3,8 @@ namespace NYql { namespace NCommon { -NKikimr::NMiniKQL::TStatKey InputBytes("Job_InputBytes", true); -NKikimr::NMiniKQL::TStatKey OutputBytes("Job_OutputBytes", true); +NKikimr::NMiniKQL::TStatKey InputBytes("Job_InputBytes", true); +NKikimr::NMiniKQL::TStatKey OutputBytes("Job_OutputBytes", true); ui32 TInputBuf::CopyVarUI32(TVector<char>& yson) { char cmd = Read(); diff --git a/ydb/library/yql/providers/common/codec/yql_codec_results.cpp b/ydb/library/yql/providers/common/codec/yql_codec_results.cpp index 96c7d784ca..ec37818dea 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec_results.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec_results.cpp @@ -1,25 +1,25 @@ -#include "yql_codec_results.h" - +#include "yql_codec_results.h" + #include <library/cpp/string_utils/base64/base64.h> - + #include <ydb/library/yql/utils/utf8.h> - -namespace NYql { -namespace NCommon { - -constexpr TStringBuf TYsonResultWriter::VoidString; - -void TYsonResultWriter::OnStringScalar(TStringBuf value) { + +namespace NYql { +namespace NCommon { + +constexpr TStringBuf TYsonResultWriter::VoidString; + +void TYsonResultWriter::OnStringScalar(TStringBuf value) { if (!IsUtf8(value)) { - TString encoded = Base64Encode(value); - Writer.OnBeginList(); + TString encoded = Base64Encode(value); + Writer.OnBeginList(); Writer.OnListItem(); - Writer.OnStringScalar(TStringBuf(encoded)); - Writer.OnEndList(); - } else { - Writer.OnStringScalar(value); - } -} - -} -} + Writer.OnStringScalar(TStringBuf(encoded)); + Writer.OnEndList(); + } else { + Writer.OnStringScalar(value); + } +} + +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_codec_results.h b/ydb/library/yql/providers/common/codec/yql_codec_results.h index e34d2a3a68..e2469474b9 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec_results.h +++ b/ydb/library/yql/providers/common/codec/yql_codec_results.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include <library/cpp/yson/writer.h> -#include <util/generic/strbuf.h> -#include <util/string/cast.h> - -namespace NYql { -namespace NCommon { - -// we should not write numbers as numbers ever -// write numbers as strings except Yson value where we use restricted dialect +#include <util/generic/strbuf.h> +#include <util/string/cast.h> + +namespace NYql { +namespace NCommon { + +// we should not write numbers as numbers ever +// write numbers as strings except Yson value where we use restricted dialect // write bool as bool -// write null as entity -class TYsonResultWriter { -public: +// write null as entity +class TYsonResultWriter { +public: static constexpr TStringBuf VoidString = "Void"; - -public: + +public: explicit TYsonResultWriter(NYson::TYsonConsumerBase& writer) : Writer(writer) { @@ -37,7 +37,7 @@ public: void OnEntity() { Writer.OnEntity(); } - // numbers + // numbers void OnInt64Scalar(i64 value) { WriteNumberAsString(value); } @@ -53,12 +53,12 @@ public: void OnBooleanScalar(bool value) { Writer.OnBooleanScalar(value); } - // strings - void OnStringScalar(TStringBuf value); + // strings + void OnStringScalar(TStringBuf value); void OnUtf8StringScalar(TStringBuf value) { Writer.OnStringScalar(value); } - // list construction + // list construction void OnBeginList() { Writer.OnBeginList(); } @@ -83,15 +83,15 @@ public: void OnEndAttributes() { Writer.OnEndAttributes(); } - -private: - template <typename T> - void WriteNumberAsString(T value) { - Writer.OnStringScalar(::ToString(value)); - } - -private: + +private: + template <typename T> + void WriteNumberAsString(T value) { + Writer.OnStringScalar(::ToString(value)); + } + +private: NYson::TYsonConsumerBase& Writer; -}; -} -} +}; +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_restricted_yson.cpp b/ydb/library/yql/providers/common/codec/yql_restricted_yson.cpp index 08b4521748..348aa0520f 100644 --- a/ydb/library/yql/providers/common/codec/yql_restricted_yson.cpp +++ b/ydb/library/yql/providers/common/codec/yql_restricted_yson.cpp @@ -1,326 +1,326 @@ -#include "yql_restricted_yson.h" - +#include "yql_restricted_yson.h" + #include <ydb/library/yql/utils/parse_double.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <library/cpp/yson/detail.h> #include <library/cpp/yson/parser.h> - + #include <library/cpp/yson/node/node_io.h> #include <library/cpp/yson/node/node_visitor.h> - -#include <util/generic/algorithm.h> -#include <util/generic/stack.h> - -namespace NYql { -namespace NCommon { - -namespace { + +#include <util/generic/algorithm.h> +#include <util/generic/stack.h> + +namespace NYql { +namespace NCommon { + +namespace { class TRestrictedYsonFormatter : public NYson::TYsonConsumerBase { -public: - TRestrictedYsonFormatter(TYsonResultWriter& writer) - : Writer(writer) { - } - +public: + TRestrictedYsonFormatter(TYsonResultWriter& writer) + : Writer(writer) { + } + void OnStringScalar(TStringBuf value) override { - Open(); + Open(); Type(TStringBuf("string")); - - Buffer.clear(); - bool isAscii = true; + + Buffer.clear(); + bool isAscii = true; for (size_t i = 0; i < value.size(); ++i) { - if (ui8(value[i]) < 128) { - if (!isAscii) { - Buffer.push_back(value[i]); - } - } else { - if (isAscii) { - Buffer.resize(i); + if (ui8(value[i]) < 128) { + if (!isAscii) { + Buffer.push_back(value[i]); + } + } else { + if (isAscii) { + Buffer.resize(i); Copy(value.data(), value.data() + i, Buffer.data()); - isAscii = false; - } - Buffer.push_back('\xC0' | (ui8(value[i]) >> 6)); - Buffer.push_back('\x80' | (ui8(value[i]) & ~'\xC0')); - } - } - - if (isAscii) { - Value(value); - } else { - Value(TStringBuf(Buffer.data(), Buffer.size())); - } - - Close(); - } - - void OnInt64Scalar(i64 value) override { - Open(); + isAscii = false; + } + Buffer.push_back('\xC0' | (ui8(value[i]) >> 6)); + Buffer.push_back('\x80' | (ui8(value[i]) & ~'\xC0')); + } + } + + if (isAscii) { + Value(value); + } else { + Value(TStringBuf(Buffer.data(), Buffer.size())); + } + + Close(); + } + + void OnInt64Scalar(i64 value) override { + Open(); Type(TStringBuf("int64")); - Value(ToString(value)); - Close(); - } - - void OnUint64Scalar(ui64 value) override { - Open(); + Value(ToString(value)); + Close(); + } + + void OnUint64Scalar(ui64 value) override { + Open(); Type(TStringBuf("uint64")); - Value(ToString(value)); - Close(); - } - - void OnDoubleScalar(double value) override { - Open(); + Value(ToString(value)); + Close(); + } + + void OnDoubleScalar(double value) override { + Open(); Type(TStringBuf("double")); Value(::FloatToString(value)); - Close(); - } - - void OnBooleanScalar(bool value) override { - Open(); + Close(); + } + + void OnBooleanScalar(bool value) override { + Open(); Type(TStringBuf("boolean")); Value(value ? TStringBuf("true") : TStringBuf("false")); - Close(); - } - - void OnEntity() override { - if (AfterAttributes) { + Close(); + } + + void OnEntity() override { + if (AfterAttributes) { Writer.OnKeyedItem(TStringBuf("$value")); - Writer.OnEntity(); - Writer.OnEndMap(); - AfterAttributes = false; - } else { - Writer.OnEntity(); - } - } - - void OnBeginList() override { - if (AfterAttributes) { + Writer.OnEntity(); + Writer.OnEndMap(); + AfterAttributes = false; + } else { + Writer.OnEntity(); + } + } + + void OnBeginList() override { + if (AfterAttributes) { Writer.OnKeyedItem(TStringBuf("$value")); - } - - Writer.OnBeginList(); - HasAttributes.push(AfterAttributes); - AfterAttributes = false; - } - - void OnListItem() override { - Writer.OnListItem(); - } - - void OnEndList() override { - Writer.OnEndList(); - if (HasAttributes.top()) { - Writer.OnEndMap(); - } - - HasAttributes.pop(); - } - - void OnBeginMap() override { - if (AfterAttributes) { + } + + Writer.OnBeginList(); + HasAttributes.push(AfterAttributes); + AfterAttributes = false; + } + + void OnListItem() override { + Writer.OnListItem(); + } + + void OnEndList() override { + Writer.OnEndList(); + if (HasAttributes.top()) { + Writer.OnEndMap(); + } + + HasAttributes.pop(); + } + + void OnBeginMap() override { + if (AfterAttributes) { Writer.OnKeyedItem(TStringBuf("$value")); - } - - Writer.OnBeginMap(); - HasAttributes.push(AfterAttributes); - AfterAttributes = false; - } - + } + + Writer.OnBeginMap(); + HasAttributes.push(AfterAttributes); + AfterAttributes = false; + } + void OnKeyedItem(TStringBuf key) override { - if (key.StartsWith('$')) { - Writer.OnKeyedItem(TString("$") + key); - } else { - Writer.OnKeyedItem(key); - } - } - - void OnEndMap() override { - Writer.OnEndMap(); - if (HasAttributes.top()) { - Writer.OnEndMap(); - } - - HasAttributes.pop(); - } - - void OnBeginAttributes() override { - Writer.OnBeginMap(); + if (key.StartsWith('$')) { + Writer.OnKeyedItem(TString("$") + key); + } else { + Writer.OnKeyedItem(key); + } + } + + void OnEndMap() override { + Writer.OnEndMap(); + if (HasAttributes.top()) { + Writer.OnEndMap(); + } + + HasAttributes.pop(); + } + + void OnBeginAttributes() override { + Writer.OnBeginMap(); Writer.OnKeyedItem(TStringBuf("$attributes")); - Writer.OnBeginMap(); - } - - void OnEndAttributes() override { - Writer.OnEndMap(); - AfterAttributes = true; - } - - void Open() { - if (!AfterAttributes) { - Writer.OnBeginMap(); - } - } - - void Close() { - Writer.OnEndMap(); - AfterAttributes = false; - } - - void Type(const TStringBuf& type) { + Writer.OnBeginMap(); + } + + void OnEndAttributes() override { + Writer.OnEndMap(); + AfterAttributes = true; + } + + void Open() { + if (!AfterAttributes) { + Writer.OnBeginMap(); + } + } + + void Close() { + Writer.OnEndMap(); + AfterAttributes = false; + } + + void Type(const TStringBuf& type) { Writer.OnKeyedItem(TStringBuf("$type")); - Writer.OnUtf8StringScalar(type); - } - - void Value(const TStringBuf& value) { + Writer.OnUtf8StringScalar(type); + } + + void Value(const TStringBuf& value) { Writer.OnKeyedItem(TStringBuf("$value")); - Writer.OnUtf8StringScalar(value); - } - -private: - TYsonResultWriter& Writer; - TStack<bool> HasAttributes; - bool AfterAttributes = false; - TVector<char> Buffer; -}; - -TString DecodeRestrictedBinaryString(const TString& data) { - TString res; + Writer.OnUtf8StringScalar(value); + } + +private: + TYsonResultWriter& Writer; + TStack<bool> HasAttributes; + bool AfterAttributes = false; + TVector<char> Buffer; +}; + +TString DecodeRestrictedBinaryString(const TString& data) { + TString res; for (size_t i = 0; i < data.size(); ++i) { - char c = data[i]; - if (((unsigned char)c) >= 128) { + char c = data[i]; + if (((unsigned char)c) >= 128) { YQL_ENSURE(i + 1 < data.size()); - res.push_back(((c & 0x03) << 6) | (data[i + 1] & 0x3f)); - ++i; - } else { - res.push_back(c); - } - } - - return res; -} - + res.push_back(((c & 0x03) << 6) | (data[i + 1] & 0x3f)); + ++i; + } else { + res.push_back(c); + } + } + + return res; +} + void DecodeRestrictedYson(const NYT::TNode& node, NYson::TYsonConsumerBase& writer) { - switch (node.GetType()) { - case NYT::TNode::String: - writer.OnStringScalar(node.AsString()); - return; - - case NYT::TNode::Int64: - writer.OnInt64Scalar(node.AsInt64()); - return; - - case NYT::TNode::Uint64: - writer.OnUint64Scalar(node.AsUint64()); - return; - - case NYT::TNode::Bool: - writer.OnBooleanScalar(node.AsBool()); - return; - - case NYT::TNode::Double: - writer.OnDoubleScalar(node.AsDouble()); - return; - - case NYT::TNode::Null: - writer.OnEntity(); - return; - - case NYT::TNode::List: - // just a list without attributes - writer.OnBeginList(); - for (const auto& item : node.AsList()) { - writer.OnListItem(); - DecodeRestrictedYson(item, writer); - } - - writer.OnEndList(); - return; - - case NYT::TNode::Map: - // process below - break; - - default: - YQL_ENSURE(false, "Unsupported node type: " << static_cast<int>(node.GetType())); - } - - YQL_ENSURE(node.IsMap()); - if (!node.HasKey("$value")) { - // just a map without attributes - writer.OnBeginMap(); - for (const auto& x : node.AsMap()) { - if (x.first.StartsWith("$$")) { - writer.OnKeyedItem(x.first.substr(1)); - } else { - writer.OnKeyedItem(x.first); - } - - DecodeRestrictedYson(x.second, writer); - } - - writer.OnEndMap(); - return; - } - - if (node.HasKey("$attributes")) { - writer.OnBeginAttributes(); - for (const auto& x : node["$attributes"].AsMap()) { - if (x.first.StartsWith("$$")) { - writer.OnKeyedItem(x.first.substr(1)); - } else { - writer.OnKeyedItem(x.first); - } - - DecodeRestrictedYson(x.second, writer); - } - - writer.OnEndAttributes(); - } - - if (!node.HasKey("$type")) { - // non-scalars with attributes - DecodeRestrictedYson(node["$value"], writer); - return; - } - - auto type = node["$type"].AsString(); - if (type == "int64") { - writer.OnInt64Scalar(FromString<i64>(node["$value"].AsString())); - } else if (type == "uint64") { - writer.OnUint64Scalar(FromString<ui64>(node["$value"].AsString())); - } else if (type == "double") { - writer.OnDoubleScalar(DoubleFromString(TStringBuf(node["$value"].AsString()))); - } else if (type == "boolean") { - writer.OnBooleanScalar(FromString<bool>(node["$value"].AsString())); - } else if (type == "string") { - writer.OnStringScalar(DecodeRestrictedBinaryString(node["$value"].AsString())); - } else { - YQL_ENSURE(false, "Unsupported type: " << type); - } -} - -} // anonymous namespace - -void EncodeRestrictedYson(TYsonResultWriter& writer, const TStringBuf& yson) { - TRestrictedYsonFormatter formatter(writer); + switch (node.GetType()) { + case NYT::TNode::String: + writer.OnStringScalar(node.AsString()); + return; + + case NYT::TNode::Int64: + writer.OnInt64Scalar(node.AsInt64()); + return; + + case NYT::TNode::Uint64: + writer.OnUint64Scalar(node.AsUint64()); + return; + + case NYT::TNode::Bool: + writer.OnBooleanScalar(node.AsBool()); + return; + + case NYT::TNode::Double: + writer.OnDoubleScalar(node.AsDouble()); + return; + + case NYT::TNode::Null: + writer.OnEntity(); + return; + + case NYT::TNode::List: + // just a list without attributes + writer.OnBeginList(); + for (const auto& item : node.AsList()) { + writer.OnListItem(); + DecodeRestrictedYson(item, writer); + } + + writer.OnEndList(); + return; + + case NYT::TNode::Map: + // process below + break; + + default: + YQL_ENSURE(false, "Unsupported node type: " << static_cast<int>(node.GetType())); + } + + YQL_ENSURE(node.IsMap()); + if (!node.HasKey("$value")) { + // just a map without attributes + writer.OnBeginMap(); + for (const auto& x : node.AsMap()) { + if (x.first.StartsWith("$$")) { + writer.OnKeyedItem(x.first.substr(1)); + } else { + writer.OnKeyedItem(x.first); + } + + DecodeRestrictedYson(x.second, writer); + } + + writer.OnEndMap(); + return; + } + + if (node.HasKey("$attributes")) { + writer.OnBeginAttributes(); + for (const auto& x : node["$attributes"].AsMap()) { + if (x.first.StartsWith("$$")) { + writer.OnKeyedItem(x.first.substr(1)); + } else { + writer.OnKeyedItem(x.first); + } + + DecodeRestrictedYson(x.second, writer); + } + + writer.OnEndAttributes(); + } + + if (!node.HasKey("$type")) { + // non-scalars with attributes + DecodeRestrictedYson(node["$value"], writer); + return; + } + + auto type = node["$type"].AsString(); + if (type == "int64") { + writer.OnInt64Scalar(FromString<i64>(node["$value"].AsString())); + } else if (type == "uint64") { + writer.OnUint64Scalar(FromString<ui64>(node["$value"].AsString())); + } else if (type == "double") { + writer.OnDoubleScalar(DoubleFromString(TStringBuf(node["$value"].AsString()))); + } else if (type == "boolean") { + writer.OnBooleanScalar(FromString<bool>(node["$value"].AsString())); + } else if (type == "string") { + writer.OnStringScalar(DecodeRestrictedBinaryString(node["$value"].AsString())); + } else { + YQL_ENSURE(false, "Unsupported type: " << type); + } +} + +} // anonymous namespace + +void EncodeRestrictedYson(TYsonResultWriter& writer, const TStringBuf& yson) { + TRestrictedYsonFormatter formatter(writer); NYson::ParseYsonStringBuffer(yson, &formatter); -} - +} + TString EncodeRestrictedYson(const NYT::TNode& node, NYson::EYsonFormat format) { - TStringStream stream; + TStringStream stream; NYson::TYsonWriter writer(&stream, format); - TYsonResultWriter resultWriter(writer); - TRestrictedYsonFormatter formatter(resultWriter); - NYT::TNodeVisitor visitor(&formatter); - visitor.Visit(node); - return stream.Str(); -} - + TYsonResultWriter resultWriter(writer); + TRestrictedYsonFormatter formatter(resultWriter); + NYT::TNodeVisitor visitor(&formatter); + visitor.Visit(node); + return stream.Str(); +} + TString DecodeRestrictedYson(const TStringBuf& yson, NYson::EYsonFormat format) { - TStringStream stream; + TStringStream stream; NYson::TYsonWriter writer(&stream, format); - auto node = NYT::NodeFromYsonString(yson); - DecodeRestrictedYson(node, writer); - return stream.Str(); -} - -} -} + auto node = NYT::NodeFromYsonString(yson); + DecodeRestrictedYson(node, writer); + return stream.Str(); +} + +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_restricted_yson.h b/ydb/library/yql/providers/common/codec/yql_restricted_yson.h index 0d0f7568d6..d20cce63a2 100644 --- a/ydb/library/yql/providers/common/codec/yql_restricted_yson.h +++ b/ydb/library/yql/providers/common/codec/yql_restricted_yson.h @@ -1,28 +1,28 @@ -#pragma once - -#include "yql_codec_results.h" - -#include <util/generic/strbuf.h> +#pragma once + +#include "yql_codec_results.h" + +#include <util/generic/strbuf.h> #include <library/cpp/yson/public.h> - + namespace NYT { class TNode; } -namespace NYql { -namespace NCommon { - -void EncodeRestrictedYson( - TYsonResultWriter& writer, - const TStringBuf& yson); - -TString EncodeRestrictedYson( - const NYT::TNode& node, +namespace NYql { +namespace NCommon { + +void EncodeRestrictedYson( + TYsonResultWriter& writer, + const TStringBuf& yson); + +TString EncodeRestrictedYson( + const NYT::TNode& node, NYson::EYsonFormat format = NYson::EYsonFormat::Binary); - -TString DecodeRestrictedYson( - const TStringBuf& yson, + +TString DecodeRestrictedYson( + const TStringBuf& yson, NYson::EYsonFormat format = NYson::EYsonFormat::Binary); - -} -} + +} +} diff --git a/ydb/library/yql/providers/common/codec/yql_restricted_yson_ut.cpp b/ydb/library/yql/providers/common/codec/yql_restricted_yson_ut.cpp index aee6715a93..a8714758ca 100644 --- a/ydb/library/yql/providers/common/codec/yql_restricted_yson_ut.cpp +++ b/ydb/library/yql/providers/common/codec/yql_restricted_yson_ut.cpp @@ -1,95 +1,95 @@ -#include "yql_restricted_yson.h" - +#include "yql_restricted_yson.h" + #include <library/cpp/yson/node/node_io.h> #include <library/cpp/yson/node/node_visitor.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NYql { - -namespace { -TString FormatNode(const NYT::TNode& node) { - TStringStream stream; + +namespace NYql { + +namespace { +TString FormatNode(const NYT::TNode& node) { + TStringStream stream; NYson::TYsonWriter writer(&stream, NYson::EYsonFormat::Text); - const auto sortMapKeys = true; - NYT::TNodeVisitor visitor(&writer, sortMapKeys); - visitor.Visit(node); - return stream.Str(); -} - -// reformat yson and sort keys -TString Normalize(const TString& yson) { - return FormatNode(NYT::NodeFromYsonString(yson)); -} - -} - -Y_UNIT_TEST_SUITE(TRestrictedYson) { - void RunTest(const NYT::TNode& node, const TString& expectedNodeStr, const TString& expectedEncodedStr) { - UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), expectedNodeStr); + const auto sortMapKeys = true; + NYT::TNodeVisitor visitor(&writer, sortMapKeys); + visitor.Visit(node); + return stream.Str(); +} + +// reformat yson and sort keys +TString Normalize(const TString& yson) { + return FormatNode(NYT::NodeFromYsonString(yson)); +} + +} + +Y_UNIT_TEST_SUITE(TRestrictedYson) { + void RunTest(const NYT::TNode& node, const TString& expectedNodeStr, const TString& expectedEncodedStr) { + UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), expectedNodeStr); TString encoded = NCommon::EncodeRestrictedYson(node, NYson::EYsonFormat::Text); - UNIT_ASSERT_VALUES_EQUAL(Normalize(encoded), expectedEncodedStr); + UNIT_ASSERT_VALUES_EQUAL(Normalize(encoded), expectedEncodedStr); TString decoded = NCommon::DecodeRestrictedYson(TStringBuf(encoded), NYson::EYsonFormat::Text); - UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), Normalize(decoded)); - } - - Y_UNIT_TEST(MapScalars) { - NYT::TNode node = NYT::TNode::CreateMap(); - node["a"] = NYT::TNode("abc"); - node["b"] = NYT::TNode::CreateEntity(); - node["c"] = NYT::TNode(true); - node["d"] = NYT::TNode(1); - node["e"] = NYT::TNode(1u); - node["f"] = NYT::TNode(1.25); - - RunTest(node, - R"({"a"="abc";"b"=#;"c"=%true;"d"=1;"e"=1u;"f"=1.25})", - R"({"a"={"$type"="string";"$value"="abc"};"b"=#;"c"={"$type"="boolean";"$value"="true"};"d"={"$type"="int64";"$value"="1"};"e"={"$type"="uint64";"$value"="1"};"f"={"$type"="double";"$value"="1.25"}})" - ); - } - - Y_UNIT_TEST(ScalarWithAttributes) { - NYT::TNode node("abc"); - node.Attributes()["d"] = NYT::TNode(true); - - RunTest(node, - R"(<"d"=%true>"abc")", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$type"="string";"$value"="abc"})" - ); - } - - Y_UNIT_TEST(MapWithAttributes) { - NYT::TNode node = NYT::TNode::CreateMap(); - node["b"] = NYT::TNode::CreateEntity(); - node["c"] = NYT::TNode(false); - node.Attributes()["d"] = NYT::TNode(true); - - RunTest(node, - R"(<"d"=%true>{"b"=#;"c"=%false})", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"={"b"=#;"c"={"$type"="boolean";"$value"="false"}}})" - ); - } - - Y_UNIT_TEST(ListWithAttributes) { - NYT::TNode node = NYT::TNode::CreateList(); - node.Add(NYT::TNode::CreateEntity()); - node.Add(NYT::TNode(false)); - node.Attributes()["d"] = NYT::TNode(true); - - RunTest(node, - R"(<"d"=%true>[#;%false])", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=[#;{"$type"="boolean";"$value"="false"}]})" - ); - } - - Y_UNIT_TEST(EntityWithAttributes) { - NYT::TNode node = NYT::TNode::CreateEntity(); - node.Attributes()["d"] = NYT::TNode(true); - - RunTest(node, - R"(<"d"=%true>#)", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=#})" - ); - } -} -} + UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), Normalize(decoded)); + } + + Y_UNIT_TEST(MapScalars) { + NYT::TNode node = NYT::TNode::CreateMap(); + node["a"] = NYT::TNode("abc"); + node["b"] = NYT::TNode::CreateEntity(); + node["c"] = NYT::TNode(true); + node["d"] = NYT::TNode(1); + node["e"] = NYT::TNode(1u); + node["f"] = NYT::TNode(1.25); + + RunTest(node, + R"({"a"="abc";"b"=#;"c"=%true;"d"=1;"e"=1u;"f"=1.25})", + R"({"a"={"$type"="string";"$value"="abc"};"b"=#;"c"={"$type"="boolean";"$value"="true"};"d"={"$type"="int64";"$value"="1"};"e"={"$type"="uint64";"$value"="1"};"f"={"$type"="double";"$value"="1.25"}})" + ); + } + + Y_UNIT_TEST(ScalarWithAttributes) { + NYT::TNode node("abc"); + node.Attributes()["d"] = NYT::TNode(true); + + RunTest(node, + R"(<"d"=%true>"abc")", + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$type"="string";"$value"="abc"})" + ); + } + + Y_UNIT_TEST(MapWithAttributes) { + NYT::TNode node = NYT::TNode::CreateMap(); + node["b"] = NYT::TNode::CreateEntity(); + node["c"] = NYT::TNode(false); + node.Attributes()["d"] = NYT::TNode(true); + + RunTest(node, + R"(<"d"=%true>{"b"=#;"c"=%false})", + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"={"b"=#;"c"={"$type"="boolean";"$value"="false"}}})" + ); + } + + Y_UNIT_TEST(ListWithAttributes) { + NYT::TNode node = NYT::TNode::CreateList(); + node.Add(NYT::TNode::CreateEntity()); + node.Add(NYT::TNode(false)); + node.Attributes()["d"] = NYT::TNode(true); + + RunTest(node, + R"(<"d"=%true>[#;%false])", + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=[#;{"$type"="boolean";"$value"="false"}]})" + ); + } + + Y_UNIT_TEST(EntityWithAttributes) { + NYT::TNode node = NYT::TNode::CreateEntity(); + node.Attributes()["d"] = NYT::TNode(true); + + RunTest(node, + R"(<"d"=%true>#)", + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=#})" + ); + } +} +} diff --git a/ydb/library/yql/providers/common/config/yql_dispatch.cpp b/ydb/library/yql/providers/common/config/yql_dispatch.cpp index 4e6df45a8f..f6701862bb 100644 --- a/ydb/library/yql/providers/common/config/yql_dispatch.cpp +++ b/ydb/library/yql/providers/common/config/yql_dispatch.cpp @@ -129,11 +129,11 @@ bool TSettingDispatcher::Dispatch(const TString& cluster, const TString& name, c handler->Handle(cluster, value, validateOnly); return !validateOnly; } else { - // ignore unknown names in config - if (stage == EStage::CONFIG) { - return false; - } - + // ignore unknown names in config + if (stage == EStage::CONFIG) { + return false; + } + TStringBuilder nearHandlerMsg; for (auto& item: Handlers) { if (NLevenshtein::Distance(normalizedName, item.first) < DefaultMistypeDistance) { diff --git a/ydb/library/yql/providers/common/http_gateway/ya.make b/ydb/library/yql/providers/common/http_gateway/ya.make index f53489973e..d4c9926dd8 100644 --- a/ydb/library/yql/providers/common/http_gateway/ya.make +++ b/ydb/library/yql/providers/common/http_gateway/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER( - g:yq + g:yq g:yql ) diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp index 1fb7bdcbba..d568188a81 100644 --- a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp +++ b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp @@ -106,29 +106,29 @@ public: { } - void SetCounter( - const TString& labelName, - const TString& labelValue, - i64 value, - bool derivative) override - { - if (UserName_) { - // per user counter - auto userCnt = GetCounter(labelName, labelValue, UserName_.Get(), - derivative); - if (userCnt) { - *userCnt = value; - } - - return; - } - - auto totalCnt = GetCounter(labelName, labelValue, nullptr, derivative); - if (totalCnt) { - *totalCnt = value; - } - } - + void SetCounter( + const TString& labelName, + const TString& labelValue, + i64 value, + bool derivative) override + { + if (UserName_) { + // per user counter + auto userCnt = GetCounter(labelName, labelValue, UserName_.Get(), + derivative); + if (userCnt) { + *userCnt = value; + } + + return; + } + + auto totalCnt = GetCounter(labelName, labelValue, nullptr, derivative); + if (totalCnt) { + *totalCnt = value; + } + } + void IncCounter( const TString& labelName, const TString& labelValue, diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.h b/ydb/library/yql/providers/common/metrics/metrics_registry.h index d344ba9dd9..70d1c4bc12 100644 --- a/ydb/library/yql/providers/common/metrics/metrics_registry.h +++ b/ydb/library/yql/providers/common/metrics/metrics_registry.h @@ -21,12 +21,12 @@ using TMetricsDecorator = std::function<IMetricsRegistryPtr( ////////////////////////////////////////////////////////////////////////////// struct IMetricsRegistry: public TThrRefBase { - virtual void SetCounter( - const TString& labelName, - const TString& labelValue, - i64 value, - bool derivative = false) = 0; - + virtual void SetCounter( + const TString& labelName, + const TString& labelValue, + i64 value, + bool derivative = false) = 0; + virtual void IncCounter( const TString& labelName, const TString& labelValue, diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 8561fe2536..7e86cc8bd7 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -38,11 +38,11 @@ enum EYtLogLevel { YL_DEBUG = 3; }; -enum EHostScheme { - HS_HTTP = 0; - HS_HTTPS = 1; -}; - +enum EHostScheme { + HS_HTTP = 0; + HS_HTTPS = 1; +}; + enum ETokenType { IAM = 0; OAUTH = 1; @@ -114,9 +114,9 @@ message TKikimrClusterConfig { optional string Database = 5; optional uint32 TvmId = 6 [default = 0]; optional string Token = 7; - optional string ProxyUrl = 8; - optional string Location = 9; - optional bool IsFromMvp = 10; + optional string ProxyUrl = 8; + optional string Location = 9; + optional bool IsFromMvp = 10; optional ETokenType TokenType = 11; repeated TAttr Settings = 101; } @@ -150,9 +150,9 @@ message TYdbClusterConfig { optional string Database = 6; optional string Id = 7; optional bool Secure = 8; - optional string ServiceAccountId = 9; - optional string ServiceAccountIdSignature = 10; - optional bool AddBearerToToken = 11; // whether to use prefix "Bearer " in token + optional string ServiceAccountId = 9; + optional string ServiceAccountIdSignature = 10; + optional bool AddBearerToToken = 11; // whether to use prefix "Bearer " in token repeated TAttr Settings = 100; } @@ -171,11 +171,11 @@ message TClickHouseClusterConfig { optional string Cluster = 3; optional string CHToken = 4; repeated TAttr Settings = 101; - optional EHostScheme HostScheme = 5; - optional uint32 HostPort = 6; - optional string CHTokenYavSecretId = 7; - optional string CHTokenYavVersionId = 8; - optional string CHTokenYavKey = 9; + optional EHostScheme HostScheme = 5; + optional uint32 HostPort = 6; + optional string CHTokenYavSecretId = 7; + optional string CHTokenYavVersionId = 8; + optional string CHTokenYavKey = 9; optional NYql.NProto.TClickHouseRunnerConfig Runner = 10; optional uint32 NativeHostPort = 11 [default = 9000]; optional bool NativeSecure = 12 [default = false]; @@ -188,36 +188,36 @@ message TClickHouseGatewayConfig { } ///////////////////////////// RTMR ///////////////////////////// -enum EYfArtifactType { - AT_NONE = 0; - AT_FILE = 1; - AT_LAYER = 2; -}; - -enum ERtmrOperationType { - OT_NONE = 0; - OT_LF_PARSE = 1; - OT_YDB_OUT = 2; - OT_SOLOMON_OUT = 3; - OT_PQ_OUT = 4; -}; - -message TYfArtifactLink { - optional EYfArtifactType Type = 1; - optional string Url = 2; // in any format suitable for file storage - optional string TargetFilePath = 3; // ignored for Type=AT_LAYER -}; - -message TRtmrOperationArtifactsInfo { - optional ERtmrOperationType Type = 1; - repeated TYfArtifactLink Links = 2; -}; - -message TRtmrPqConsumerInfo { - optional string Cluster = 1; - optional string Consumer = 2; -} - +enum EYfArtifactType { + AT_NONE = 0; + AT_FILE = 1; + AT_LAYER = 2; +}; + +enum ERtmrOperationType { + OT_NONE = 0; + OT_LF_PARSE = 1; + OT_YDB_OUT = 2; + OT_SOLOMON_OUT = 3; + OT_PQ_OUT = 4; +}; + +message TYfArtifactLink { + optional EYfArtifactType Type = 1; + optional string Url = 2; // in any format suitable for file storage + optional string TargetFilePath = 3; // ignored for Type=AT_LAYER +}; + +message TRtmrOperationArtifactsInfo { + optional ERtmrOperationType Type = 1; + repeated TYfArtifactLink Links = 2; +}; + +message TRtmrPqConsumerInfo { + optional string Cluster = 1; + optional string Consumer = 2; +} + message TRtmrClusterConfig { optional string Name = 1; optional bool Default = 2 [default = false]; @@ -228,14 +228,14 @@ message TRtmrClusterConfig { optional string S3FileCachePath = 7; optional uint32 TvmId = 8 [default = 0]; optional string TvmSecretPath = 9; - repeated string Users = 10; - optional bool UploadViaYfClient = 11 [default = false]; - optional string MdsTorrentUrl = 12; - repeated TRtmrOperationArtifactsInfo ArtifactOverrides = 13; // override CommonArtifacts by ERtmrOperationType - optional string PqConsumer = 14; - repeated TRtmrPqConsumerInfo PqConsumerMapping = 15; - optional uint32 MaxPqPartitions = 16; - optional uint32 PreviewCollectTimeoutMs = 17; + repeated string Users = 10; + optional bool UploadViaYfClient = 11 [default = false]; + optional string MdsTorrentUrl = 12; + repeated TRtmrOperationArtifactsInfo ArtifactOverrides = 13; // override CommonArtifacts by ERtmrOperationType + optional string PqConsumer = 14; + repeated TRtmrPqConsumerInfo PqConsumerMapping = 15; + optional uint32 MaxPqPartitions = 16; + optional uint32 PreviewCollectTimeoutMs = 17; repeated TAttr Settings = 101; } @@ -243,33 +243,33 @@ message TRtmrGatewayConfig { repeated TRtmrClusterConfig ClusterMapping = 1; repeated TAttr DefaultSettings = 2; optional string YqlRtmrDynLib = 3; // path to libyql-dynlib.so - optional bool UseFakeYfUpload = 4 [default = false]; // use in tests only to speedup upload + optional bool UseFakeYfUpload = 4 [default = false]; // use in tests only to speedup upload repeated string Artifacts = 5; /// default artifacts to put into every function - repeated TRtmrOperationArtifactsInfo CommonArtifacts = 6; // common artifacts for all clusters - optional uint32 MaxPqPartitions = 7 [default = 10]; - optional uint32 PreviewCollectTimeoutMs = 8 [default = 2000]; + repeated TRtmrOperationArtifactsInfo CommonArtifacts = 6; // common artifacts for all clusters + optional uint32 MaxPqPartitions = 7 [default = 10]; + optional uint32 PreviewCollectTimeoutMs = 8 [default = 2000]; } ///////////////////////////// Pq ////////////////////////////// message TPqClusterConfig { enum EClusterType { - CT_UNSPECIFIED = 0; - CT_PERS_QUEUE = 1; - CT_DATA_STREAMS = 2; + CT_UNSPECIFIED = 0; + CT_PERS_QUEUE = 1; + CT_DATA_STREAMS = 2; } optional string Name = 1; - optional EClusterType ClusterType = 2 [default = CT_PERS_QUEUE]; + optional EClusterType ClusterType = 2 [default = CT_PERS_QUEUE]; optional string Endpoint = 3; optional string ConfigManagerEndpoint = 4; optional string Token = 5; optional string Database = 6 [default = "/Root"]; optional uint32 TvmId = 7 [default = 0]; optional bool UseSsl = 8; // grpcs - optional string ServiceAccountId = 9; - optional string ServiceAccountIdSignature = 10; - optional bool AddBearerToToken = 11; // whether to use prefix "Bearer " in token + optional string ServiceAccountId = 9; + optional string ServiceAccountIdSignature = 10; + optional bool AddBearerToToken = 11; // whether to use prefix "Bearer " in token optional string DatabaseId = 12; repeated TAttr Settings = 100; } @@ -316,8 +316,8 @@ message TS3ClusterConfig { optional string Name = 1; // Short cluster name optional string Url = 2; optional string Token = 3; - optional string ServiceAccountId = 4; - optional string ServiceAccountIdSignature = 5; + optional string ServiceAccountId = 4; + optional string ServiceAccountIdSignature = 5; repeated TAttr Settings = 100; } @@ -347,11 +347,11 @@ message TSolomonClusterConfig { optional string Name = 1; optional string Cluster = 2; - optional bool UseSsl = 3; + optional bool UseSsl = 3; optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON]; - optional string Token = 5; - optional string ServiceAccountId = 6; - optional string ServiceAccountIdSignature = 7; + optional string Token = 5; + optional string ServiceAccountId = 6; + optional string ServiceAccountIdSignature = 7; optional TShardPath Path = 8; repeated TAttr Settings = 100; @@ -362,10 +362,10 @@ message TSolomonGatewayConfig { repeated TAttr DefaultSettings = 2; } -message TFileStorageAdditionalConfig { - repeated string AllowedUrlPatterns = 1; -} - +message TFileStorageAdditionalConfig { + repeated string AllowedUrlPatterns = 1; +} + /////////////////////////////// Postgresql ///////////////////////////// message TPostgresqlClusterConfig { @@ -444,7 +444,7 @@ message TGatewaysConfig { optional TStatGatewayConfig Stat = 6; optional TChytGatewayConfig Chyt = 7; optional TSolomonGatewayConfig Solomon = 8; - optional TFileStorageAdditionalConfig Fs = 9; + optional TFileStorageAdditionalConfig Fs = 9; optional TYqlCoreConfig YqlCore = 10; optional TPostgresqlGatewayConfig Postgresql = 11; optional TSqlCoreConfig SqlCore = 12; diff --git a/ydb/library/yql/providers/common/proto/udf_resolver.proto b/ydb/library/yql/providers/common/proto/udf_resolver.proto index 170d13e6d7..e949e714e4 100644 --- a/ydb/library/yql/providers/common/proto/udf_resolver.proto +++ b/ydb/library/yql/providers/common/proto/udf_resolver.proto @@ -1,5 +1,5 @@ package NYql; -option java_package = "ru.yandex.yql.proto"; +option java_package = "ru.yandex.yql.proto"; message TFunction { required string Name = 1; @@ -7,26 +7,26 @@ message TFunction { optional string UserType = 3; }; -message TArgResult { - required string Name = 1; - required string Type = 2; - required string Doc = 3; - required uint32 Flags = 4; -}; - +message TArgResult { + required string Name = 1; + required string Type = 2; + required string Doc = 3; + required uint32 Flags = 4; +}; + message TFunctionResult { optional string NormalizedUserType = 1; optional string RunConfigType = 2; optional string CallableType = 3; optional string Error = 4; - optional string Name = 5; - optional string Doc = 7; - optional bool IsTypeAwareness = 8; - optional uint32 ArgCount = 9; - optional uint32 OptionalArgCount = 10; - repeated TArgResult Args = 11; - optional string ReturnType = 12; - optional string ReturnDoc = 13; + optional string Name = 5; + optional string Doc = 7; + optional bool IsTypeAwareness = 8; + optional uint32 ArgCount = 9; + optional uint32 OptionalArgCount = 10; + repeated TArgResult Args = 11; + optional string ReturnType = 12; + optional string ReturnDoc = 13; }; message TImport { @@ -39,8 +39,8 @@ message TImportResult { required string FileAlias = 1; repeated string Modules = 2; optional string Error = 3; - optional uint64 FileSize = 4; - optional string Md5 = 5; + optional uint64 FileSize = 4; + optional string Md5 = 5; }; message TResolve { diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index aa782ee93f..8a2d83cca4 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -432,26 +432,26 @@ bool FillUsedFilesImpl( if (node.IsCallable("FilePath") || node.IsCallable("FileContent")) { const auto& name = node.Head().Content(); - const auto block = types.UserDataStorage->FindUserDataBlock(name); + const auto block = types.UserDataStorage->FindUserDataBlock(name); if (!block) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << name)); return false; } else { - auto iter = files.insert({ TUserDataStorage::ComposeUserDataKey(name), *block }).first; + auto iter = files.insert({ TUserDataStorage::ComposeUserDataKey(name), *block }).first; iter->second.Usage.Set(node.IsCallable("FilePath") ? EUserDataBlockUsage::Path : EUserDataBlockUsage::Content); } } if (node.IsCallable("FolderPath")) { const auto& name = node.Head().Content(); - auto blocks = types.UserDataStorage->FindUserDataFolder(name); + auto blocks = types.UserDataStorage->FindUserDataFolder(name); if (!blocks) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Folder not found: " << name)); return false; } else { for (const auto& x : *blocks) { - auto iter = files.insert({ x.first, *x.second }).first; + auto iter = files.insert({ x.first, *x.second }).first; iter->second.Usage.Set(EUserDataBlockUsage::Path); } } @@ -508,32 +508,32 @@ bool FillUsedFilesImpl( } if (addSysModule) { - auto pathWithMd5 = types.UdfResolver->GetSystemModulePath(moduleName); - YQL_ENSURE(pathWithMd5); + auto pathWithMd5 = types.UdfResolver->GetSystemModulePath(moduleName); + YQL_ENSURE(pathWithMd5); TUserDataBlock sysBlock; sysBlock.Type = EUserDataType::PATH; - sysBlock.Data = pathWithMd5->Path; - sysBlock.Usage.Set(EUserDataBlockUsage::Udf); - - auto alias = TFsPath(sysBlock.Data).GetName(); - auto key = TUserDataKey::Udf(alias); - if (const auto block = types.UserDataStorage->FindUserDataBlock(key)) { - files[key] = *block; - YQL_ENSURE(block->FrozenFile); - } else { - // Check alias clash with user files - if (files.contains(TUserDataStorage::ComposeUserDataKey(alias))) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "System module name " << alias << " clashes with one of the user's files")); - return false; - } - - if (!alias.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix) && !files.contains(key)) { - // CreateFakeFileLink calculates md5 for file, let's do it once - sysBlock.FrozenFile = CreateFakeFileLink(sysBlock.Data, pathWithMd5->Md5); - files[key] = sysBlock; - types.UserDataStorage->AddUserDataBlock(key, sysBlock); - } - } + sysBlock.Data = pathWithMd5->Path; + sysBlock.Usage.Set(EUserDataBlockUsage::Udf); + + auto alias = TFsPath(sysBlock.Data).GetName(); + auto key = TUserDataKey::Udf(alias); + if (const auto block = types.UserDataStorage->FindUserDataBlock(key)) { + files[key] = *block; + YQL_ENSURE(block->FrozenFile); + } else { + // Check alias clash with user files + if (files.contains(TUserDataStorage::ComposeUserDataKey(alias))) { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "System module name " << alias << " clashes with one of the user's files")); + return false; + } + + if (!alias.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix) && !files.contains(key)) { + // CreateFakeFileLink calculates md5 for file, let's do it once + sysBlock.FrozenFile = CreateFakeFileLink(sysBlock.Data, pathWithMd5->Md5); + files[key] = sysBlock; + types.UserDataStorage->AddUserDataBlock(key, sysBlock); + } + } } } @@ -627,9 +627,9 @@ bool FillUsedFiles( std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter, const TUserDataTable& crutches) { if (!FillUsedFiles(node, files, types, ctx, crutches)) { return SyncError(); - } - - auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter); + } + + auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter); if (future.Wait(TDuration::Zero())) { files = future.GetValue()(); return SyncOk(); @@ -666,18 +666,18 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedF }); })); } -} - -bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter) { +} + +bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter) { if (!FillUsedFiles(node, files, types, ctx)) { - return false; - } - - auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter); - files = future.GetValueSync()(); - return true; -} - + return false; + } + + auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter); + files = future.GetValueSync()(); + return true; +} + void WriteColumns(NYson::TYsonWriter& writer, const TExprBase& columns) { if (auto maybeList = columns.Maybe<TCoAtomList>()) { writer.OnBeginList(); diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index 9e80cfd135..48981bdcdc 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -106,7 +106,7 @@ bool FillUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnno std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter, const TUserDataTable& crutches = {}); -bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter); +bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter); void WriteColumns(NYson::TYsonWriter& writer, const NNodes::TExprBase& columns); diff --git a/ydb/library/yql/providers/common/structured_token/ut/ya.make b/ydb/library/yql/providers/common/structured_token/ut/ya.make index 99ad5b2da0..c8e0c696a4 100644 --- a/ydb/library/yql/providers/common/structured_token/ut/ya.make +++ b/ydb/library/yql/providers/common/structured_token/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(ydb/library/yql/providers/common/structured_token) - -OWNER( + +OWNER( g:yq - g:yql -) - -SRCS( - yql_structured_token_ut.cpp -) - -END() + g:yql +) + +SRCS( + yql_structured_token_ut.cpp +) + +END() diff --git a/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp b/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp index 95c97cdbb8..dbf4a47cd4 100644 --- a/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp +++ b/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp @@ -1,77 +1,77 @@ #include <ydb/library/yql/providers/common/structured_token/yql_structured_token.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(StructuredTokenTest) { - Y_UNIT_TEST(EmptyToken) { - const TStructuredToken t1; - UNIT_ASSERT_VALUES_EQUAL("{}", t1.ToJson()); - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("key", "xyz")); - - auto t2 = ParseStructuredToken("{}"); - UNIT_ASSERT_VALUES_EQUAL("{}", t2.ToJson()); - } - - Y_UNIT_TEST(IsStructuredTokenJson) { - UNIT_ASSERT(!IsStructuredTokenJson("")); - UNIT_ASSERT(!IsStructuredTokenJson("my_token")); - UNIT_ASSERT(IsStructuredTokenJson("{}")); - UNIT_ASSERT(IsStructuredTokenJson(R"({"f1":"my_token"})")); - } - - Y_UNIT_TEST(SetField) { - TStructuredToken t1; - t1.SetField("f1", "xxx"); - t1.SetField("f2", "yyy"); - t1.SetField("oops", "zzzzzzzzzzzzz"); - - UNIT_ASSERT_VALUES_EQUAL("xxx", t1.GetFieldOrDefault("f1", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("yyy", t1.GetField("f2")); - UNIT_ASSERT_VALUES_EQUAL(R"({"f1":"xxx","f2":"yyy","oops":"zzzzzzzzzzzzz"})", t1.ToJson()); - - const TStructuredToken t2 = ParseStructuredToken(t1.ToJson()); - UNIT_ASSERT_VALUES_EQUAL("xxx", t2.GetFieldOrDefault("f1", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f3", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("yyy", t2.GetField("f2")); - UNIT_ASSERT_VALUES_EQUAL(t1.ToJson(), t2.ToJson()); - } - - Y_UNIT_TEST(SetNonUtf8Field) { - TStructuredToken t1; - TString nonUtf8(TStringBuf("\xF0\x9F\x94")); - t1.SetField("f1", nonUtf8); - UNIT_ASSERT_VALUES_EQUAL(R"foo({"f1(base64)":"8J+U"})foo", t1.ToJson()); - UNIT_ASSERT_VALUES_EQUAL(nonUtf8, t1.GetField("f1")); - UNIT_ASSERT_VALUES_EQUAL("empty", t1.GetFieldOrDefault("f1(base64)", "empty")); - - const TStructuredToken t2 = ParseStructuredToken(t1.ToJson()); - UNIT_ASSERT_VALUES_EQUAL(nonUtf8, t2.GetField("f1")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f2", "xyz")); - UNIT_ASSERT_VALUES_EQUAL(t1.ToJson(), t2.ToJson()); - } - - Y_UNIT_TEST(Move) { - TStructuredToken t1; - t1 - .SetField("f1", "xxx") - .SetField("f2", "yyy"); - - UNIT_ASSERT_VALUES_EQUAL("xxx", t1.GetFieldOrDefault("f1", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("yyy", t1.GetFieldOrDefault("f2", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); - - const TStructuredToken t2(std::move(t1)); - UNIT_ASSERT_VALUES_EQUAL("xxx", t2.GetFieldOrDefault("f1", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("yyy", t2.GetFieldOrDefault("f2", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f3", "xyz")); - - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f1", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f2", "xyz")); - UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); - } -} - -} + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(StructuredTokenTest) { + Y_UNIT_TEST(EmptyToken) { + const TStructuredToken t1; + UNIT_ASSERT_VALUES_EQUAL("{}", t1.ToJson()); + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("key", "xyz")); + + auto t2 = ParseStructuredToken("{}"); + UNIT_ASSERT_VALUES_EQUAL("{}", t2.ToJson()); + } + + Y_UNIT_TEST(IsStructuredTokenJson) { + UNIT_ASSERT(!IsStructuredTokenJson("")); + UNIT_ASSERT(!IsStructuredTokenJson("my_token")); + UNIT_ASSERT(IsStructuredTokenJson("{}")); + UNIT_ASSERT(IsStructuredTokenJson(R"({"f1":"my_token"})")); + } + + Y_UNIT_TEST(SetField) { + TStructuredToken t1; + t1.SetField("f1", "xxx"); + t1.SetField("f2", "yyy"); + t1.SetField("oops", "zzzzzzzzzzzzz"); + + UNIT_ASSERT_VALUES_EQUAL("xxx", t1.GetFieldOrDefault("f1", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("yyy", t1.GetField("f2")); + UNIT_ASSERT_VALUES_EQUAL(R"({"f1":"xxx","f2":"yyy","oops":"zzzzzzzzzzzzz"})", t1.ToJson()); + + const TStructuredToken t2 = ParseStructuredToken(t1.ToJson()); + UNIT_ASSERT_VALUES_EQUAL("xxx", t2.GetFieldOrDefault("f1", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f3", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("yyy", t2.GetField("f2")); + UNIT_ASSERT_VALUES_EQUAL(t1.ToJson(), t2.ToJson()); + } + + Y_UNIT_TEST(SetNonUtf8Field) { + TStructuredToken t1; + TString nonUtf8(TStringBuf("\xF0\x9F\x94")); + t1.SetField("f1", nonUtf8); + UNIT_ASSERT_VALUES_EQUAL(R"foo({"f1(base64)":"8J+U"})foo", t1.ToJson()); + UNIT_ASSERT_VALUES_EQUAL(nonUtf8, t1.GetField("f1")); + UNIT_ASSERT_VALUES_EQUAL("empty", t1.GetFieldOrDefault("f1(base64)", "empty")); + + const TStructuredToken t2 = ParseStructuredToken(t1.ToJson()); + UNIT_ASSERT_VALUES_EQUAL(nonUtf8, t2.GetField("f1")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f2", "xyz")); + UNIT_ASSERT_VALUES_EQUAL(t1.ToJson(), t2.ToJson()); + } + + Y_UNIT_TEST(Move) { + TStructuredToken t1; + t1 + .SetField("f1", "xxx") + .SetField("f2", "yyy"); + + UNIT_ASSERT_VALUES_EQUAL("xxx", t1.GetFieldOrDefault("f1", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("yyy", t1.GetFieldOrDefault("f2", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); + + const TStructuredToken t2(std::move(t1)); + UNIT_ASSERT_VALUES_EQUAL("xxx", t2.GetFieldOrDefault("f1", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("yyy", t2.GetFieldOrDefault("f2", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t2.GetFieldOrDefault("f3", "xyz")); + + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f1", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f2", "xyz")); + UNIT_ASSERT_VALUES_EQUAL("xyz", t1.GetFieldOrDefault("f3", "xyz")); + } +} + +} diff --git a/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp b/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp index 3675014d48..e417e18aa4 100644 --- a/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp +++ b/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp @@ -1,94 +1,94 @@ #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(TokenBuilderTest) { - Y_UNIT_TEST(Empty) { - const TStructuredTokenBuilder b; - UNIT_ASSERT_VALUES_EQUAL("{}", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(!p.HasServiceAccountIdAuth()); - UNIT_ASSERT(!p.HasBasicAuth()); - UNIT_ASSERT(!p.HasIAMToken()); - UNIT_ASSERT(!p.IsNoAuth()); - } - - Y_UNIT_TEST(ServiceAccountId) { - TStructuredTokenBuilder b; - b.SetServiceAccountIdAuth("my_sa_id", "my_sa_sign"); - UNIT_ASSERT_VALUES_EQUAL(R"({"sa_id":"my_sa_id","sa_id_signature":"my_sa_sign"})", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(p.HasServiceAccountIdAuth()); - UNIT_ASSERT(!p.HasBasicAuth()); - UNIT_ASSERT(!p.HasIAMToken()); - UNIT_ASSERT(!p.IsNoAuth()); - TString id, sign; - UNIT_ASSERT(p.GetServiceAccountIdAuth(id, sign)); - UNIT_ASSERT_VALUES_EQUAL(id, "my_sa_id"); - UNIT_ASSERT_VALUES_EQUAL(sign, "my_sa_sign"); - } - - Y_UNIT_TEST(BasicAuth) { - TStructuredTokenBuilder b; - b.SetBasicAuth("my_login", "my_passw"); - UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password":"my_passw"})", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(!p.HasServiceAccountIdAuth()); - UNIT_ASSERT(p.HasBasicAuth()); - UNIT_ASSERT(!p.HasIAMToken()); - UNIT_ASSERT(!p.IsNoAuth()); - TString login, password; - UNIT_ASSERT(p.GetBasicAuth(login, password)); - UNIT_ASSERT_VALUES_EQUAL(login, "my_login"); - UNIT_ASSERT_VALUES_EQUAL(password, "my_passw"); - } - - Y_UNIT_TEST(IAMToken) { - TStructuredTokenBuilder b; - b.SetIAMToken("my_token"); - UNIT_ASSERT_VALUES_EQUAL(R"({"token":"my_token"})", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(!p.HasServiceAccountIdAuth()); - UNIT_ASSERT(!p.HasBasicAuth()); - UNIT_ASSERT(p.HasIAMToken()); - UNIT_ASSERT(!p.IsNoAuth()); - TString token = p.GetIAMToken(); - UNIT_ASSERT_VALUES_EQUAL(token, "my_token"); - } - - Y_UNIT_TEST(NoAuth) { - TStructuredTokenBuilder b; - b.SetNoAuth(); - UNIT_ASSERT_VALUES_EQUAL(R"({"no_auth":""})", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(!p.HasServiceAccountIdAuth()); - UNIT_ASSERT(!p.HasBasicAuth()); - UNIT_ASSERT(!p.HasIAMToken()); - UNIT_ASSERT(p.IsNoAuth()); - } - - Y_UNIT_TEST(BasicAuthAndToken) { - TStructuredTokenBuilder b; - b.SetBasicAuth("my_login", "my_passw"); - b.SetIAMToken("my_token"); - UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password":"my_passw","token":"my_token"})", b.ToJson()); - const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); - UNIT_ASSERT(!p.HasServiceAccountIdAuth()); - UNIT_ASSERT(p.HasBasicAuth()); - UNIT_ASSERT(p.HasIAMToken()); - UNIT_ASSERT(!p.IsNoAuth()); - - TString login, password; - UNIT_ASSERT(p.GetBasicAuth(login, password)); - UNIT_ASSERT_VALUES_EQUAL(login, "my_login"); - UNIT_ASSERT_VALUES_EQUAL(password, "my_passw"); - - TString token = p.GetIAMToken(); - UNIT_ASSERT_VALUES_EQUAL(token, "my_token"); - - } -} - -} + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TokenBuilderTest) { + Y_UNIT_TEST(Empty) { + const TStructuredTokenBuilder b; + UNIT_ASSERT_VALUES_EQUAL("{}", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(!p.HasServiceAccountIdAuth()); + UNIT_ASSERT(!p.HasBasicAuth()); + UNIT_ASSERT(!p.HasIAMToken()); + UNIT_ASSERT(!p.IsNoAuth()); + } + + Y_UNIT_TEST(ServiceAccountId) { + TStructuredTokenBuilder b; + b.SetServiceAccountIdAuth("my_sa_id", "my_sa_sign"); + UNIT_ASSERT_VALUES_EQUAL(R"({"sa_id":"my_sa_id","sa_id_signature":"my_sa_sign"})", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(p.HasServiceAccountIdAuth()); + UNIT_ASSERT(!p.HasBasicAuth()); + UNIT_ASSERT(!p.HasIAMToken()); + UNIT_ASSERT(!p.IsNoAuth()); + TString id, sign; + UNIT_ASSERT(p.GetServiceAccountIdAuth(id, sign)); + UNIT_ASSERT_VALUES_EQUAL(id, "my_sa_id"); + UNIT_ASSERT_VALUES_EQUAL(sign, "my_sa_sign"); + } + + Y_UNIT_TEST(BasicAuth) { + TStructuredTokenBuilder b; + b.SetBasicAuth("my_login", "my_passw"); + UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password":"my_passw"})", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(!p.HasServiceAccountIdAuth()); + UNIT_ASSERT(p.HasBasicAuth()); + UNIT_ASSERT(!p.HasIAMToken()); + UNIT_ASSERT(!p.IsNoAuth()); + TString login, password; + UNIT_ASSERT(p.GetBasicAuth(login, password)); + UNIT_ASSERT_VALUES_EQUAL(login, "my_login"); + UNIT_ASSERT_VALUES_EQUAL(password, "my_passw"); + } + + Y_UNIT_TEST(IAMToken) { + TStructuredTokenBuilder b; + b.SetIAMToken("my_token"); + UNIT_ASSERT_VALUES_EQUAL(R"({"token":"my_token"})", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(!p.HasServiceAccountIdAuth()); + UNIT_ASSERT(!p.HasBasicAuth()); + UNIT_ASSERT(p.HasIAMToken()); + UNIT_ASSERT(!p.IsNoAuth()); + TString token = p.GetIAMToken(); + UNIT_ASSERT_VALUES_EQUAL(token, "my_token"); + } + + Y_UNIT_TEST(NoAuth) { + TStructuredTokenBuilder b; + b.SetNoAuth(); + UNIT_ASSERT_VALUES_EQUAL(R"({"no_auth":""})", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(!p.HasServiceAccountIdAuth()); + UNIT_ASSERT(!p.HasBasicAuth()); + UNIT_ASSERT(!p.HasIAMToken()); + UNIT_ASSERT(p.IsNoAuth()); + } + + Y_UNIT_TEST(BasicAuthAndToken) { + TStructuredTokenBuilder b; + b.SetBasicAuth("my_login", "my_passw"); + b.SetIAMToken("my_token"); + UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password":"my_passw","token":"my_token"})", b.ToJson()); + const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson()); + UNIT_ASSERT(!p.HasServiceAccountIdAuth()); + UNIT_ASSERT(p.HasBasicAuth()); + UNIT_ASSERT(p.HasIAMToken()); + UNIT_ASSERT(!p.IsNoAuth()); + + TString login, password; + UNIT_ASSERT(p.GetBasicAuth(login, password)); + UNIT_ASSERT_VALUES_EQUAL(login, "my_login"); + UNIT_ASSERT_VALUES_EQUAL(password, "my_passw"); + + TString token = p.GetIAMToken(); + UNIT_ASSERT_VALUES_EQUAL(token, "my_token"); + + } +} + +} diff --git a/ydb/library/yql/providers/common/structured_token/ya.make b/ydb/library/yql/providers/common/structured_token/ya.make index 777ab09910..73df2c2600 100644 --- a/ydb/library/yql/providers/common/structured_token/ya.make +++ b/ydb/library/yql/providers/common/structured_token/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - yql_structured_token.cpp - yql_structured_token.h - yql_token_builder.cpp - yql_token_builder.h -) - -PEERDIR( - library/cpp/json - library/cpp/string_utils/base64 +LIBRARY() + +OWNER(g:yql) + +SRCS( + yql_structured_token.cpp + yql_structured_token.h + yql_token_builder.cpp + yql_token_builder.h +) + +PEERDIR( + library/cpp/json + library/cpp/string_utils/base64 ydb/library/yql/utils -) - -END() - -RECURSE_FOR_TESTS( - ut -) +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp b/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp index 84a2f06b3c..7a05271165 100644 --- a/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp +++ b/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp @@ -1,84 +1,84 @@ -#include "yql_structured_token.h" - +#include "yql_structured_token.h" + #include <ydb/library/yql/utils/utf8.h> - -#include <library/cpp/json/json_reader.h> -#include <library/cpp/json/json_writer.h> -#include <library/cpp/string_utils/base64/base64.h> - - -namespace NYql { - -TStructuredToken::TStructuredToken(TMap<TString, TString>&& data) - : Data(std::move(data)) { - -} - -TString TStructuredToken::GetField(const TString& name) const { - return Data.at(name); -} - -TString TStructuredToken::GetFieldOrDefault(const TString& name, const TString& defaultValue) const { - return FindField(name).GetOrElse(defaultValue); -} - -TMaybe<TString> TStructuredToken::FindField(const TString& name) const { - auto* r = Data.FindPtr(name); - return r ? MakeMaybe(*r) : Nothing(); -} - -bool TStructuredToken::HasField(const TString& name) const { - return Data.contains(name); -} - -TStructuredToken& TStructuredToken::SetField(const TString& name, const TString& value) { - Data[name] = value; - return *this; -} - -TString TStructuredToken::ToJson() const { - TStringStream output; - // set "format output" to false, no need for extra indents - // "sort keys" value is not used actually - // turn on UTF8 validation (need for keys checks) - NJson::TJsonWriter writer(&output, false, true, true); - writer.OpenMap(); - - for (auto&[k, v] : Data) { - if (!IsUtf8(v)) { - writer.Write(k + "(base64)", Base64Encode(TStringBuf(v))); - } else { - writer.Write(k, v); - } - } - writer.CloseMap(); - writer.Flush(); - - return output.Str(); -} - -TStructuredToken ParseStructuredToken(const TString& content) { - Y_VERIFY(IsStructuredTokenJson(content)); - - NJson::TJsonValue v; - // will throw on error - NJson::ReadJsonTree(content, &v, true); - TMap<TString, TString> data; - const auto& m = v.GetMapSafe(); - for (auto&[k, v] : m) { - TStringBuf key(k); - if (key.ChopSuffix("(base64)")) { - const auto& s = v.GetStringSafe(); - data[TString(key)] = Base64Decode(TStringBuf(s)); - } else { - data[k] = v.GetStringSafe(); - } - } - return TStructuredToken(std::move(data)); -} - -bool IsStructuredTokenJson(const TString& content) { - return content.StartsWith("{") && content.EndsWith("}"); -} - -} // namespace NYql + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> +#include <library/cpp/string_utils/base64/base64.h> + + +namespace NYql { + +TStructuredToken::TStructuredToken(TMap<TString, TString>&& data) + : Data(std::move(data)) { + +} + +TString TStructuredToken::GetField(const TString& name) const { + return Data.at(name); +} + +TString TStructuredToken::GetFieldOrDefault(const TString& name, const TString& defaultValue) const { + return FindField(name).GetOrElse(defaultValue); +} + +TMaybe<TString> TStructuredToken::FindField(const TString& name) const { + auto* r = Data.FindPtr(name); + return r ? MakeMaybe(*r) : Nothing(); +} + +bool TStructuredToken::HasField(const TString& name) const { + return Data.contains(name); +} + +TStructuredToken& TStructuredToken::SetField(const TString& name, const TString& value) { + Data[name] = value; + return *this; +} + +TString TStructuredToken::ToJson() const { + TStringStream output; + // set "format output" to false, no need for extra indents + // "sort keys" value is not used actually + // turn on UTF8 validation (need for keys checks) + NJson::TJsonWriter writer(&output, false, true, true); + writer.OpenMap(); + + for (auto&[k, v] : Data) { + if (!IsUtf8(v)) { + writer.Write(k + "(base64)", Base64Encode(TStringBuf(v))); + } else { + writer.Write(k, v); + } + } + writer.CloseMap(); + writer.Flush(); + + return output.Str(); +} + +TStructuredToken ParseStructuredToken(const TString& content) { + Y_VERIFY(IsStructuredTokenJson(content)); + + NJson::TJsonValue v; + // will throw on error + NJson::ReadJsonTree(content, &v, true); + TMap<TString, TString> data; + const auto& m = v.GetMapSafe(); + for (auto&[k, v] : m) { + TStringBuf key(k); + if (key.ChopSuffix("(base64)")) { + const auto& s = v.GetStringSafe(); + data[TString(key)] = Base64Decode(TStringBuf(s)); + } else { + data[k] = v.GetStringSafe(); + } + } + return TStructuredToken(std::move(data)); +} + +bool IsStructuredTokenJson(const TString& content) { + return content.StartsWith("{") && content.EndsWith("}"); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/common/structured_token/yql_structured_token.h b/ydb/library/yql/providers/common/structured_token/yql_structured_token.h index 6ee1f478b7..365ac7462e 100644 --- a/ydb/library/yql/providers/common/structured_token/yql_structured_token.h +++ b/ydb/library/yql/providers/common/structured_token/yql_structured_token.h @@ -1,30 +1,30 @@ -#pragma once - -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/string.h> - -namespace NYql { - -class TStructuredToken { -public: - explicit TStructuredToken(TMap<TString, TString>&& data = {}); - TStructuredToken(TStructuredToken&&) = default; - TStructuredToken(const TStructuredToken&) = default; - - TString GetField(const TString& name) const; - TString GetFieldOrDefault(const TString& name, const TString& defaultValue) const; - TMaybe<TString> FindField(const TString& name) const; - bool HasField(const TString& name) const; - TStructuredToken& SetField(const TString& name, const TString& value); - TString ToJson() const; - -private: - TMap<TString, TString> Data; -}; - -// is used for backward compatibility when content contains just token -bool IsStructuredTokenJson(const TString& content); -TStructuredToken ParseStructuredToken(const TString& content); - -} // namespace NYql +#pragma once + +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/string.h> + +namespace NYql { + +class TStructuredToken { +public: + explicit TStructuredToken(TMap<TString, TString>&& data = {}); + TStructuredToken(TStructuredToken&&) = default; + TStructuredToken(const TStructuredToken&) = default; + + TString GetField(const TString& name) const; + TString GetFieldOrDefault(const TString& name, const TString& defaultValue) const; + TMaybe<TString> FindField(const TString& name) const; + bool HasField(const TString& name) const; + TStructuredToken& SetField(const TString& name, const TString& value); + TString ToJson() const; + +private: + TMap<TString, TString> Data; +}; + +// is used for backward compatibility when content contains just token +bool IsStructuredTokenJson(const TString& content); +TStructuredToken ParseStructuredToken(const TString& content); + +} // namespace NYql diff --git a/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp b/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp index 3bace92c54..3c0cacd370 100644 --- a/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp +++ b/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp @@ -1,102 +1,102 @@ -#include "yql_token_builder.h" - -namespace NYql { -TStructuredTokenBuilder::TStructuredTokenBuilder() { - -} - -TStructuredTokenBuilder::TStructuredTokenBuilder(const TStructuredToken& data) - : Data(data) -{ -} - -TStructuredTokenBuilder& TStructuredTokenBuilder::SetServiceAccountIdAuth(const TString& accountId, const TString& accountIdSignature) { - Data.SetField("sa_id", accountId); - Data.SetField("sa_id_signature", accountIdSignature); - return *this; -} - -TStructuredTokenBuilder& TStructuredTokenBuilder::SetBasicAuth(const TString& login, const TString& password) { - Data.SetField("basic_login", login); - Data.SetField("basic_password", password); - return *this; -} - -TStructuredTokenBuilder& TStructuredTokenBuilder::SetIAMToken(const TString& token) { - Data.SetField("token", token); - return *this; -} - -TStructuredTokenBuilder& TStructuredTokenBuilder::SetNoAuth() { - Data.SetField("no_auth", {}); - return *this; -} - -TString TStructuredTokenBuilder::ToJson() const { - return Data.ToJson(); -} - -/////////////////////////////////////////////////////////////////// - -TStructuredTokenParser::TStructuredTokenParser(TStructuredToken&& data) - : Data(std::move(data)) -{ -} - -bool TStructuredTokenParser::HasServiceAccountIdAuth() const { - return Data.HasField("sa_id"); -} - -bool TStructuredTokenParser::GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const { - accountId = Data.GetField("sa_id"); - accountIdSignature = Data.GetField("sa_id_signature"); - return true; -} - -bool TStructuredTokenParser::HasBasicAuth() const { - return Data.HasField("basic_login"); -} - -bool TStructuredTokenParser::GetBasicAuth(TString& login, TString& password) const { - login = Data.GetField("basic_login"); - password = Data.GetField("basic_password"); - return true; -} - -bool TStructuredTokenParser::HasIAMToken() const { - return Data.HasField("token"); -} - -TString TStructuredTokenParser::GetIAMToken() const { - return Data.GetField("token"); -} - -bool TStructuredTokenParser::IsNoAuth() const { - return Data.HasField("no_auth"); -} - -TStructuredTokenBuilder TStructuredTokenParser::ToBuilder() const { - return TStructuredTokenBuilder(Data); -} - -TStructuredTokenParser CreateStructuredTokenParser(const TString& content = {}) { - return content ? TStructuredTokenParser(ParseStructuredToken(content)) : TStructuredTokenParser(TStructuredToken({})); -} - -TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token) { - TStructuredTokenBuilder result; - if (serviceAccountId && serviceAccountIdSignature) { - result.SetServiceAccountIdAuth(serviceAccountId, serviceAccountIdSignature); - return result.ToJson(); - } - - if (token) { - result.SetIAMToken(token); - return result.ToJson(); - } - - result.SetNoAuth(); - return result.ToJson(); -} - -} +#include "yql_token_builder.h" + +namespace NYql { +TStructuredTokenBuilder::TStructuredTokenBuilder() { + +} + +TStructuredTokenBuilder::TStructuredTokenBuilder(const TStructuredToken& data) + : Data(data) +{ +} + +TStructuredTokenBuilder& TStructuredTokenBuilder::SetServiceAccountIdAuth(const TString& accountId, const TString& accountIdSignature) { + Data.SetField("sa_id", accountId); + Data.SetField("sa_id_signature", accountIdSignature); + return *this; +} + +TStructuredTokenBuilder& TStructuredTokenBuilder::SetBasicAuth(const TString& login, const TString& password) { + Data.SetField("basic_login", login); + Data.SetField("basic_password", password); + return *this; +} + +TStructuredTokenBuilder& TStructuredTokenBuilder::SetIAMToken(const TString& token) { + Data.SetField("token", token); + return *this; +} + +TStructuredTokenBuilder& TStructuredTokenBuilder::SetNoAuth() { + Data.SetField("no_auth", {}); + return *this; +} + +TString TStructuredTokenBuilder::ToJson() const { + return Data.ToJson(); +} + +/////////////////////////////////////////////////////////////////// + +TStructuredTokenParser::TStructuredTokenParser(TStructuredToken&& data) + : Data(std::move(data)) +{ +} + +bool TStructuredTokenParser::HasServiceAccountIdAuth() const { + return Data.HasField("sa_id"); +} + +bool TStructuredTokenParser::GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const { + accountId = Data.GetField("sa_id"); + accountIdSignature = Data.GetField("sa_id_signature"); + return true; +} + +bool TStructuredTokenParser::HasBasicAuth() const { + return Data.HasField("basic_login"); +} + +bool TStructuredTokenParser::GetBasicAuth(TString& login, TString& password) const { + login = Data.GetField("basic_login"); + password = Data.GetField("basic_password"); + return true; +} + +bool TStructuredTokenParser::HasIAMToken() const { + return Data.HasField("token"); +} + +TString TStructuredTokenParser::GetIAMToken() const { + return Data.GetField("token"); +} + +bool TStructuredTokenParser::IsNoAuth() const { + return Data.HasField("no_auth"); +} + +TStructuredTokenBuilder TStructuredTokenParser::ToBuilder() const { + return TStructuredTokenBuilder(Data); +} + +TStructuredTokenParser CreateStructuredTokenParser(const TString& content = {}) { + return content ? TStructuredTokenParser(ParseStructuredToken(content)) : TStructuredTokenParser(TStructuredToken({})); +} + +TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token) { + TStructuredTokenBuilder result; + if (serviceAccountId && serviceAccountIdSignature) { + result.SetServiceAccountIdAuth(serviceAccountId, serviceAccountIdSignature); + return result.ToJson(); + } + + if (token) { + result.SetIAMToken(token); + return result.ToJson(); + } + + result.SetNoAuth(); + return result.ToJson(); +} + +} diff --git a/ydb/library/yql/providers/common/structured_token/yql_token_builder.h b/ydb/library/yql/providers/common/structured_token/yql_token_builder.h index aec2659446..55217d0cfc 100644 --- a/ydb/library/yql/providers/common/structured_token/yql_token_builder.h +++ b/ydb/library/yql/providers/common/structured_token/yql_token_builder.h @@ -1,43 +1,43 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/structured_token/yql_structured_token.h> - -namespace NYql { - -class TStructuredTokenBuilder { -public: - TStructuredTokenBuilder(); - explicit TStructuredTokenBuilder(const TStructuredToken& data); - TStructuredTokenBuilder(TStructuredTokenBuilder&&) = default; - - TStructuredTokenBuilder& SetServiceAccountIdAuth(const TString& accountId, const TString& accountIdSignature); - TStructuredTokenBuilder& SetBasicAuth(const TString& login, const TString& password); - TStructuredTokenBuilder& SetIAMToken(const TString& token); - TStructuredTokenBuilder& SetNoAuth(); - - TString ToJson() const; - -private: - TStructuredToken Data; -}; - -class TStructuredTokenParser { -public: - explicit TStructuredTokenParser(TStructuredToken&& data); - bool HasServiceAccountIdAuth() const; - bool GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const; - bool HasBasicAuth() const; - bool GetBasicAuth(TString& login, TString& password) const; - bool HasIAMToken() const; - TString GetIAMToken() const; - bool IsNoAuth() const; - - TStructuredTokenBuilder ToBuilder() const; - -private: - const TStructuredToken Data; -}; - -TStructuredTokenParser CreateStructuredTokenParser(const TString& content); -TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token); -} + +namespace NYql { + +class TStructuredTokenBuilder { +public: + TStructuredTokenBuilder(); + explicit TStructuredTokenBuilder(const TStructuredToken& data); + TStructuredTokenBuilder(TStructuredTokenBuilder&&) = default; + + TStructuredTokenBuilder& SetServiceAccountIdAuth(const TString& accountId, const TString& accountIdSignature); + TStructuredTokenBuilder& SetBasicAuth(const TString& login, const TString& password); + TStructuredTokenBuilder& SetIAMToken(const TString& token); + TStructuredTokenBuilder& SetNoAuth(); + + TString ToJson() const; + +private: + TStructuredToken Data; +}; + +class TStructuredTokenParser { +public: + explicit TStructuredTokenParser(TStructuredToken&& data); + bool HasServiceAccountIdAuth() const; + bool GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const; + bool HasBasicAuth() const; + bool GetBasicAuth(TString& login, TString& password) const; + bool HasIAMToken() const; + TString GetIAMToken() const; + bool IsNoAuth() const; + + TStructuredTokenBuilder ToBuilder() const; + +private: + const TStructuredToken Data; +}; + +TStructuredTokenParser CreateStructuredTokenParser(const TString& content); +TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token); +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp index fd0554a315..685b8e06d4 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.cpp @@ -1,54 +1,54 @@ -#include "bearer_credentials_provider.h" -#include <util/string/cast.h> - -namespace NYql { - -namespace { -class TBearerCredentialsProvider : public NYdb::ICredentialsProvider { -public: - explicit TBearerCredentialsProvider(std::shared_ptr<NYdb::ICredentialsProvider> delegatee) - : Delegatee(delegatee) { - } - - TString GetAuthInfo() const override { - TString result = Delegatee->GetAuthInfo(); - if (!result || result.StartsWith("Bearer ")) { - return result; - } - - return "Bearer " + result; - } - - bool IsValid() const override { - return Delegatee->IsValid(); - } - -private: - const std::shared_ptr<NYdb::ICredentialsProvider> Delegatee; -}; - -class TBearerCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { -public: - explicit TBearerCredentialsProviderFactory(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee) - : Delegatee(delegatee) { - } - - TString GetClientIdentity() const override { - return "BEARER_CRED_PROV_FACTORY" + ToString((ui64)this); - } - - std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { - return std::make_shared<TBearerCredentialsProvider>(Delegatee->CreateProvider()); - } - -private: - const std::shared_ptr<NYdb::ICredentialsProviderFactory> Delegatee; -}; - -} - -std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapCredentialsProviderFactoryWithBearer(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee) { - return std::make_shared<TBearerCredentialsProviderFactory>(delegatee); -} - -} +#include "bearer_credentials_provider.h" +#include <util/string/cast.h> + +namespace NYql { + +namespace { +class TBearerCredentialsProvider : public NYdb::ICredentialsProvider { +public: + explicit TBearerCredentialsProvider(std::shared_ptr<NYdb::ICredentialsProvider> delegatee) + : Delegatee(delegatee) { + } + + TString GetAuthInfo() const override { + TString result = Delegatee->GetAuthInfo(); + if (!result || result.StartsWith("Bearer ")) { + return result; + } + + return "Bearer " + result; + } + + bool IsValid() const override { + return Delegatee->IsValid(); + } + +private: + const std::shared_ptr<NYdb::ICredentialsProvider> Delegatee; +}; + +class TBearerCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { +public: + explicit TBearerCredentialsProviderFactory(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee) + : Delegatee(delegatee) { + } + + TString GetClientIdentity() const override { + return "BEARER_CRED_PROV_FACTORY" + ToString((ui64)this); + } + + std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { + return std::make_shared<TBearerCredentialsProvider>(Delegatee->CreateProvider()); + } + +private: + const std::shared_ptr<NYdb::ICredentialsProviderFactory> Delegatee; +}; + +} + +std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapCredentialsProviderFactoryWithBearer(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee) { + return std::make_shared<TBearerCredentialsProviderFactory>(delegatee); +} + +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h index 87034a9da2..bee3e89f8f 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h +++ b/ydb/library/yql/providers/common/token_accessor/client/bearer_credentials_provider.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> - -namespace NYql { - -std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapCredentialsProviderFactoryWithBearer( - std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee -); - -} + +namespace NYql { + +std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapCredentialsProviderFactoryWithBearer( + std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee +); + +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/factory.cpp b/ydb/library/yql/providers/common/token_accessor/client/factory.cpp index f849fec67b..755feab4a3 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/factory.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/factory.cpp @@ -1,84 +1,84 @@ -#include "factory.h" -#include "bearer_credentials_provider.h" -#include "token_accessor_client_factory.h" +#include "factory.h" +#include "bearer_credentials_provider.h" +#include "token_accessor_client_factory.h" #include <ydb/library/yql/providers/common/structured_token/yql_structured_token.h> #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> -#include <util/string/cast.h> - -namespace NYql { - -namespace { - -class TSecuredServiceAccountCredentialsFactoryImpl : public ISecuredServiceAccountCredentialsFactory { -public: - TSecuredServiceAccountCredentialsFactoryImpl( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TDuration& refreshPeriod, - const TDuration& requestTimeout - ) - : TokenAccessorEndpoint(tokenAccessorEndpoint) - , UseSsl(useSsl) - , RefreshPeriod(refreshPeriod) - , RequestTimeout(requestTimeout) { - } - - std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString& serviceAccountId, const TString& serviceAccountIdSignature) override { - Y_ENSURE(serviceAccountId); - Y_ENSURE(serviceAccountIdSignature); - - return CreateTokenAccessorCredentialsProviderFactory(TokenAccessorEndpoint, UseSsl, serviceAccountId, serviceAccountIdSignature, RefreshPeriod, RequestTimeout); - } - -private: - const TString TokenAccessorEndpoint; - const bool UseSsl; - const TDuration RefreshPeriod; - const TDuration RequestTimeout; -}; - -std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapWithBearerIfNeeded(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee, bool addBearerToToken) { - if (!addBearerToToken) { - return delegatee; - } - return WrapCredentialsProviderFactoryWithBearer(delegatee); -} -} - -ISecuredServiceAccountCredentialsFactory::TPtr CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TDuration& refreshPeriod, - const TDuration& requestTimeout) { - return std::make_shared<TSecuredServiceAccountCredentialsFactoryImpl>(tokenAccessorEndpoint, useSsl, refreshPeriod, requestTimeout); -} - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateCredentialsProviderFactoryForStructuredToken(ISecuredServiceAccountCredentialsFactory::TPtr factory, const TString& structuredTokenJson, bool addBearerToToken) { - if (!NYql::IsStructuredTokenJson(structuredTokenJson)) { - return WrapWithBearerIfNeeded(NYdb::CreateOAuthCredentialsProviderFactory(structuredTokenJson), addBearerToToken); - } - - NYql::TStructuredTokenParser parser = NYql::CreateStructuredTokenParser(structuredTokenJson); - if (parser.HasIAMToken()) { - return WrapWithBearerIfNeeded(NYdb::CreateOAuthCredentialsProviderFactory(parser.GetIAMToken()), addBearerToToken); // OK for any static token (OAuth, IAM). - } - - if (parser.HasServiceAccountIdAuth()) { - TString id; - TString signature; - parser.GetServiceAccountIdAuth(id, signature); - - if (!factory) { - ythrow yexception() << "Service account id credentials are not supported, service account id: " << id; - } - return WrapWithBearerIfNeeded(factory->Create(id, signature), addBearerToToken); - } - - if (parser.IsNoAuth()) { - return NYdb::CreateInsecureCredentialsProviderFactory(); - } - - ythrow yexception() << "Unrecognized token of length " << structuredTokenJson.size(); -} - -} +#include <util/string/cast.h> + +namespace NYql { + +namespace { + +class TSecuredServiceAccountCredentialsFactoryImpl : public ISecuredServiceAccountCredentialsFactory { +public: + TSecuredServiceAccountCredentialsFactoryImpl( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TDuration& refreshPeriod, + const TDuration& requestTimeout + ) + : TokenAccessorEndpoint(tokenAccessorEndpoint) + , UseSsl(useSsl) + , RefreshPeriod(refreshPeriod) + , RequestTimeout(requestTimeout) { + } + + std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString& serviceAccountId, const TString& serviceAccountIdSignature) override { + Y_ENSURE(serviceAccountId); + Y_ENSURE(serviceAccountIdSignature); + + return CreateTokenAccessorCredentialsProviderFactory(TokenAccessorEndpoint, UseSsl, serviceAccountId, serviceAccountIdSignature, RefreshPeriod, RequestTimeout); + } + +private: + const TString TokenAccessorEndpoint; + const bool UseSsl; + const TDuration RefreshPeriod; + const TDuration RequestTimeout; +}; + +std::shared_ptr<NYdb::ICredentialsProviderFactory> WrapWithBearerIfNeeded(std::shared_ptr<NYdb::ICredentialsProviderFactory> delegatee, bool addBearerToToken) { + if (!addBearerToToken) { + return delegatee; + } + return WrapCredentialsProviderFactoryWithBearer(delegatee); +} +} + +ISecuredServiceAccountCredentialsFactory::TPtr CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TDuration& refreshPeriod, + const TDuration& requestTimeout) { + return std::make_shared<TSecuredServiceAccountCredentialsFactoryImpl>(tokenAccessorEndpoint, useSsl, refreshPeriod, requestTimeout); +} + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateCredentialsProviderFactoryForStructuredToken(ISecuredServiceAccountCredentialsFactory::TPtr factory, const TString& structuredTokenJson, bool addBearerToToken) { + if (!NYql::IsStructuredTokenJson(structuredTokenJson)) { + return WrapWithBearerIfNeeded(NYdb::CreateOAuthCredentialsProviderFactory(structuredTokenJson), addBearerToToken); + } + + NYql::TStructuredTokenParser parser = NYql::CreateStructuredTokenParser(structuredTokenJson); + if (parser.HasIAMToken()) { + return WrapWithBearerIfNeeded(NYdb::CreateOAuthCredentialsProviderFactory(parser.GetIAMToken()), addBearerToToken); // OK for any static token (OAuth, IAM). + } + + if (parser.HasServiceAccountIdAuth()) { + TString id; + TString signature; + parser.GetServiceAccountIdAuth(id, signature); + + if (!factory) { + ythrow yexception() << "Service account id credentials are not supported, service account id: " << id; + } + return WrapWithBearerIfNeeded(factory->Create(id, signature), addBearerToToken); + } + + if (parser.IsNoAuth()) { + return NYdb::CreateInsecureCredentialsProviderFactory(); + } + + ythrow yexception() << "Unrecognized token of length " << structuredTokenJson.size(); +} + +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/factory.h b/ydb/library/yql/providers/common/token_accessor/client/factory.h index 610fb773d6..b05dfc3c3d 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/factory.h +++ b/ydb/library/yql/providers/common/token_accessor/client/factory.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> -#include <util/datetime/base.h> - -namespace NYql { - -class ISecuredServiceAccountCredentialsFactory { -public: - typedef std::shared_ptr<ISecuredServiceAccountCredentialsFactory> TPtr; - -public: - virtual ~ISecuredServiceAccountCredentialsFactory() {} - virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString& serviceAccountId, const TString& serviceAccountIdSignature) = 0; -}; - -ISecuredServiceAccountCredentialsFactory::TPtr CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TDuration& refreshPeriod = TDuration::Hours(1), - const TDuration& requestTimeout = TDuration::Seconds(10) -); - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateCredentialsProviderFactoryForStructuredToken(ISecuredServiceAccountCredentialsFactory::TPtr factory, const TString& structuredTokenJson, bool addBearerToToken = false); -} +#include <util/datetime/base.h> + +namespace NYql { + +class ISecuredServiceAccountCredentialsFactory { +public: + typedef std::shared_ptr<ISecuredServiceAccountCredentialsFactory> TPtr; + +public: + virtual ~ISecuredServiceAccountCredentialsFactory() {} + virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString& serviceAccountId, const TString& serviceAccountIdSignature) = 0; +}; + +ISecuredServiceAccountCredentialsFactory::TPtr CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TDuration& refreshPeriod = TDuration::Hours(1), + const TDuration& requestTimeout = TDuration::Seconds(10) +); + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateCredentialsProviderFactoryForStructuredToken(ISecuredServiceAccountCredentialsFactory::TPtr factory, const TString& structuredTokenJson, bool addBearerToToken = false); +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp index 3d65263cb9..bacf735549 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.cpp @@ -1,185 +1,185 @@ -#include "token_accessor_client.h" - +#include "token_accessor_client.h" + #include <ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.pb.h> #include <ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.grpc.pb.h> - -#include <library/cpp/grpc/client/grpc_client_low.h> -#include <library/cpp/threading/atomic/bool.h> -#include <library/cpp/threading/future/core/future.h> - -#include <util/string/builder.h> - -namespace NYql { - -namespace { - -const TDuration BACKOFF_START = TDuration::MilliSeconds(50); -const TDuration BACKOFF_MAX = TDuration::Seconds(10); - -class TTokenAccessorCredentialsProvider : public NYdb::ICredentialsProvider { -private: - class TImpl : public std::enable_shared_from_this<TImpl> { - public: - TImpl(const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod, - const TDuration& requestTimeout) - : Client(std::make_unique<NGrpc::TGRpcClientLow>()) - , NextTicketUpdate(TInstant::Zero()) - , TokenAccessorEndpoint(tokenAccessorEndpoint) - , ServiceAccountId(serviceAccountId) - , ServiceAccountIdSignature(serviceAccountIdSignature) - , RefreshPeriod(refreshPeriod) - , RequestTimeout(requestTimeout) - , Infly(0) - { - NGrpc::TGRpcClientConfig grpcConf; - grpcConf.Locator = tokenAccessorEndpoint; - grpcConf.EnableSsl = useSsl; - Connection = Client->CreateGRpcServiceConnection<TokenAccessorService>(grpcConf); - } - - void UpdateTicket(bool sync = false) const { - if (NeedStop || RequestInflight) { - return; - } - - RequestInflight = true; - auto resultPromise = NThreading::NewPromise(); - - std::shared_ptr<const TImpl> self = shared_from_this(); - auto cb = [self, resultPromise, sync](NGrpc::TGrpcStatus&& status, GetTokenResponse&& result) mutable { - self->ProcessResponse(std::move(status), std::move(result), sync); - resultPromise.SetValue(); - }; - - GetTokenRequest req; - req.set_type(GetTokenRequest::TYPE_SERVICE_ACCOUNT); - req.set_token_id(ServiceAccountId); - req.set_signature(ServiceAccountIdSignature); - with_lock(Lock) { - Infly++; - Connection->DoRequest<GetTokenRequest, GetTokenResponse>( - std::move(req), - std::move(cb), - &TokenAccessorService::Stub::AsyncGetToken, - { - {}, {}, RequestTimeout - } - ); - } - if (sync) { - resultPromise.GetFuture().Wait(RequestTimeout + TDuration::Seconds(10)); - } - } - - TString GetTicket() const { - TInstant nextTicketUpdate; - TString ticket; - with_lock(Lock) { - ticket = Ticket; - nextTicketUpdate = NextTicketUpdate; - if (ticket.empty()) { - ythrow yexception() << "IAM-token not ready yet. " << LastRequestError; - } - } - if (TInstant::Now() >= nextTicketUpdate) { - UpdateTicket(); - } - return ticket; - } - - void Stop() { - NeedStop = true; - - Client.reset(); // Will trigger destroy - } - - private: - void ProcessResponse(NGrpc::TGrpcStatus&& status, GetTokenResponse&& result, bool sync) const { - if (!status.Ok()) { - with_lock(Lock) { - --Infly; - LastRequestError = TStringBuilder() << "Last request error was at " << TInstant::Now() - << ". GrpcStatusCode: " << status.GRpcStatusCode << " Message: \"" << status.Msg - << "\" internal: " << status.InternalError << " token accessor endpoint: \"" << TokenAccessorEndpoint << "\""; - } - RequestInflight = false; - Sleep(std::min(BackoffTimeout, BACKOFF_MAX)); - BackoffTimeout *= 2; - UpdateTicket(sync); - } else { - with_lock(Lock) { - --Infly; - LastRequestError = ""; - Ticket = result.token(); - NextTicketUpdate = TInstant::Now() + RefreshPeriod - RequestTimeout; - } - RequestInflight = false; - BackoffTimeout = BACKOFF_START; - } - } - - private: - std::unique_ptr<NGrpc::TGRpcClientLow> Client; - std::unique_ptr<NGrpc::TServiceConnection<TokenAccessorService>> Connection; - mutable TString Ticket; - mutable TInstant NextTicketUpdate; - const TString TokenAccessorEndpoint; - const TString ServiceAccountId; - const TString ServiceAccountIdSignature; - const TDuration RefreshPeriod; - const TDuration RequestTimeout; - TAdaptiveLock Lock; - mutable NAtomic::TBool RequestInflight; - mutable TString LastRequestError; - NAtomic::TBool NeedStop = false; - mutable TDuration BackoffTimeout = BACKOFF_START; - mutable ui32 Infly; - }; - -public: - TTokenAccessorCredentialsProvider( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod, - const TDuration& requestTimeout - ) - : Impl(std::make_shared<TImpl>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout)) - { - Impl->UpdateTicket(true); - } - - ~TTokenAccessorCredentialsProvider() { - Impl->Stop(); - } - - TString GetAuthInfo() const override { - return Impl->GetTicket(); - } - - bool IsValid() const override { - return true; - } - -private: - std::shared_ptr<TImpl> Impl; -}; - -} - -std::shared_ptr<NYdb::ICredentialsProvider> CreateTokenAccessorCredentialsProvider( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod, - const TDuration& requestTimeout -) { - return std::make_shared<TTokenAccessorCredentialsProvider>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout); -} -} + +#include <library/cpp/grpc/client/grpc_client_low.h> +#include <library/cpp/threading/atomic/bool.h> +#include <library/cpp/threading/future/core/future.h> + +#include <util/string/builder.h> + +namespace NYql { + +namespace { + +const TDuration BACKOFF_START = TDuration::MilliSeconds(50); +const TDuration BACKOFF_MAX = TDuration::Seconds(10); + +class TTokenAccessorCredentialsProvider : public NYdb::ICredentialsProvider { +private: + class TImpl : public std::enable_shared_from_this<TImpl> { + public: + TImpl(const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod, + const TDuration& requestTimeout) + : Client(std::make_unique<NGrpc::TGRpcClientLow>()) + , NextTicketUpdate(TInstant::Zero()) + , TokenAccessorEndpoint(tokenAccessorEndpoint) + , ServiceAccountId(serviceAccountId) + , ServiceAccountIdSignature(serviceAccountIdSignature) + , RefreshPeriod(refreshPeriod) + , RequestTimeout(requestTimeout) + , Infly(0) + { + NGrpc::TGRpcClientConfig grpcConf; + grpcConf.Locator = tokenAccessorEndpoint; + grpcConf.EnableSsl = useSsl; + Connection = Client->CreateGRpcServiceConnection<TokenAccessorService>(grpcConf); + } + + void UpdateTicket(bool sync = false) const { + if (NeedStop || RequestInflight) { + return; + } + + RequestInflight = true; + auto resultPromise = NThreading::NewPromise(); + + std::shared_ptr<const TImpl> self = shared_from_this(); + auto cb = [self, resultPromise, sync](NGrpc::TGrpcStatus&& status, GetTokenResponse&& result) mutable { + self->ProcessResponse(std::move(status), std::move(result), sync); + resultPromise.SetValue(); + }; + + GetTokenRequest req; + req.set_type(GetTokenRequest::TYPE_SERVICE_ACCOUNT); + req.set_token_id(ServiceAccountId); + req.set_signature(ServiceAccountIdSignature); + with_lock(Lock) { + Infly++; + Connection->DoRequest<GetTokenRequest, GetTokenResponse>( + std::move(req), + std::move(cb), + &TokenAccessorService::Stub::AsyncGetToken, + { + {}, {}, RequestTimeout + } + ); + } + if (sync) { + resultPromise.GetFuture().Wait(RequestTimeout + TDuration::Seconds(10)); + } + } + + TString GetTicket() const { + TInstant nextTicketUpdate; + TString ticket; + with_lock(Lock) { + ticket = Ticket; + nextTicketUpdate = NextTicketUpdate; + if (ticket.empty()) { + ythrow yexception() << "IAM-token not ready yet. " << LastRequestError; + } + } + if (TInstant::Now() >= nextTicketUpdate) { + UpdateTicket(); + } + return ticket; + } + + void Stop() { + NeedStop = true; + + Client.reset(); // Will trigger destroy + } + + private: + void ProcessResponse(NGrpc::TGrpcStatus&& status, GetTokenResponse&& result, bool sync) const { + if (!status.Ok()) { + with_lock(Lock) { + --Infly; + LastRequestError = TStringBuilder() << "Last request error was at " << TInstant::Now() + << ". GrpcStatusCode: " << status.GRpcStatusCode << " Message: \"" << status.Msg + << "\" internal: " << status.InternalError << " token accessor endpoint: \"" << TokenAccessorEndpoint << "\""; + } + RequestInflight = false; + Sleep(std::min(BackoffTimeout, BACKOFF_MAX)); + BackoffTimeout *= 2; + UpdateTicket(sync); + } else { + with_lock(Lock) { + --Infly; + LastRequestError = ""; + Ticket = result.token(); + NextTicketUpdate = TInstant::Now() + RefreshPeriod - RequestTimeout; + } + RequestInflight = false; + BackoffTimeout = BACKOFF_START; + } + } + + private: + std::unique_ptr<NGrpc::TGRpcClientLow> Client; + std::unique_ptr<NGrpc::TServiceConnection<TokenAccessorService>> Connection; + mutable TString Ticket; + mutable TInstant NextTicketUpdate; + const TString TokenAccessorEndpoint; + const TString ServiceAccountId; + const TString ServiceAccountIdSignature; + const TDuration RefreshPeriod; + const TDuration RequestTimeout; + TAdaptiveLock Lock; + mutable NAtomic::TBool RequestInflight; + mutable TString LastRequestError; + NAtomic::TBool NeedStop = false; + mutable TDuration BackoffTimeout = BACKOFF_START; + mutable ui32 Infly; + }; + +public: + TTokenAccessorCredentialsProvider( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod, + const TDuration& requestTimeout + ) + : Impl(std::make_shared<TImpl>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout)) + { + Impl->UpdateTicket(true); + } + + ~TTokenAccessorCredentialsProvider() { + Impl->Stop(); + } + + TString GetAuthInfo() const override { + return Impl->GetTicket(); + } + + bool IsValid() const override { + return true; + } + +private: + std::shared_ptr<TImpl> Impl; +}; + +} + +std::shared_ptr<NYdb::ICredentialsProvider> CreateTokenAccessorCredentialsProvider( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod, + const TDuration& requestTimeout +) { + return std::make_shared<TTokenAccessorCredentialsProvider>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout); +} +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h index b8bcd10dbc..c38c665399 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> -#include <util/datetime/base.h> - -namespace NYql { - -std::shared_ptr<NYdb::ICredentialsProvider> CreateTokenAccessorCredentialsProvider( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod = TDuration::Hours(1), - const TDuration& requestTimeout = TDuration::Seconds(10) -); - -} +#include <util/datetime/base.h> + +namespace NYql { + +std::shared_ptr<NYdb::ICredentialsProvider> CreateTokenAccessorCredentialsProvider( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod = TDuration::Hours(1), + const TDuration& requestTimeout = TDuration::Seconds(10) +); + +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.cpp b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.cpp index b3105b220d..7103ee4a6e 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.cpp +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.cpp @@ -1,58 +1,58 @@ -#include "token_accessor_client_factory.h" -#include "token_accessor_client.h" - -#include <util/string/cast.h> - -namespace NYql { - -namespace { - -class TTokenAccessorCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { -public: - TTokenAccessorCredentialsProviderFactory( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod, - const TDuration& requestTimeout - ) - : TokenAccessorEndpoint(tokenAccessorEndpoint) - , UseSsl(useSsl) - , ServiceAccountId(serviceAccountId) - , ServiceAccountIdSignature(serviceAccountIdSignature) - , RefreshPeriod(refreshPeriod) - , RequestTimeout(requestTimeout) - { - } - - TString GetClientIdentity() const override { - return "TOKEN_ACCESSOR_CLIENT" + ToString((ui64)this); - } - - std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { - return CreateTokenAccessorCredentialsProvider(TokenAccessorEndpoint, UseSsl, ServiceAccountId, ServiceAccountIdSignature, RefreshPeriod, RequestTimeout); - } - -private: - const TString TokenAccessorEndpoint; - const bool UseSsl; - const TString ServiceAccountId; - const TString ServiceAccountIdSignature; - const TDuration RefreshPeriod; - const TDuration RequestTimeout; -}; - -} - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateTokenAccessorCredentialsProviderFactory( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod, - const TDuration& requestTimeout) -{ - return std::make_shared<TTokenAccessorCredentialsProviderFactory>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout); -} -} +#include "token_accessor_client_factory.h" +#include "token_accessor_client.h" + +#include <util/string/cast.h> + +namespace NYql { + +namespace { + +class TTokenAccessorCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { +public: + TTokenAccessorCredentialsProviderFactory( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod, + const TDuration& requestTimeout + ) + : TokenAccessorEndpoint(tokenAccessorEndpoint) + , UseSsl(useSsl) + , ServiceAccountId(serviceAccountId) + , ServiceAccountIdSignature(serviceAccountIdSignature) + , RefreshPeriod(refreshPeriod) + , RequestTimeout(requestTimeout) + { + } + + TString GetClientIdentity() const override { + return "TOKEN_ACCESSOR_CLIENT" + ToString((ui64)this); + } + + std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { + return CreateTokenAccessorCredentialsProvider(TokenAccessorEndpoint, UseSsl, ServiceAccountId, ServiceAccountIdSignature, RefreshPeriod, RequestTimeout); + } + +private: + const TString TokenAccessorEndpoint; + const bool UseSsl; + const TString ServiceAccountId; + const TString ServiceAccountIdSignature; + const TDuration RefreshPeriod; + const TDuration RequestTimeout; +}; + +} + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateTokenAccessorCredentialsProviderFactory( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod, + const TDuration& requestTimeout) +{ + return std::make_shared<TTokenAccessorCredentialsProviderFactory>(tokenAccessorEndpoint, useSsl, serviceAccountId, serviceAccountIdSignature, refreshPeriod, requestTimeout); +} +} diff --git a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h index 7a9b4a9327..b73f9bf03d 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h +++ b/ydb/library/yql/providers/common/token_accessor/client/token_accessor_client_factory.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> -#include <util/datetime/base.h> - -namespace NYql { - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateTokenAccessorCredentialsProviderFactory( - const TString& tokenAccessorEndpoint, - bool useSsl, - const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - const TDuration& refreshPeriod = TDuration::Hours(1), - const TDuration& requestTimeout = TDuration::Seconds(10) -); - -} - +#include <util/datetime/base.h> + +namespace NYql { + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateTokenAccessorCredentialsProviderFactory( + const TString& tokenAccessorEndpoint, + bool useSsl, + const TString& serviceAccountId, + const TString& serviceAccountIdSignature, + const TDuration& refreshPeriod = TDuration::Hours(1), + const TDuration& requestTimeout = TDuration::Seconds(10) +); + +} + diff --git a/ydb/library/yql/providers/common/token_accessor/client/ya.make b/ydb/library/yql/providers/common/token_accessor/client/ya.make index 99338525cf..c05860d49e 100644 --- a/ydb/library/yql/providers/common/token_accessor/client/ya.make +++ b/ydb/library/yql/providers/common/token_accessor/client/ya.make @@ -1,20 +1,20 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - bearer_credentials_provider.cpp - factory.cpp - token_accessor_client.cpp - token_accessor_client_factory.cpp -) - -PEERDIR( - library/cpp/grpc/client - library/cpp/threading/future + +LIBRARY() + +SRCS( + bearer_credentials_provider.cpp + factory.cpp + token_accessor_client.cpp + token_accessor_client_factory.cpp +) + +PEERDIR( + library/cpp/grpc/client + library/cpp/threading/future ydb/library/yql/providers/common/structured_token ydb/library/yql/providers/common/token_accessor/grpc ydb/public/sdk/cpp/client/ydb_types/credentials -) - -END() +) + +END() diff --git a/ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.proto b/ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.proto index ff2ae3e056..5ab2b24a1b 100644 --- a/ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.proto +++ b/ydb/library/yql/providers/common/token_accessor/grpc/token_accessor_pb.proto @@ -1,24 +1,24 @@ -syntax = "proto3"; - -package NYql; - +syntax = "proto3"; + +package NYql; + option go_package = "a.yandex-team.ru/ydb/library/yql/providers/common/token_accessor/grpc;token_accessor_pb"; - -service TokenAccessorService { - rpc GetToken(GetTokenRequest) returns (GetTokenResponse) {} -} - -message GetTokenRequest { - enum Type { - TYPE_UNKNOWN = 0; - TYPE_SERVICE_ACCOUNT = 1; - } - - Type type = 1; - string token_id = 2; - string signature = 3; -} - -message GetTokenResponse { - bytes token = 3; -} + +service TokenAccessorService { + rpc GetToken(GetTokenRequest) returns (GetTokenResponse) {} +} + +message GetTokenRequest { + enum Type { + TYPE_UNKNOWN = 0; + TYPE_SERVICE_ACCOUNT = 1; + } + + Type type = 1; + string token_id = 2; + string signature = 3; +} + +message GetTokenResponse { + bytes token = 3; +} diff --git a/ydb/library/yql/providers/common/token_accessor/grpc/ya.make b/ydb/library/yql/providers/common/token_accessor/grpc/ya.make index 2297218e2b..3a1d6d4024 100644 --- a/ydb/library/yql/providers/common/token_accessor/grpc/ya.make +++ b/ydb/library/yql/providers/common/token_accessor/grpc/ya.make @@ -1,13 +1,13 @@ OWNER(g:yq) -PROTO_LIBRARY() +PROTO_LIBRARY() -INCLUDE_TAGS(GO_PROTO) +INCLUDE_TAGS(GO_PROTO) + +GRPC() -GRPC() - -SRCS( - token_accessor_pb.proto -) - -END() +SRCS( + token_accessor_pb.proto +) + +END() diff --git a/ydb/library/yql/providers/common/token_accessor/ya.make b/ydb/library/yql/providers/common/token_accessor/ya.make index 119cbb9c4b..bded9e3ea9 100644 --- a/ydb/library/yql/providers/common/token_accessor/ya.make +++ b/ydb/library/yql/providers/common/token_accessor/ya.make @@ -1,4 +1,4 @@ -RECURSE( - client - grpc -) +RECURSE( + client + grpc +) diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp index 95fe68f080..c3e798a874 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp @@ -1,39 +1,39 @@ -#include "yql_files_box.h" - +#include "yql_files_box.h" + #include <ydb/library/yql/core/file_storage/storage.h> #include <ydb/library/yql/utils/log/log.h> -#include <util/system/fs.h> +#include <util/system/fs.h> #include <util/system/error.h> #include <util/system/sysstat.h> #include <util/folder/dirut.h> - -namespace NYql { -namespace NCommon { - -TFilesBox::TFilesBox(TFsPath dir, TRandGuid randGuid) - : Dir(std::move(dir)) - , RandGuid(std::move(randGuid)) -{ -} - -TFilesBox::~TFilesBox() { - try { - Destroy(); - } catch (...) { - YQL_LOG(ERROR) << "Error occurred in files box destroy: " << CurrentExceptionMessage(); - } -} - -TString TFilesBox::MakeLinkFrom(const TString& source, const TString& filename) { - if (!filename) { - if (auto* existingPath = Mapping.FindPtr(source)) { - return *existingPath; - } - } - - TFsPath sourcePath(source); - TString sourceAbsolutePath = sourcePath.IsAbsolute() ? source : (TFsPath::Cwd() / sourcePath).GetPath(); + +namespace NYql { +namespace NCommon { + +TFilesBox::TFilesBox(TFsPath dir, TRandGuid randGuid) + : Dir(std::move(dir)) + , RandGuid(std::move(randGuid)) +{ +} + +TFilesBox::~TFilesBox() { + try { + Destroy(); + } catch (...) { + YQL_LOG(ERROR) << "Error occurred in files box destroy: " << CurrentExceptionMessage(); + } +} + +TString TFilesBox::MakeLinkFrom(const TString& source, const TString& filename) { + if (!filename) { + if (auto* existingPath = Mapping.FindPtr(source)) { + return *existingPath; + } + } + + TFsPath sourcePath(source); + TString sourceAbsolutePath = sourcePath.IsAbsolute() ? source : (TFsPath::Cwd() / sourcePath).GetPath(); TString path; if (filename) { path = Dir / filename; @@ -49,24 +49,24 @@ TString TFilesBox::MakeLinkFrom(const TString& source, const TString& filename) ythrow TSystemError() << "Failed to create symlink for file " << sourceAbsolutePath.Quote() << " to file " << path.Quote(); } } - Mapping.emplace(source, path); - } - return path; -} - -TString TFilesBox::GetDir() const { - return Dir; -} - -void TFilesBox::Destroy() { - Mapping.clear(); - Dir.ForceDelete(); -} - -THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir) { - TRandGuid randGuid; - TFsPath path = baseDir / randGuid.GenGuid(); - + Mapping.emplace(source, path); + } + return path; +} + +TString TFilesBox::GetDir() const { + return Dir; +} + +void TFilesBox::Destroy() { + Mapping.clear(); + Dir.ForceDelete(); +} + +THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir) { + TRandGuid randGuid; + TFsPath path = baseDir / randGuid.GenGuid(); + while (true) { if (!path.Exists()) { int r = Mkdir(path.c_str(), MODE0711); @@ -80,8 +80,8 @@ THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir) { path = baseDir / randGuid.GenGuid(); } - return MakeHolder<TFilesBox>(std::move(path), std::move(randGuid)); -} - -} -} + return MakeHolder<TFilesBox>(std::move(path), std::move(randGuid)); +} + +} +} diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.h b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.h index fd7b778ab4..64b4f67832 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.h +++ b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/utils/rand_guid.h> - -#include <util/generic/hash.h> -#include <util/folder/path.h> - -namespace NYql { -namespace NCommon { - -/* - Resembles sandbox for external UDFs -*/ -class TFilesBox { -public: - TFilesBox(TFsPath dir, TRandGuid randGuid); - ~TFilesBox(); - - TString MakeLinkFrom(const TString& source, const TString& filename = {}); - TString GetDir() const; - - void Destroy(); - -private: - TFsPath Dir; - TRandGuid RandGuid; - THashMap<TString, TString> Mapping; -}; - -THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir); - -} -} + +#include <util/generic/hash.h> +#include <util/folder/path.h> + +namespace NYql { +namespace NCommon { + +/* + Resembles sandbox for external UDFs +*/ +class TFilesBox { +public: + TFilesBox(TFsPath dir, TRandGuid randGuid); + ~TFilesBox(); + + TString MakeLinkFrom(const TString& source, const TString& filename = {}); + TString GetDir() const; + + void Destroy(); + +private: + TFsPath Dir; + TRandGuid RandGuid; + THashMap<TString, TString> Mapping; +}; + +THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir); + +} +} diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index dc31e0ef30..1e23aa00e9 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -1,6 +1,6 @@ #include "yql_outproc_udf_resolver.h" -#include "yql_simple_udf_resolver.h" -#include "yql_files_box.h" +#include "yql_simple_udf_resolver.h" +#include "yql_files_box.h" #include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> #include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h> @@ -16,13 +16,13 @@ #include <library/cpp/protobuf/util/pb_io.h> -#include <util/generic/scope.h> +#include <util/generic/scope.h> #include <util/stream/str.h> #include <util/string/strip.h> #include <util/system/shellcommand.h> #include <util/string/split.h> -#include <regex> +#include <regex> namespace NYql { namespace NCommon { @@ -30,151 +30,151 @@ namespace NCommon { using namespace NKikimr; using namespace NKikimr::NMiniKQL; -namespace { -template <typename F> -void RunResolver( - const TString& resolverPath, - const TList<TString>& args, - IInputStream* input, - const F& outputHandler, - const TString& ldLibraryPath = {}) { - - TShellCommandOptions shellOptions; - shellOptions - .SetUseShell(false) - .SetDetachSession(false) - .SetInputStream(input); // input can be nullptr - - if (ldLibraryPath) { - YQL_LOG(DEBUG) << "Using LD_LIBRARY_PATH = " << ldLibraryPath << " for Udf resolver"; - shellOptions.Environment["LD_LIBRARY_PATH"] = ldLibraryPath; - } - - TShellCommand shell(resolverPath, args, shellOptions); - - switch (shell.Run().GetStatus()) { - case TShellCommand::SHELL_INTERNAL_ERROR: - ythrow yexception() << "Udf resolver internal error: " - << shell.GetInternalError(); - case TShellCommand::SHELL_ERROR: - ythrow yexception() << "Udf resolver shell error: " - << StripString(shell.GetError()); - case TShellCommand::SHELL_FINISHED: - break; - default: - ythrow yexception() << "Unexpected udf resolver state: " - << int(shell.GetStatus()); - } - - if (shell.GetError()) { - YQL_LOG(INFO) << "UdfResolver stderr: " << shell.GetError(); - } - - outputHandler(shell.GetOutput()); -} - -template <typename F> -void RunResolver( - const TString& resolverPath, - const TList<TString>& args, - const TResolve& request, - const F& outputHandler, - const TString& ldLibraryPath = {}) { - - TStringStream input; +namespace { +template <typename F> +void RunResolver( + const TString& resolverPath, + const TList<TString>& args, + IInputStream* input, + const F& outputHandler, + const TString& ldLibraryPath = {}) { + + TShellCommandOptions shellOptions; + shellOptions + .SetUseShell(false) + .SetDetachSession(false) + .SetInputStream(input); // input can be nullptr + + if (ldLibraryPath) { + YQL_LOG(DEBUG) << "Using LD_LIBRARY_PATH = " << ldLibraryPath << " for Udf resolver"; + shellOptions.Environment["LD_LIBRARY_PATH"] = ldLibraryPath; + } + + TShellCommand shell(resolverPath, args, shellOptions); + + switch (shell.Run().GetStatus()) { + case TShellCommand::SHELL_INTERNAL_ERROR: + ythrow yexception() << "Udf resolver internal error: " + << shell.GetInternalError(); + case TShellCommand::SHELL_ERROR: + ythrow yexception() << "Udf resolver shell error: " + << StripString(shell.GetError()); + case TShellCommand::SHELL_FINISHED: + break; + default: + ythrow yexception() << "Unexpected udf resolver state: " + << int(shell.GetStatus()); + } + + if (shell.GetError()) { + YQL_LOG(INFO) << "UdfResolver stderr: " << shell.GetError(); + } + + outputHandler(shell.GetOutput()); +} + +template <typename F> +void RunResolver( + const TString& resolverPath, + const TList<TString>& args, + const TResolve& request, + const F& outputHandler, + const TString& ldLibraryPath = {}) { + + TStringStream input; YQL_ENSURE(request.SerializeToArcadiaStream(&input), "Cannot serialize TResolve proto message"); - RunResolver(resolverPath, args, &input, outputHandler, ldLibraryPath); -} - -TString ExtractSharedObjectNameFromErrorMessage(const char* message) { - if (!message) { - return ""; - } - - // example: - // util/system/dynlib.cpp:56: libcuda.so.1: cannot open shared object file: No such file or directory - static std::regex re(".*: (.+): cannot open shared object file: No such file or directory"); - std::cmatch match; - if (!std::regex_match(message, match, re)) { - return ""; - } - - return TString(match[1].str()); -} -} - + RunResolver(resolverPath, args, &input, outputHandler, ldLibraryPath); +} + +TString ExtractSharedObjectNameFromErrorMessage(const char* message) { + if (!message) { + return ""; + } + + // example: + // util/system/dynlib.cpp:56: libcuda.so.1: cannot open shared object file: No such file or directory + static std::regex re(".*: (.+): cannot open shared object file: No such file or directory"); + std::cmatch match; + if (!std::regex_match(message, match, re)) { + return ""; + } + + return TString(match[1].str()); +} +} + class TOutProcUdfResolver : public IUdfResolver { public: TOutProcUdfResolver(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const TFileStoragePtr& fileStorage, const TString& resolverPath, + const TFileStoragePtr& fileStorage, const TString& resolverPath, const TString& user, const TString& group, bool filterSyscalls, const TString& udfDependencyStubPath, const TMap<TString, TString>& path2md5) : FunctionRegistry_(functionRegistry) - , TypeInfoHelper_(new TTypeInfoHelper) + , TypeInfoHelper_(new TTypeInfoHelper) , FileStorage_(fileStorage) , ResolverPath_(resolverPath) - , UdfDependencyStubPath_(udfDependencyStubPath) - , Path2Md5_(path2md5) - { - if (user) { - UserGroupArgs_ = { "-U", user, "-G", group }; - } + , UdfDependencyStubPath_(udfDependencyStubPath) + , Path2Md5_(path2md5) + { + if (user) { + UserGroupArgs_ = { "-U", user, "-G", group }; + } if (filterSyscalls) { UserGroupArgs_.push_back("-F"); } + } + + TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { + auto path = FunctionRegistry_->FindUdfPath(moduleName); + if (!path) { + return Nothing(); + } + + const TString md5 = Path2Md5_.Value(*path, ""); + return MakeMaybe<TFilePathWithMd5>(*path, md5); } - TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { - auto path = FunctionRegistry_->FindUdfPath(moduleName); - if (!path) { - return Nothing(); - } - - const TString md5 = Path2Md5_.Value(*path, ""); - return MakeMaybe<TFilePathWithMd5>(*path, md5); - } - - bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override { - THashSet<TString> requiredLoadedModules; - THashSet<TString> requiredExternalModules; - TVector<TFunction*> loadedFunctions; - TVector<TFunction*> externalFunctions; - + bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override { + THashSet<TString> requiredLoadedModules; + THashSet<TString> requiredExternalModules; + TVector<TFunction*> loadedFunctions; + TVector<TFunction*> externalFunctions; + bool hasErrors = false; - for (auto udf : functions) { + for (auto udf : functions) { TStringBuf moduleName, funcName; if (!SplitUdfName(udf->Name, moduleName, funcName) || moduleName.empty() || funcName.empty()) { ctx.AddError(TIssue(udf->Pos, TStringBuilder() << "Incorrect format of function name: " << udf->Name)); hasErrors = true; } else { - if (FunctionRegistry_->IsLoadedUdfModule(moduleName)) { - requiredLoadedModules.insert(TString(moduleName)); - loadedFunctions.push_back(udf); - } else { - requiredExternalModules.insert(TString(moduleName)); - externalFunctions.push_back(udf); - } + if (FunctionRegistry_->IsLoadedUdfModule(moduleName)) { + requiredLoadedModules.insert(TString(moduleName)); + loadedFunctions.push_back(udf); + } else { + requiredExternalModules.insert(TString(moduleName)); + externalFunctions.push_back(udf); + } } } TResolve request; THashMap<TString, TImport*> importMap; - THoldingFileStorage holdingFileStorage(FileStorage_); - THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp(); + THoldingFileStorage holdingFileStorage(FileStorage_); + THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp(); - THashMap<TString, TImport*> path2LoadedImport; - for (auto import : imports) { + THashMap<TString, TImport*> path2LoadedImport; + for (auto import : imports) { if (import->Modules) { bool needLibrary = false; for (auto& m : *import->Modules) { - if (requiredLoadedModules.contains(m)) { - YQL_ENSURE(import->Block->Type == EUserDataType::PATH); - path2LoadedImport[import->Block->Data] = import; - } - - if (requiredExternalModules.contains(m)) { + if (requiredLoadedModules.contains(m)) { + YQL_ENSURE(import->Block->Type == EUserDataType::PATH); + path2LoadedImport[import->Block->Data] = import; + } + + if (requiredExternalModules.contains(m)) { needLibrary = true; break; } @@ -183,32 +183,32 @@ public: if (!needLibrary) { continue; } - } else { - import->Modules.ConstructInPlace(); + } else { + import->Modules.ConstructInPlace(); } importMap[import->FileAlias] = import; try { - LoadImport(holdingFileStorage, *filesBox, *import, request); + LoadImport(holdingFileStorage, *filesBox, *import, request); } catch (const std::exception& e) { ctx.AddError(ExceptionToIssue(e)); hasErrors = true; } } - for (auto& module : requiredExternalModules) { + for (auto& module : requiredExternalModules) { if (auto path = FunctionRegistry_->FindUdfPath(module)) { auto importRequest = request.AddImports(); - const TString hiddenPath = filesBox->MakeLinkFrom(*path); - importRequest->SetFileAlias(hiddenPath); - importRequest->SetPath(hiddenPath); + const TString hiddenPath = filesBox->MakeLinkFrom(*path); + importRequest->SetFileAlias(hiddenPath); + importRequest->SetPath(hiddenPath); importRequest->SetSystem(true); } } - - for (auto udf : externalFunctions) { + + for (auto udf : externalFunctions) { auto udfRequest = request.AddUdfs(); udfRequest->SetName(udf->Name); udfRequest->SetTypeConfig(udf->TypeConfig); @@ -217,106 +217,106 @@ public: } } - TResolveResult response; - try { - response = RunResolverAndParseResult(request, { }, *filesBox); - filesBox->Destroy(); - } catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e)); + TResolveResult response; + try { + response = RunResolverAndParseResult(request, { }, *filesBox); + filesBox->Destroy(); + } catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e)); return false; } - - // extract regardless of hasErrors value - hasErrors = !ExtractMetadata(response, importMap, externalFunctions, ctx) || hasErrors; - hasErrors = !LoadFunctionsMetadata(loadedFunctions, *FunctionRegistry_, TypeInfoHelper_, ctx) || hasErrors; - - if (!hasErrors) { - for (auto& m : FunctionRegistry_->GetAllModuleNames()) { - auto path = *FunctionRegistry_->FindUdfPath(m); - if (auto import = path2LoadedImport.FindPtr(path)) { - (*import)->Modules->push_back(m); - } - } - } - - return !hasErrors; - } - - TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { - TResolve request; - THoldingFileStorage holdingFileStorage(FileStorage_); - THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp(); - Y_DEFER { - filesBox->Destroy(); - }; - - for (auto import : imports) { - LoadImport(holdingFileStorage, *filesBox, import, request); - } - - return RunResolverAndParseResult(request, { "--discover-proto" }, *filesBox); - } - -private: - THolder<TFilesBox> CreateFilesBoxOverFileStorageTemp() const { - return CreateFilesBox(FileStorage_->GetTemp()); - } - - void LoadImport(THoldingFileStorage& holdingFileStorage, TFilesBox& filesBox, const TImport& import, TResolve& request) const { - const TString path = (import.Block->Type == EUserDataType::PATH) ? import.Block->Data : holdingFileStorage.FreezeFile(*import.Block)->GetPath().GetPath(); - - const TString hiddenPath = filesBox.MakeLinkFrom(path); - auto importRequest = request.AddImports(); - importRequest->SetFileAlias(import.FileAlias); - importRequest->SetPath(hiddenPath); - } - - TResolveResult RunResolverAndParseResult(const TResolve& request, const TVector<TString>& additionalArgs, TFilesBox& filesBox) const { - auto args = UserGroupArgs_; - args.insert(args.end(), additionalArgs.begin(), additionalArgs.end()); - - TString ldLibraryPath; - TSet<TString> stubbedLibraries; - return WithRetry<yexception>(10, [&]() { - TResolveResult response; - RunResolver(ResolverPath_, args, request, [&](const TString& output) { - YQL_ENSURE(response.ParseFromString(output), "Cannot deserialize TResolveResult proto message"); - }, ldLibraryPath); - return response; - }, [&](const yexception& e, int, int) { - TStringStream stream; - SerializeToTextFormat(request, stream); - YQL_LOG(DEBUG) << "Exception from UdfResolver: " << e.what() << " for request " << stream.Str(); - if (!UdfDependencyStubPath_) { - YQL_LOG(DEBUG) << "UdfDependencyStubPath is not specified, unable to recover error " << e.what(); - throw e; - } - - TString sharedLibrary = ExtractSharedObjectNameFromErrorMessage(e.what()); - if (!sharedLibrary) { - throw e; - } - - YQL_LOG(DEBUG) << "UdfResolver needs shared library " << sharedLibrary; - if (!stubbedLibraries.emplace(sharedLibrary).second) { - // already tried, giving up - YQL_LOG(ERROR) << "Unable to load shared library " << sharedLibrary << " even after using dependency stub"; - throw e; - } - - YQL_LOG(DEBUG) << "Using dependency stub for shared library " << sharedLibrary; - PutSharedLibraryStub(sharedLibrary, filesBox); - ldLibraryPath = filesBox.GetDir(); - }); - } - - void PutSharedLibraryStub(const TString& sharedLibrary, TFilesBox& filesBox) const { - YQL_ENSURE(UdfDependencyStubPath_); - filesBox.MakeLinkFrom(UdfDependencyStubPath_, sharedLibrary); - } - - static bool ExtractMetadata(const TResolveResult& response, const THashMap<TString, TImport*>& importMap, const TVector<TFunction*>& functions, TExprContext& ctx) { - bool hasErrors = false; + + // extract regardless of hasErrors value + hasErrors = !ExtractMetadata(response, importMap, externalFunctions, ctx) || hasErrors; + hasErrors = !LoadFunctionsMetadata(loadedFunctions, *FunctionRegistry_, TypeInfoHelper_, ctx) || hasErrors; + + if (!hasErrors) { + for (auto& m : FunctionRegistry_->GetAllModuleNames()) { + auto path = *FunctionRegistry_->FindUdfPath(m); + if (auto import = path2LoadedImport.FindPtr(path)) { + (*import)->Modules->push_back(m); + } + } + } + + return !hasErrors; + } + + TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { + TResolve request; + THoldingFileStorage holdingFileStorage(FileStorage_); + THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp(); + Y_DEFER { + filesBox->Destroy(); + }; + + for (auto import : imports) { + LoadImport(holdingFileStorage, *filesBox, import, request); + } + + return RunResolverAndParseResult(request, { "--discover-proto" }, *filesBox); + } + +private: + THolder<TFilesBox> CreateFilesBoxOverFileStorageTemp() const { + return CreateFilesBox(FileStorage_->GetTemp()); + } + + void LoadImport(THoldingFileStorage& holdingFileStorage, TFilesBox& filesBox, const TImport& import, TResolve& request) const { + const TString path = (import.Block->Type == EUserDataType::PATH) ? import.Block->Data : holdingFileStorage.FreezeFile(*import.Block)->GetPath().GetPath(); + + const TString hiddenPath = filesBox.MakeLinkFrom(path); + auto importRequest = request.AddImports(); + importRequest->SetFileAlias(import.FileAlias); + importRequest->SetPath(hiddenPath); + } + + TResolveResult RunResolverAndParseResult(const TResolve& request, const TVector<TString>& additionalArgs, TFilesBox& filesBox) const { + auto args = UserGroupArgs_; + args.insert(args.end(), additionalArgs.begin(), additionalArgs.end()); + + TString ldLibraryPath; + TSet<TString> stubbedLibraries; + return WithRetry<yexception>(10, [&]() { + TResolveResult response; + RunResolver(ResolverPath_, args, request, [&](const TString& output) { + YQL_ENSURE(response.ParseFromString(output), "Cannot deserialize TResolveResult proto message"); + }, ldLibraryPath); + return response; + }, [&](const yexception& e, int, int) { + TStringStream stream; + SerializeToTextFormat(request, stream); + YQL_LOG(DEBUG) << "Exception from UdfResolver: " << e.what() << " for request " << stream.Str(); + if (!UdfDependencyStubPath_) { + YQL_LOG(DEBUG) << "UdfDependencyStubPath is not specified, unable to recover error " << e.what(); + throw e; + } + + TString sharedLibrary = ExtractSharedObjectNameFromErrorMessage(e.what()); + if (!sharedLibrary) { + throw e; + } + + YQL_LOG(DEBUG) << "UdfResolver needs shared library " << sharedLibrary; + if (!stubbedLibraries.emplace(sharedLibrary).second) { + // already tried, giving up + YQL_LOG(ERROR) << "Unable to load shared library " << sharedLibrary << " even after using dependency stub"; + throw e; + } + + YQL_LOG(DEBUG) << "Using dependency stub for shared library " << sharedLibrary; + PutSharedLibraryStub(sharedLibrary, filesBox); + ldLibraryPath = filesBox.GetDir(); + }); + } + + void PutSharedLibraryStub(const TString& sharedLibrary, TFilesBox& filesBox) const { + YQL_ENSURE(UdfDependencyStubPath_); + filesBox.MakeLinkFrom(UdfDependencyStubPath_, sharedLibrary); + } + + static bool ExtractMetadata(const TResolveResult& response, const THashMap<TString, TImport*>& importMap, const TVector<TFunction*>& functions, TExprContext& ctx) { + bool hasErrors = false; YQL_ENSURE(response.UdfsSize() == functions.size(), "Number of returned udf signatures doesn't match original one"); for (size_t i = 0; i < response.ImportsSize(); ++i) { @@ -331,7 +331,7 @@ private: hasErrors = true; } else if (import) { import->Modules.ConstructInPlace(); - for (auto& module : importRes.GetModules()) { + for (auto& module : importRes.GetModules()) { import->Modules->push_back(module); } } @@ -371,12 +371,12 @@ private: private: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; - NUdf::ITypeInfoHelper::TPtr TypeInfoHelper_; + NUdf::ITypeInfoHelper::TPtr TypeInfoHelper_; TFileStoragePtr FileStorage_; - const TString ResolverPath_; - const TString UdfDependencyStubPath_; - TList<TString> UserGroupArgs_; - const TMap<TString, TString> Path2Md5_; + const TString ResolverPath_; + const TString UdfDependencyStubPath_; + TList<TString> UserGroupArgs_; + const TMap<TString, TString> Path2Md5_; }; void LoadSystemModulePaths( @@ -384,31 +384,31 @@ void LoadSystemModulePaths( const TString& dir, TUdfModulePathsMap* paths) { - const TList<TString> args = { TString("--list"), dir }; - RunResolver(resolverPath, args, nullptr, [&](const TString& output) { - // output format is: - // {{module_name}}\t{{module_path}}\n - - for (const auto& it : StringSplitter(output).Split('\n')) { - TStringBuf moduleName, modulePath; - const TStringBuf& line = it.Token(); + const TList<TString> args = { TString("--list"), dir }; + RunResolver(resolverPath, args, nullptr, [&](const TString& output) { + // output format is: + // {{module_name}}\t{{module_path}}\n + + for (const auto& it : StringSplitter(output).Split('\n')) { + TStringBuf moduleName, modulePath; + const TStringBuf& line = it.Token(); if (!line.empty()) { - line.Split('\t', moduleName, modulePath); - paths->emplace(moduleName, modulePath); - } + line.Split('\t', moduleName, modulePath); + paths->emplace(moduleName, modulePath); + } } - }); + }); } IUdfResolver::TPtr CreateOutProcUdfResolver( const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, const TFileStoragePtr& fileStorage, - const TString& resolverPath, - const TString& user, - const TString& group, + const TString& resolverPath, + const TString& user, + const TString& group, bool filterSyscalls, - const TString& udfDependencyStubPath, - const TMap<TString, TString>& path2md5) { + const TString& udfDependencyStubPath, + const TMap<TString, TString>& path2md5) { return new TOutProcUdfResolver(functionRegistry, fileStorage, resolverPath, user, group, filterSyscalls, udfDependencyStubPath, path2md5); } diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h index 5659a833e4..58a0d97e56 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h +++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h @@ -5,7 +5,7 @@ #include <ydb/library/yql/minikql/mkql_function_registry.h> #include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> -#include <util/generic/map.h> +#include <util/generic/map.h> #include <util/generic/string.h> namespace NYql { @@ -19,12 +19,12 @@ void LoadSystemModulePaths( IUdfResolver::TPtr CreateOutProcUdfResolver( const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, const TFileStoragePtr& fileStorage, - const TString& resolverPath, - const TString& user, - const TString& group, + const TString& resolverPath, + const TString& user, + const TString& group, bool filterSysCalls, - const TString& udfDependencyStubPath, - const TMap<TString, TString>& path2md5 = {}); + const TString& udfDependencyStubPath, + const TMap<TString, TString>& path2md5 = {}); } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index 3aac3487c0..c2138ca5d9 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -2,7 +2,7 @@ #include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h> #include <ydb/library/yql/core/yql_holding_file_storage.h> - + #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/minikql/mkql_program_builder.h> @@ -31,16 +31,16 @@ public: , UseFakeMD5_(useFakeMD5) {} - TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { + TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { with_lock(Lock_) { - auto path = FunctionRegistry_->FindUdfPath(moduleName); + auto path = FunctionRegistry_->FindUdfPath(moduleName); return path ? MakeMaybe<TFilePathWithMd5>(*path, UseFakeMD5_ ? *path : TString()) : Nothing(); } } - bool LoadMetadata(const TVector<TImport*>& imports, - const TVector<TFunction*>& functions, TExprContext& ctx) const override { - + bool LoadMetadata(const TVector<TImport*>& imports, + const TVector<TFunction*>& functions, TExprContext& ctx) const override { + with_lock(Lock_) { bool hasErrors = false; THashSet<TString> requiredModules; @@ -56,7 +56,7 @@ public: } } - THoldingFileStorage holdingFileStorage(FileStorage_); + THoldingFileStorage holdingFileStorage(FileStorage_); auto newRegistry = FunctionRegistry_->Clone(); THashMap<TString, TImport*> path2import; for (auto import: imports) { @@ -78,10 +78,10 @@ public: try { if (FileStorage_) { - auto link = holdingFileStorage.FreezeFile(*import->Block); - auto path = link->GetPath().GetPath(); - newRegistry->LoadUdfs(path, {}, NUdf::IRegistrator::TFlags::TypesOnly); - path2import[path] = import; + auto link = holdingFileStorage.FreezeFile(*import->Block); + auto path = link->GetPath().GetPath(); + newRegistry->LoadUdfs(path, {}, NUdf::IRegistrator::TFlags::TypesOnly); + path2import[path] = import; } else { if (import->Block->Type != EUserDataType::PATH) { ctx.AddError(TIssue(import->Pos, TStringBuilder() << @@ -110,16 +110,16 @@ public: } } - hasErrors = !LoadFunctionsMetadata(functions, *newRegistry, TypeInfoHelper_, ctx) || hasErrors; + hasErrors = !LoadFunctionsMetadata(functions, *newRegistry, TypeInfoHelper_, ctx) || hasErrors; return !hasErrors; } } - TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { - Y_UNUSED(imports); - ythrow yexception() << "LoadRichMetadata is not supported in SimpleUdfResolver"; - } - + TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { + Y_UNUSED(imports); + ythrow yexception() << "LoadRichMetadata is not supported in SimpleUdfResolver"; + } + private: mutable TAdaptiveLock Lock_; const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; @@ -136,64 +136,64 @@ IUdfResolver::TPtr CreateSimpleUdfResolver( return new TSimpleUdfResolver(functionRegistry, fileStorage, useFakeMD5); } -bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, - const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - TExprContext& ctx) { - - bool hasErrors = false; - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - - for (auto udfPtr : functions) { - auto& udf = *udfPtr; - try { - TProgramBuilder pgmBuilder(env, functionRegistry); - TType* mkqlUserType = nullptr; - if (udf.UserType) { - TStringStream err; +bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, + const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + TExprContext& ctx) { + + bool hasErrors = false; + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + + for (auto udfPtr : functions) { + auto& udf = *udfPtr; + try { + TProgramBuilder pgmBuilder(env, functionRegistry); + TType* mkqlUserType = nullptr; + if (udf.UserType) { + TStringStream err; mkqlUserType = BuildType(*udf.UserType, pgmBuilder, err, true); - if (!mkqlUserType) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Invalid user type for function: " - << udf.Name << ", error: " << err.Str())); - hasErrors = true; - continue; - } - } - + if (!mkqlUserType) { + ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Invalid user type for function: " + << udf.Name << ", error: " << err.Str())); + hasErrors = true; + continue; + } + } + auto secureParamsProvider = MakeSimpleSecureParamsProvider(udf.SecureParams); - TFunctionTypeInfo funcInfo; - auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, nullptr, + TFunctionTypeInfo funcInfo; + auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udf.Name, mkqlUserType, udf.TypeConfig, NUdf::IUdfModule::TFlags::TypesOnly, {}, secureParamsProvider.get(), &funcInfo); - if (!status.IsOk()) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Failed to find UDF function: " << udf.Name - << ", reason: " << status.GetError())); - hasErrors = true; - continue; - } - - udf.CallableType = ConvertMiniKQLType(udf.Pos, funcInfo.FunctionType, ctx); - YQL_ENSURE(udf.CallableType); - if (funcInfo.RunConfigType) { - udf.RunConfigType = ConvertMiniKQLType(udf.Pos, const_cast<TType*>(funcInfo.RunConfigType), ctx); - YQL_ENSURE(udf.RunConfigType); - } - - if (funcInfo.UserType) { - udf.NormalizedUserType = ConvertMiniKQLType(udf.Pos, const_cast<TType*>(funcInfo.UserType), ctx); - YQL_ENSURE(udf.NormalizedUserType); - } - } catch (const std::exception& e) { - ctx.AddError(TIssue(udf.Pos, TStringBuilder() - << "Internal error was found when udf metadata is loading for function: " << udf.Name - << ", reason: " << e.what())); - hasErrors = true; - } - } - - return !hasErrors; -} - + if (!status.IsOk()) { + ctx.AddError(TIssue(udf.Pos, TStringBuilder() << "Failed to find UDF function: " << udf.Name + << ", reason: " << status.GetError())); + hasErrors = true; + continue; + } + + udf.CallableType = ConvertMiniKQLType(udf.Pos, funcInfo.FunctionType, ctx); + YQL_ENSURE(udf.CallableType); + if (funcInfo.RunConfigType) { + udf.RunConfigType = ConvertMiniKQLType(udf.Pos, const_cast<TType*>(funcInfo.RunConfigType), ctx); + YQL_ENSURE(udf.RunConfigType); + } + + if (funcInfo.UserType) { + udf.NormalizedUserType = ConvertMiniKQLType(udf.Pos, const_cast<TType*>(funcInfo.UserType), ctx); + YQL_ENSURE(udf.NormalizedUserType); + } + } catch (const std::exception& e) { + ctx.AddError(TIssue(udf.Pos, TStringBuilder() + << "Internal error was found when udf metadata is loading for function: " << udf.Name + << ", reason: " << e.what())); + hasErrors = true; + } + } + + return !hasErrors; +} + } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h index 4aec038e43..df2c668ba1 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h +++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h @@ -2,7 +2,7 @@ #include <ydb/library/yql/core/yql_type_annotation.h> #include <ydb/library/yql/core/file_storage/file_storage.h> - + #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> @@ -13,10 +13,10 @@ IUdfResolver::TPtr CreateSimpleUdfResolver( const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, const TFileStoragePtr& fileStorage = {}, bool useFakeMD5 = false); -bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, - const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - TExprContext& ctx); - +bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, + const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + TExprContext& ctx); + } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp index ac43a629b5..fe28fed28c 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp @@ -1,237 +1,237 @@ -#include "yql_udf_resolver_with_index.h" - +#include "yql_udf_resolver_with_index.h" + #include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h> #include <ydb/library/yql/core/yql_type_annotation.h> - + #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/minikql/mkql_program_builder.h> #include <ydb/library/yql/minikql/mkql_utils.h> - -#include <util/generic/hash_set.h> -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/set.h> -#include <util/generic/string.h> -#include <util/system/guard.h> + +#include <util/generic/hash_set.h> +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/generic/string.h> +#include <util/system/guard.h> #include <util/system/mutex.h> - -namespace NYql { -namespace NCommon { - -using namespace NKikimr; -using namespace NKikimr::NMiniKQL; - -class TUdfResolverWithIndex : public IUdfResolver { - class TResourceFile : public TThrRefBase { - public: - typedef TIntrusivePtr<TResourceFile> TPtr; - - public: - TResourceFile(TString alias, const TVector<TString>& modules, TFileLinkPtr link) - : Link_(std::move(link)) - { - Import_.FileAlias = alias; - Import_.Block = &Block_; - Import_.Modules = MakeMaybe(modules); - - Block_.Type = EUserDataType::PATH; - Block_.Data = Link_->GetPath(); - Block_.Usage.Set(EUserDataBlockUsage::Udf); - } - - static TResourceFile::TPtr Create(const TString& packageName, const TSet<TString>& modules, TFileLinkPtr link) { - // assume package name has no bad symbols for file name - TString basename = link->GetPath().Basename(); - TString alias = basename.StartsWith("lib") ? basename : ("lib_" + packageName + "_udf.so"); - alias.to_lower(); - return MakeIntrusive<TResourceFile>(std::move(alias), TVector<TString>(modules.begin(), modules.end()), std::move(link)); - } - - public: - TFileLinkPtr Link_; - TUserDataBlock Block_; - TImport Import_; - }; - -public: - TUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage) - : UdfIndex_(udfIndex) - , Fallback_(fallback) - , FileStorage_(fileStorage) - { - Y_ENSURE(UdfIndex_); - Y_ENSURE(FileStorage_); - // fallback is required only to handle type aware functions and loading rich metadata - Y_ENSURE(Fallback_); - } - - TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { - with_lock(Lock_) { - TString moduleNameStr(moduleName); - if (!UdfIndex_->ContainsModule(moduleNameStr)) { - return Nothing(); - } - - auto file = DownloadFileWithModule(moduleNameStr); - return MakeMaybe<TFilePathWithMd5>(file->Link_->GetPath(), file->Link_->GetMd5()); - } - } - - bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override { - with_lock(Lock_) { - bool hasErrors = false; - THashSet<TString> requiredModules; - TVector<TFunction*> fallbackFunctions; - TVector<TImport*> fallbackImports = imports; - TSet<TImport*> additionalImports; // avoid duplicates - - for (auto udfPtr : functions) { - TImport* additionalImport = nullptr; - TFunction* fallbackFunction = nullptr; - if (!LoadFunctionMetadata(*udfPtr, ctx, fallbackFunction, additionalImport)) { - hasErrors = true; - continue; - } - - if (additionalImport) { - additionalImports.insert(additionalImport); - } - - if (fallbackFunction) { - fallbackFunctions.push_back(fallbackFunction); - } - } - - fallbackImports.insert(fallbackImports.end(), additionalImports.begin(), additionalImports.end()); - - return Fallback_->LoadMetadata(fallbackImports, fallbackFunctions, ctx) && !hasErrors; - } - } - - TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { - return Fallback_->LoadRichMetadata(imports); - } - -private: - bool LoadFunctionMetadata(TFunction& function, TExprContext& ctx, TFunction*& fallbackFunction, TImport*& additionalImport) const { - TStringBuf moduleName, funcName; + +namespace NYql { +namespace NCommon { + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +class TUdfResolverWithIndex : public IUdfResolver { + class TResourceFile : public TThrRefBase { + public: + typedef TIntrusivePtr<TResourceFile> TPtr; + + public: + TResourceFile(TString alias, const TVector<TString>& modules, TFileLinkPtr link) + : Link_(std::move(link)) + { + Import_.FileAlias = alias; + Import_.Block = &Block_; + Import_.Modules = MakeMaybe(modules); + + Block_.Type = EUserDataType::PATH; + Block_.Data = Link_->GetPath(); + Block_.Usage.Set(EUserDataBlockUsage::Udf); + } + + static TResourceFile::TPtr Create(const TString& packageName, const TSet<TString>& modules, TFileLinkPtr link) { + // assume package name has no bad symbols for file name + TString basename = link->GetPath().Basename(); + TString alias = basename.StartsWith("lib") ? basename : ("lib_" + packageName + "_udf.so"); + alias.to_lower(); + return MakeIntrusive<TResourceFile>(std::move(alias), TVector<TString>(modules.begin(), modules.end()), std::move(link)); + } + + public: + TFileLinkPtr Link_; + TUserDataBlock Block_; + TImport Import_; + }; + +public: + TUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage) + : UdfIndex_(udfIndex) + , Fallback_(fallback) + , FileStorage_(fileStorage) + { + Y_ENSURE(UdfIndex_); + Y_ENSURE(FileStorage_); + // fallback is required only to handle type aware functions and loading rich metadata + Y_ENSURE(Fallback_); + } + + TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { + with_lock(Lock_) { + TString moduleNameStr(moduleName); + if (!UdfIndex_->ContainsModule(moduleNameStr)) { + return Nothing(); + } + + auto file = DownloadFileWithModule(moduleNameStr); + return MakeMaybe<TFilePathWithMd5>(file->Link_->GetPath(), file->Link_->GetMd5()); + } + } + + bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override { + with_lock(Lock_) { + bool hasErrors = false; + THashSet<TString> requiredModules; + TVector<TFunction*> fallbackFunctions; + TVector<TImport*> fallbackImports = imports; + TSet<TImport*> additionalImports; // avoid duplicates + + for (auto udfPtr : functions) { + TImport* additionalImport = nullptr; + TFunction* fallbackFunction = nullptr; + if (!LoadFunctionMetadata(*udfPtr, ctx, fallbackFunction, additionalImport)) { + hasErrors = true; + continue; + } + + if (additionalImport) { + additionalImports.insert(additionalImport); + } + + if (fallbackFunction) { + fallbackFunctions.push_back(fallbackFunction); + } + } + + fallbackImports.insert(fallbackImports.end(), additionalImports.begin(), additionalImports.end()); + + return Fallback_->LoadMetadata(fallbackImports, fallbackFunctions, ctx) && !hasErrors; + } + } + + TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const override { + return Fallback_->LoadRichMetadata(imports); + } + +private: + bool LoadFunctionMetadata(TFunction& function, TExprContext& ctx, TFunction*& fallbackFunction, TImport*& additionalImport) const { + TStringBuf moduleName, funcName; if (!SplitUdfName(function.Name, moduleName, funcName) || moduleName.empty() || funcName.empty()) { - ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Incorrect format of function name: " << function.Name)); - return false; - } - - /* - the order is really important: - 1) check we have such module - no-> fallback function - 2) check we have such function - no -> error - 3) download resource file - fail -> error - 4) if polymorphic function -> fallback function with additional Import for downloaded file - */ - - TString moduleNameStr = TString(moduleName); - if (!UdfIndex_->ContainsModule(moduleNameStr)) { - fallbackFunction = &function; - return true; - } - - TFunctionInfo info; - if (!UdfIndex_->FindFunction(moduleNameStr, function.Name, info)) { - ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Function not found: " << function.Name)); - return false; - } - - TResourceFile::TPtr file = DownloadFileWithModule(moduleName, function.Pos, ctx); - if (!file) { - return false; - } - - additionalImport = &file->Import_; - - if (info.IsTypeAwareness) { - fallbackFunction = &function; - return true; - } - - if (!info.CallableType) { - ctx.AddError(TIssue(function.Pos, TStringBuilder() << "CallableType for function " << function.Name << " is empty. Check UDF source code for errors.")); - return false; - } - + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Incorrect format of function name: " << function.Name)); + return false; + } + + /* + the order is really important: + 1) check we have such module + no-> fallback function + 2) check we have such function + no -> error + 3) download resource file + fail -> error + 4) if polymorphic function -> fallback function with additional Import for downloaded file + */ + + TString moduleNameStr = TString(moduleName); + if (!UdfIndex_->ContainsModule(moduleNameStr)) { + fallbackFunction = &function; + return true; + } + + TFunctionInfo info; + if (!UdfIndex_->FindFunction(moduleNameStr, function.Name, info)) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Function not found: " << function.Name)); + return false; + } + + TResourceFile::TPtr file = DownloadFileWithModule(moduleName, function.Pos, ctx); + if (!file) { + return false; + } + + additionalImport = &file->Import_; + + if (info.IsTypeAwareness) { + fallbackFunction = &function; + return true; + } + + if (!info.CallableType) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "CallableType for function " << function.Name << " is empty. Check UDF source code for errors.")); + return false; + } + function.CallableType = ParseTypeFromYson(TStringBuf{info.CallableType}, ctx, function.Pos); - if (!function.CallableType) { - ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build callable type from YSON for function " << function.Name)); - return false; - } - - if (info.RunConfigType) { + if (!function.CallableType) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build callable type from YSON for function " << function.Name)); + return false; + } + + if (info.RunConfigType) { function.RunConfigType = ParseTypeFromYson(TStringBuf{info.RunConfigType}, ctx, function.Pos); - if (!function.RunConfigType) { - ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build run config type from YSON for function " << function.Name)); - return false; - } - } else { - function.RunConfigType = std::get<0>(ctx.SingletonTypeCache); - } - - function.NormalizedUserType = std::get<0>(ctx.SingletonTypeCache); - return true; - } - - TResourceFile::TPtr DownloadFileWithModule(const TStringBuf& module, const TPosition& pos, TExprContext& ctx) const { - try { - return DownloadFileWithModule(module); - } catch (const std::exception& e) { - ctx.AddError(ExceptionToIssue(e, pos)); - } - - return nullptr; - } - - TResourceFile::TPtr DownloadFileWithModule(const TStringBuf& module) const { - TString moduleName(module); - - const auto it = DownloadedFiles_.find(module); - if (it != DownloadedFiles_.end()) { - return it->second; - } - - auto resource = UdfIndex_->FindResourceByModule(moduleName); - if (!resource) { - ythrow yexception() << "No resource has been found for registered module " << moduleName; - } - - // token is empty for urls for now - // assumption: file path is frozen already, no need to put into file storage - const TDownloadLink& downloadLink = resource->Link; - TFileLinkPtr link = downloadLink.IsUrl ? FileStorage_->PutUrl(downloadLink.Path, {}) : CreateFakeFileLink(downloadLink.Path, downloadLink.Md5); - TResourceFile::TPtr file = TResourceFile::Create(moduleName, resource->Modules, link); - for (auto& d : resource->Modules) { - auto p = DownloadedFiles_.emplace(d, file); - if (!p.second) { - // should not happen because UdfIndex handles conflicts - ythrow yexception() << "file already downloaded for module " << moduleName << ", conflicting path " << downloadLink.Path << ", existing local file " << p.first->second->Link_->GetPath(); - } - } - - return file; - } - -private: - mutable TMutex Lock_; - const TUdfIndex::TPtr UdfIndex_; - const IUdfResolver::TPtr Fallback_; - const TFileStoragePtr FileStorage_; - // module -> downloaded resource file - mutable TMap<TString, TResourceFile::TPtr> DownloadedFiles_; -}; - -IUdfResolver::TPtr CreateUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage) { - return new TUdfResolverWithIndex(udfIndex, fallback, fileStorage); -} - -} // namespace NCommon -} // namespace NYql + if (!function.RunConfigType) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build run config type from YSON for function " << function.Name)); + return false; + } + } else { + function.RunConfigType = std::get<0>(ctx.SingletonTypeCache); + } + + function.NormalizedUserType = std::get<0>(ctx.SingletonTypeCache); + return true; + } + + TResourceFile::TPtr DownloadFileWithModule(const TStringBuf& module, const TPosition& pos, TExprContext& ctx) const { + try { + return DownloadFileWithModule(module); + } catch (const std::exception& e) { + ctx.AddError(ExceptionToIssue(e, pos)); + } + + return nullptr; + } + + TResourceFile::TPtr DownloadFileWithModule(const TStringBuf& module) const { + TString moduleName(module); + + const auto it = DownloadedFiles_.find(module); + if (it != DownloadedFiles_.end()) { + return it->second; + } + + auto resource = UdfIndex_->FindResourceByModule(moduleName); + if (!resource) { + ythrow yexception() << "No resource has been found for registered module " << moduleName; + } + + // token is empty for urls for now + // assumption: file path is frozen already, no need to put into file storage + const TDownloadLink& downloadLink = resource->Link; + TFileLinkPtr link = downloadLink.IsUrl ? FileStorage_->PutUrl(downloadLink.Path, {}) : CreateFakeFileLink(downloadLink.Path, downloadLink.Md5); + TResourceFile::TPtr file = TResourceFile::Create(moduleName, resource->Modules, link); + for (auto& d : resource->Modules) { + auto p = DownloadedFiles_.emplace(d, file); + if (!p.second) { + // should not happen because UdfIndex handles conflicts + ythrow yexception() << "file already downloaded for module " << moduleName << ", conflicting path " << downloadLink.Path << ", existing local file " << p.first->second->Link_->GetPath(); + } + } + + return file; + } + +private: + mutable TMutex Lock_; + const TUdfIndex::TPtr UdfIndex_; + const IUdfResolver::TPtr Fallback_; + const TFileStoragePtr FileStorage_; + // module -> downloaded resource file + mutable TMap<TString, TResourceFile::TPtr> DownloadedFiles_; +}; + +IUdfResolver::TPtr CreateUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage) { + return new TUdfResolverWithIndex(udfIndex, fallback, fileStorage); +} + +} // namespace NCommon +} // namespace NYql diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.h b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.h index 972d7534d4..67a5d87a15 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.h +++ b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_udf_index.h> #include <ydb/library/yql/core/file_storage/file_storage.h> - -namespace NYql { -namespace NCommon { - -IUdfResolver::TPtr CreateUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage); - -} // namespace NCommon -} // namespace NYql + +namespace NYql { +namespace NCommon { + +IUdfResolver::TPtr CreateUdfResolverWithIndex(TUdfIndex::TPtr udfIndex, IUdfResolver::TPtr fallback, TFileStoragePtr fileStorage); + +} // namespace NCommon +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp index 927ac06043..ca826f2596 100644 --- a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp +++ b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp @@ -32,7 +32,7 @@ public: const NActors::TActorId& aggregatorId) : NActors::TActor<TFullResultWriterActor>(&TFullResultWriterActor::Handler) , TraceID(traceId) - , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, TVector<TString>())) + , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, TVector<TString>())) , FullResultWriter(std::move(writer)) , AggregatorID(aggregatorId) { diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.cpp b/ydb/library/yql/providers/dq/actors/proto_builder.cpp index e4b16403c3..d6d5c3b125 100644 --- a/ydb/library/yql/providers/dq/actors/proto_builder.cpp +++ b/ydb/library/yql/providers/dq/actors/proto_builder.cpp @@ -30,13 +30,13 @@ TVector<ui32> BuildColumnOrder(const TVector<TString>& columns, NKikimr::NMiniKQ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) { const auto columnName = TString(structType->GetMemberName(i)); column2id[columnName] = i; - } + } columnOrder.resize(columns.size()); int id = 0; for (const auto& columnName : columns) { columnOrder[id++] = column2id[columnName]; - } + } return columnOrder; } @@ -50,10 +50,10 @@ NDqProto::EDataTransportVersion GetTransportVersion(const NYql::NDqProto::TData& return NDqProto::EDataTransportVersion::DATA_TRANSPORT_ARROW_1_0; default: break; - } + } return NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED; } - + } // unnamed TProtoBuilder::TProtoBuilder(const TString& type, const TVector<TString>& columns) diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.h b/ydb/library/yql/providers/dq/actors/proto_builder.h index 8ba05c8c3d..e7de7b0e70 100644 --- a/ydb/library/yql/providers/dq/actors/proto_builder.h +++ b/ydb/library/yql/providers/dq/actors/proto_builder.h @@ -17,12 +17,12 @@ namespace NYql::NDqs { class TProtoBuilder { public: - TProtoBuilder(const TString& type, const TVector<TString>& columns); + TProtoBuilder(const TString& type, const TVector<TString>& columns); ~TProtoBuilder(); - bool CanBuildResultSet() const; - Ydb::ResultSet BuildResultSet(const TVector<NYql::NDqProto::TData>& data); - TString BuildYson(const TVector<NYql::NDqProto::TData>& data, ui64 maxBytesLimit = std::numeric_limits<ui64>::max()); + bool CanBuildResultSet() const; + Ydb::ResultSet BuildResultSet(const TVector<NYql::NDqProto::TData>& data); + TString BuildYson(const TVector<NYql::NDqProto::TData>& data, ui64 maxBytesLimit = std::numeric_limits<ui64>::max()); bool WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func); bool WriteData(const NYql::NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func); bool WriteData(const TVector<NYql::NDqProto::TData>& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func); @@ -33,7 +33,7 @@ private: NKikimr::NMiniKQL::TScopedAlloc Alloc; NKikimr::NMiniKQL::TTypeEnvironment TypeEnv; NKikimr::NMiniKQL::TType* ResultType; - const TVector<ui32> ColumnOrder; + const TVector<ui32> ColumnOrder; }; } // NYql::NDqs diff --git a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp index 59ae95c3af..a1e97efe6a 100644 --- a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp +++ b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp @@ -68,7 +68,7 @@ public: , Discard(discard) , TraceId(traceId) , Settings(settings) - , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) + , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) , ResultYsonOut(new THoldingStream<TCountingOutput>(MakeHolder<TStringOutput>(ResultYson))) , ResultYsonWriter(MakeHolder<NYson::TYsonWriter>(ResultYsonOut.Get(), NYson::EYsonFormat::Binary, ::NYson::EYsonType::Node, true)) { diff --git a/ydb/library/yql/providers/dq/actors/result_receiver.cpp b/ydb/library/yql/providers/dq/actors/result_receiver.cpp index 7fdb73f257..b13337a4c7 100644 --- a/ydb/library/yql/providers/dq/actors/result_receiver.cpp +++ b/ydb/library/yql/providers/dq/actors/result_receiver.cpp @@ -47,7 +47,7 @@ public: , SecureParams(std::move(secureParams)) , ResultBuilder( resultType - ? MakeHolder<TProtoBuilder>(resultType, columns) + ? MakeHolder<TProtoBuilder>(resultType, columns) : nullptr) , Discard(discard) { diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp index 238828e453..c881a99f59 100644 --- a/ydb/library/yql/providers/dq/actors/task_controller.cpp +++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp @@ -74,7 +74,7 @@ public: } ~TTaskController() override { - SetTaskCountMetric(0); + SetTaskCountMetric(0); } private: @@ -214,19 +214,19 @@ private: *group->GetCounter(name) = v.Count; if (ServiceCounters.PublicCounters && taskId == 0) { TString publicCounterName; - bool isDeriv = false; - if (name == "MkqlMaxMemoryUsage") { - publicCounterName = "query.memory_usage_bytes"; - } else if (name == "ComputeCpuTimeUs") { - publicCounterName = "query.cpu_usage_us"; - isDeriv = true; - } else if (name == "Bytes") { + bool isDeriv = false; + if (name == "MkqlMaxMemoryUsage") { + publicCounterName = "query.memory_usage_bytes"; + } else if (name == "ComputeCpuTimeUs") { + publicCounterName = "query.cpu_usage_us"; + isDeriv = true; + } else if (name == "Bytes") { if (labels.find("Source") != labels.end()) publicCounterName = "query.input_bytes"; else if (labels.find("Sink") != labels.end()) publicCounterName = "query.output_bytes"; - isDeriv = true; + isDeriv = true; } if (publicCounterName) { - *ServiceCounters.PublicCounters->GetNamedCounter("name", publicCounterName, isDeriv) = v.Count; + *ServiceCounters.PublicCounters->GetNamedCounter("name", publicCounterName, isDeriv) = v.Count; } } } @@ -387,18 +387,18 @@ private: } } - void SetTaskCountMetric(ui64 count) { - if (!ServiceCounters.Counters) { - return; - } - *ServiceCounters.Counters->GetCounter("TaskCount") = count; - - if (!ServiceCounters.PublicCounters) { - return; - } - *ServiceCounters.PublicCounters->GetNamedCounter("name", "query.running_tasks") = count; - } - + void SetTaskCountMetric(ui64 count) { + if (!ServiceCounters.Counters) { + return; + } + *ServiceCounters.Counters->GetCounter("TaskCount") = count; + + if (!ServiceCounters.PublicCounters) { + return; + } + *ServiceCounters.PublicCounters->GetNamedCounter("name", "query.running_tasks") = count; + } + void OnReadyState(TEvReadyState::TPtr& ev) { YQL_LOG_CTX_SCOPE(TraceId); @@ -408,7 +408,7 @@ private: const auto& actorIds = ev->Get()->Record.GetActorId(); Y_VERIFY(tasks.size() == actorIds.size()); - SetTaskCountMetric(tasks.size()); + SetTaskCountMetric(tasks.size()); for (int i = 0; i < static_cast<int>(tasks.size()); ++i) { auto actorId = ActorIdFromProto(actorIds[i]); diff --git a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto index 6e3d08e595..c9592ea810 100644 --- a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto +++ b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto @@ -29,13 +29,13 @@ message TCommandHeader { GET_STATS = 14; GET_STATS_INPUT = 15; GET_STATS_OUTPUT = 16; - GET_STATS_SOURCE = 17; - - GET_FREE_SPACE_SOURCE = 18; - GET_STORED_BYTES_SOURCE = 19; - PUSH_SOURCE = 20; - GET_SOURCE_TYPE = 21; - FINISH_SOURCE = 22; + GET_STATS_SOURCE = 17; + + GET_FREE_SPACE_SOURCE = 18; + GET_STORED_BYTES_SOURCE = 19; + PUSH_SOURCE = 20; + GET_SOURCE_TYPE = 21; + FINISH_SOURCE = 22; // Sink SINK_POP = 23; // TSinkPopRequest -> TSinkPopResponse @@ -91,7 +91,7 @@ message TPrepareResponse { repeated TMetric Metric = 2; } -message TGetTypeResponse { +message TGetTypeResponse { bytes Result = 1; } @@ -118,11 +118,11 @@ message TGetStatsOutputResponse { NDqProto.TDqOutputChannelStats Stats = 2; } -message TGetStatsSourceResponse { +message TGetStatsSourceResponse { reserved 1; // NKqpProto.TKqpStatsTask.TSourceStats Stats = 1; NDqProto.TDqSourceStats Stats = 2; -} - +} + message TSinkStatsResponse { NDqProto.TDqSinkStats Stats = 1; } @@ -130,13 +130,13 @@ message TSinkStatsResponse { message TPrepareRequest { NYql.NDqProto.TDqTask Task = 1; } - -message TSourcePushRequest { - TData Data = 1; + +message TSourcePushRequest { + TData Data = 1; int64 Space = 2; repeated bytes String = 3; int32 Chunks = 4; -} +} message TSourcePushChunk { int64 Parts = 1; diff --git a/ydb/library/yql/providers/dq/counters/counters.h b/ydb/library/yql/providers/dq/counters/counters.h index 4a457d24fc..8f3d2ac200 100644 --- a/ydb/library/yql/providers/dq/counters/counters.h +++ b/ydb/library/yql/providers/dq/counters/counters.h @@ -224,20 +224,20 @@ struct TCounters { NDq::TDqSourceStats& oldStats, ui64 taskId, ui64 inputIndex) { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)}, - {"SourceIndex", ToString(inputIndex)} - }; - - ADD_COUNTER(Chunks); - ADD_COUNTER(Bytes); - ADD_COUNTER(RowsIn); - ADD_COUNTER(RowsOut); - ADD_COUNTER(RowsInMemory); - ADD_COUNTER(MaxMemoryUsage); - ADD_COUNTER(InputIndex); - } - + std::map<TString, TString> labels = { + {"Task", ToString(taskId)}, + {"SourceIndex", ToString(inputIndex)} + }; + + ADD_COUNTER(Chunks); + ADD_COUNTER(Bytes); + ADD_COUNTER(RowsIn); + ADD_COUNTER(RowsOut); + ADD_COUNTER(RowsInMemory); + ADD_COUNTER(MaxMemoryUsage); + ADD_COUNTER(InputIndex); + } + void AddOutputChannelStats( const NDq::TDqOutputChannelStats& currentStats, NDq::TDqOutputChannelStats& oldStats, diff --git a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp index 1a8d3172e4..63d870a32d 100644 --- a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp +++ b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp @@ -24,24 +24,24 @@ namespace NYql { using namespace NActors; using NDqs::MakeWorkerManagerActorID; -namespace { +namespace { // TODO: Use the only driver for both sources. NDq::IDqSourceActorFactory::TPtr CreateSourceActorFactory(const NYdb::TDriver& driver, IHTTPGateway::TPtr httpGateway) { - auto factory = MakeIntrusive<NYql::NDq::TDqSourceFactory>(); + auto factory = MakeIntrusive<NYql::NDq::TDqSourceFactory>(); RegisterDqPqReadActorFactory(*factory, driver, nullptr); - RegisterYdbReadActorFactory(*factory, driver, nullptr); + RegisterYdbReadActorFactory(*factory, driver, nullptr); RegisterS3ReadActorFactory(*factory, nullptr, httpGateway); RegisterClickHouseReadActorFactory(*factory, nullptr, httpGateway); - return factory; - } + return factory; + } NDq::IDqSinkActorFactory::TPtr CreateSinkActorFactory(const NYdb::TDriver& driver) { auto factory = MakeIntrusive<NYql::NDq::TDqSinkFactory>(); RegisterDqPqWriteActorFactory(*factory, driver, nullptr); return factory; } -} - +} + class TLocalServiceHolder { public: TLocalServiceHolder(NYdb::TDriver driver, IHTTPGateway::TPtr httpGateway, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 8759f44221..1ef883e58c 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -172,9 +172,9 @@ public: } auto serializedResultType = GetSerializedResultType(lambda); - NYql::NDqs::TProtoBuilder protoBuilder(serializedResultType, columns); + NYql::NDqs::TProtoBuilder protoBuilder(serializedResultType, columns); - result.Data = protoBuilder.BuildYson(rows); + result.Data = protoBuilder.BuildYson(rows); AddCounter("LocalRun", TInstant::Now() - t); diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp index 9b41920b17..3fd60a2275 100644 --- a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp +++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp @@ -38,7 +38,7 @@ namespace NTaskRunnerProxy { // static const int CurrentProtocolVersion = 1; // static const int CurrentProtocolVersion = 2; // GetFreeSpace -// static const int CurrentProtocolVersion = 3; // Calls for ComputeActor +// static const int CurrentProtocolVersion = 3; // Calls for ComputeActor // static const int CurrentProtocolVersion = 4; // Calls for Sources static const int CurrentProtocolVersion = 5; // Calls for Sinks @@ -55,17 +55,17 @@ void ToProto(T* s1, const NDq::TDqInputChannelStats* ss) } template<typename T> -void ToProto(T* s1, const NDq::TDqSourceStats* ss) -{ - s1->SetChunks(ss->Chunks); - s1->SetBytes(ss->Bytes); - s1->SetRowsIn(ss->RowsIn); - s1->SetRowsOut(ss->RowsOut); - s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); - s1->SetInputIndex(ss->InputIndex); -} - -template<typename T> +void ToProto(T* s1, const NDq::TDqSourceStats* ss) +{ + s1->SetChunks(ss->Chunks); + s1->SetBytes(ss->Bytes); + s1->SetRowsIn(ss->RowsIn); + s1->SetRowsOut(ss->RowsOut); + s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); + s1->SetInputIndex(ss->InputIndex); +} + +template<typename T> void ToProto(T* s1, const NDq::TDqOutputChannelStats* ss) { s1->SetChannelId(ss->ChannelId); @@ -173,16 +173,16 @@ public: Runner->GetTaskId(), channelId); } - void UpdateSourceStats(ui64 inputIndex) - { - if (!Runner) { - return; - } - auto s = Runner->GetStats(); - auto maybeSourceStats = s->Sources.find(inputIndex); + void UpdateSourceStats(ui64 inputIndex) + { + if (!Runner) { + return; + } + auto s = Runner->GetStats(); + auto maybeSourceStats = s->Sources.find(inputIndex); if (maybeSourceStats == s->Sources.end() || !maybeSourceStats->second) { - return; - } + return; + } auto maybeSourceOldStats = CurrentSourcesStats.find(inputIndex); if (maybeSourceOldStats == CurrentSourcesStats.end()) { maybeSourceOldStats = CurrentSourcesStats.emplace_hint( @@ -192,8 +192,8 @@ public: *maybeSourceStats->second, maybeSourceOldStats->second, Runner->GetTaskId(), inputIndex); - } - + } + template<typename T> void UpdateStats(T& t) { UpdateStats(); @@ -299,14 +299,14 @@ public: UpdateInputChannelStats(channelId); break; } - case NDqProto::TCommandHeader::PUSH_SOURCE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::PUSH_SOURCE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto source = Runner->GetSource(channelId); - - NDqProto::TSourcePushRequest request; - request.Load(&input); - + auto source = Runner->GetSource(channelId); + + NDqProto::TSourcePushRequest request; + request.Load(&input); + auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit auto transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED; switch (request.GetData().GetTransportVersion()) { @@ -326,8 +326,8 @@ public: transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED; } NDq::TDqDataSerializer dataSerializer(Runner->GetTypeEnv(), Runner->GetHolderFactory(), transportVersion); - NKikimr::NMiniKQL::TUnboxedValueVector buffer; - buffer.reserve(request.GetData().GetRows()); + NKikimr::NMiniKQL::TUnboxedValueVector buffer; + buffer.reserve(request.GetData().GetRows()); if (request.GetString().empty() && request.GetChunks() == 0) { dataSerializer.Deserialize(request.GetData(), source->GetInputType(), buffer); } else if (!request.GetString().empty()) { @@ -356,11 +356,11 @@ public: } } } - - source->Push(std::move(buffer), request.GetSpace()); + + source->Push(std::move(buffer), request.GetSpace()); UpdateSourceStats(channelId); - break; - } + break; + } case NDqProto::TCommandHeader::FINISH: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -375,13 +375,13 @@ public: UpdateOutputChannelStats(channelId); break; } - case NDqProto::TCommandHeader::FINISH_SOURCE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::FINISH_SOURCE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - Runner->GetSource(channelId)->Finish(); - UpdateSourceStats(channelId); - break; - } + Runner->GetSource(channelId)->Finish(); + UpdateSourceStats(channelId); + break; + } case NDqProto::TCommandHeader::DROP_OUTPUT: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -409,16 +409,16 @@ public: response.Save(&output); break; } - case NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto res = Runner->GetSource(channelId)->GetStoredBytes(); - - NDqProto::TGetStoredBytesResponse response; - response.SetResult(res); - response.Save(&output); - break; - } + auto res = Runner->GetSource(channelId)->GetStoredBytes(); + + NDqProto::TGetStoredBytesResponse response; + response.SetResult(res); + response.Save(&output); + break; + } case NDqProto::TCommandHeader::POP: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit @@ -488,26 +488,26 @@ public: auto channel = Runner->GetInputChannel(channelId); auto inputType = channel->GetInputType(); - NDqProto::TGetTypeResponse response; + NDqProto::TGetTypeResponse response; response.SetResult(NKikimr::NMiniKQL::SerializeNode(inputType, Runner->GetTypeEnv())); response.Save(&output); break; } - case NDqProto::TCommandHeader::GET_SOURCE_TYPE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::GET_SOURCE_TYPE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - + Y_ENSURE(taskId == Runner->GetTaskId()); - auto source = Runner->GetSource(channelId); - auto inputType = source->GetInputType(); - - NDqProto::TGetTypeResponse response; - response.SetResult(NKikimr::NMiniKQL::SerializeNode(inputType, Runner->GetTypeEnv())); - response.Save(&output); - - break; - } + auto source = Runner->GetSource(channelId); + auto inputType = source->GetInputType(); + + NDqProto::TGetTypeResponse response; + response.SetResult(NKikimr::NMiniKQL::SerializeNode(inputType, Runner->GetTypeEnv())); + response.Save(&output); + + break; + } case NDqProto::TCommandHeader::GET_FREE_SPACE: { Y_ENSURE(header.GetVersion() >= 2); @@ -519,17 +519,17 @@ public: break; } - case NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE: { + case NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE: { Y_ENSURE(header.GetVersion() >= 4); - + Y_ENSURE(taskId == Runner->GetTaskId()); - auto source = Runner->GetSource(channelId); - NDqProto::TGetFreeSpaceResponse response; - response.SetFreeSpace(source->GetFreeSpace()); - response.Save(&output); - - break; - } + auto source = Runner->GetSource(channelId); + NDqProto::TGetFreeSpaceResponse response; + response.SetFreeSpace(source->GetFreeSpace()); + response.Save(&output); + + break; + } case NDqProto::TCommandHeader::GET_STATS: { Y_ENSURE(header.GetVersion() >= 3); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -568,11 +568,11 @@ public: for (const auto& [id, ss] : stats->InputChannels) { ToProto(s->AddInputChannels(), ss); } - - for (const auto& [id, ss] : stats->Sources) { - ToProto(s->AddSources(), ss); - } - + + for (const auto& [id, ss] : stats->Sources) { + ToProto(s->AddSources(), ss); + } + response.Save(&output); break; @@ -586,15 +586,15 @@ public: response.Save(&output); break; } - case NDqProto::TCommandHeader::GET_STATS_SOURCE: { + case NDqProto::TCommandHeader::GET_STATS_SOURCE: { Y_ENSURE(header.GetVersion() >= 4); Y_ENSURE(taskId == Runner->GetTaskId()); - auto ss = Runner->GetSource(channelId)->GetStats(); - NDqProto::TGetStatsSourceResponse response; - ToProto(response.MutableStats(), ss); - response.Save(&output); - break; - } + auto ss = Runner->GetSource(channelId)->GetStats(); + NDqProto::TGetStatsSourceResponse response; + ToProto(response.MutableStats(), ss); + response.Save(&output); + break; + } case NDqProto::TCommandHeader::GET_STATS_OUTPUT: { Y_ENSURE(header.GetVersion() >= 3); Y_ENSURE(taskId == Runner->GetTaskId()); diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp index ab8769f44e..c42ee58149 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp @@ -175,10 +175,10 @@ public: return Runner->GetTypeEnv(); } - const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override { - return Runner->GetHolderFactory(); - } - + const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override { + return Runner->GetHolderFactory(); + } + TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override { return Runner->BindAllocator(memoryLimit); } diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp index cdcbd32ef5..cd59ce0092 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp @@ -73,19 +73,19 @@ void FromProto(TDqInputChannelStats* s, const T& f) } template<typename T> -void FromProto(TDqSourceStats* s, const T& f) -{ - s->InputIndex = f.GetInputIndex(); - s->Chunks = f.GetChunks(); - s->Bytes = f.GetBytes(); - s->RowsIn = f.GetRowsIn(); - s->RowsOut = f.GetRowsOut(); - s->MaxMemoryUsage = f.GetMaxMemoryUsage(); +void FromProto(TDqSourceStats* s, const T& f) +{ + s->InputIndex = f.GetInputIndex(); + s->Chunks = f.GetChunks(); + s->Bytes = f.GetBytes(); + s->RowsIn = f.GetRowsIn(); + s->RowsOut = f.GetRowsOut(); + s->MaxMemoryUsage = f.GetMaxMemoryUsage(); //s->StartTs = TInstant::MilliSeconds(f.GetStartTs()); //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs()); -} - -template<typename T> +} + +template<typename T> void FromProto(TDqSinkStats* s, const T& f) { s->Chunks = f.GetChunks(); @@ -519,11 +519,11 @@ public: class TInputChannel : public IInputChannel { public: TInputChannel(const ITaskRunner::TPtr& taskRunner, ui64 taskId, ui64 channelId, IInputStream& input, IOutputStream& output) - : TaskId(taskId) + : TaskId(taskId) , ChannelId(channelId) , Input(input) , Output(output) - , ProtocolVersion(taskRunner->GetProtocolVersion()) + , ProtocolVersion(taskRunner->GetProtocolVersion()) { } i64 GetFreeSpace() override { @@ -693,47 +693,47 @@ private: }; class TDqSource: public IStringSource { -public: - TDqSource(ui64 taskId, ui64 inputIndex, IPipeTaskRunner* taskRunner) - : TaskId(taskId) - , InputIndex(inputIndex) - , Stats(inputIndex) - , TaskRunner(taskRunner) - , Input(TaskRunner->GetInput()) - , Output(TaskRunner->GetOutput()) - { } - - i64 GetFreeSpace() const override { - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - - NDqProto::TGetFreeSpaceResponse response; - response.Load(&Input); - return response.GetFreeSpace(); - } - - ui64 GetStoredBytes() const override { - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - - NDqProto::TGetStoredBytesResponse response; - response.Load(&Input); - - return response.GetResult(); - } - - bool Empty() const override { - ythrow yexception() << "unimplemented"; - } - +public: + TDqSource(ui64 taskId, ui64 inputIndex, IPipeTaskRunner* taskRunner) + : TaskId(taskId) + , InputIndex(inputIndex) + , Stats(inputIndex) + , TaskRunner(taskRunner) + , Input(TaskRunner->GetInput()) + , Output(TaskRunner->GetOutput()) + { } + + i64 GetFreeSpace() const override { + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + + NDqProto::TGetFreeSpaceResponse response; + response.Load(&Input); + return response.GetFreeSpace(); + } + + ui64 GetStoredBytes() const override { + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + + NDqProto::TGetStoredBytesResponse response; + response.Load(&Input); + + return response.GetResult(); + } + + bool Empty() const override { + ythrow yexception() << "unimplemented"; + } + void PushString(TVector<TString>&& batch, i64 space) override { if (space > static_cast<i64>(256_MB)) { throw yexception() << "Too big batch " << space; @@ -769,83 +769,83 @@ public: } } - void Push(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, i64 space) override { - auto inputType = GetInputType(); - NDqProto::TSourcePushRequest data; + void Push(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, i64 space) override { + auto inputType = GetInputType(); + NDqProto::TSourcePushRequest data; TDqDataSerializer dataSerializer(TaskRunner->GetTypeEnv(), TaskRunner->GetHolderFactory(), NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); - *data.MutableData() = dataSerializer.Serialize(batch, static_cast<NKikimr::NMiniKQL::TType*>(inputType)); - data.SetSpace(space); - - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::PUSH_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - data.Save(&Output); - } - - [[nodiscard]] - bool Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch) override { - Y_UNUSED(batch); - ythrow yexception() << "unimplemented"; - } - - void Finish() override { - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::FINISH_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - } - - bool IsFinished() const override { - ythrow yexception() << "unimplemented"; - } - - ui64 GetInputIndex() const override { - return InputIndex; - } - + *data.MutableData() = dataSerializer.Serialize(batch, static_cast<NKikimr::NMiniKQL::TType*>(inputType)); + data.SetSpace(space); + + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::PUSH_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + data.Save(&Output); + } + + [[nodiscard]] + bool Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch) override { + Y_UNUSED(batch); + ythrow yexception() << "unimplemented"; + } + + void Finish() override { + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::FINISH_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + } + + bool IsFinished() const override { + ythrow yexception() << "unimplemented"; + } + + ui64 GetInputIndex() const override { + return InputIndex; + } + const TDqSourceStats* GetStats() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::GET_STATS_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - - NDqProto::TGetStatsSourceResponse response; - response.Load(&Input); - - FromProto(&Stats, response.GetStats()); - return &Stats; - } catch (...) { - TaskRunner->RaiseException(); - } - } - - NKikimr::NMiniKQL::TType* GetInputType() const override { - if (InputType) { - return InputType; - } - - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::GET_SOURCE_TYPE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - - NDqProto::TGetTypeResponse response; - response.Load(&Input); - - InputType = static_cast<NKikimr::NMiniKQL::TType*>(NKikimr::NMiniKQL::DeserializeNode(response.GetResult(), TaskRunner->GetTypeEnv())); - return InputType; - } - + try { + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::GET_STATS_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + + NDqProto::TGetStatsSourceResponse response; + response.Load(&Input); + + FromProto(&Stats, response.GetStats()); + return &Stats; + } catch (...) { + TaskRunner->RaiseException(); + } + } + + NKikimr::NMiniKQL::TType* GetInputType() const override { + if (InputType) { + return InputType; + } + + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::GET_SOURCE_TYPE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + + NDqProto::TGetTypeResponse response; + response.Load(&Input); + + InputType = static_cast<NKikimr::NMiniKQL::TType*>(NKikimr::NMiniKQL::DeserializeNode(response.GetResult(), TaskRunner->GetTypeEnv())); + return InputType; + } + void Pause() override { Y_FAIL("Checkpoints are not supported"); } @@ -858,17 +858,17 @@ public: return false; } -private: - ui64 TaskId; - ui64 InputIndex; - mutable TDqSourceStats Stats; - - IPipeTaskRunner* TaskRunner; - IInputStream& Input; - IOutputStream& Output; - mutable NKikimr::NMiniKQL::TType* InputType = nullptr; -}; - +private: + ui64 TaskId; + ui64 InputIndex; + mutable TDqSourceStats Stats; + + IPipeTaskRunner* TaskRunner; + IInputStream& Input; + IOutputStream& Output; + mutable NKikimr::NMiniKQL::TType* InputType = nullptr; +}; + /*______________________________________________________________________________________________*/ class TOutputChannel : public IOutputChannel { @@ -1370,7 +1370,7 @@ public: return TypeEnv; } - const NMiniKQL::THolderFactory& GetHolderFactory() const override { + const NMiniKQL::THolderFactory& GetHolderFactory() const override { return HolderFactory; } @@ -1536,15 +1536,15 @@ public: } IDqSource::TPtr GetSource(ui64 inputIndex) override { - auto& source = Sources[inputIndex]; - if (!source) { - source = new TDqSource( - Task.GetId(), - inputIndex, - Delegate.Get()); - Stats.Sources[inputIndex] = source->GetStats(); - } - return source; + auto& source = Sources[inputIndex]; + if (!source) { + source = new TDqSource( + Task.GetId(), + inputIndex, + Delegate.Get()); + Stats.Sources[inputIndex] = source->GetStats(); + } + return source; } IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) override @@ -1623,7 +1623,7 @@ public: TString Save() const override { ythrow yexception() << "unimplemented"; } - + void Load(TStringBuf in) override { Y_UNUSED(in); ythrow yexception() << "unimplemented"; @@ -1635,7 +1635,7 @@ private: mutable TDqTaskRunnerStats Stats; THashMap<ui64, IDqInputChannel::TPtr> InputChannels; - THashMap<ui64, IDqSource::TPtr> Sources; + THashMap<ui64, IDqSource::TPtr> Sources; THashMap<ui64, IDqOutputChannel::TPtr> OutputChannels; THashMap<ui64, IDqSink::TPtr> Sinks; }; diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h index 31a6e82784..a008739d02 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h @@ -67,8 +67,8 @@ public: virtual const THashMap<TString,TString>& GetTaskParams() const = 0; virtual const THashMap<TString,TString>& GetSecureParams() const = 0; virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0; - virtual const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const = 0; - + virtual const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const = 0; + // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any) // if memoryLimit = 0 then set unlimited // otherwise use particular memory limit diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp index e26956545e..000cb66f4b 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp @@ -98,7 +98,7 @@ public: NPq::NProto::TDqPqTopicSource&& sourceParams, NPq::NProto::TDqReadTaskParams&& readParams, NYdb::TDriver driver, - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, ICallbacks* callbacks, i64 bufferSize) : TActor<TDqPqReadActor>(&TDqPqReadActor::StateFunc) @@ -107,13 +107,13 @@ public: , BufferSize(bufferSize) , HolderFactory(holderFactory) , Driver(std::move(driver)) - , CredentialsProviderFactory(std::move(credentialsProviderFactory)) + , CredentialsProviderFactory(std::move(credentialsProviderFactory)) , SourceParams(std::move(sourceParams)) , ReadParams(std::move(readParams)) , StartingMessageTimestamp(TInstant::Now()) , Callbacks(callbacks) { - Y_UNUSED(HolderFactory); + Y_UNUSED(HolderFactory); } NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() const { @@ -121,7 +121,7 @@ public: opts.Database(SourceParams.GetDatabase()) .DiscoveryEndpoint(SourceParams.GetEndpoint()) .EnableSsl(SourceParams.GetUseSsl()) - .CredentialsProviderFactory(CredentialsProviderFactory); + .CredentialsProviderFactory(CredentialsProviderFactory); return opts; } @@ -310,8 +310,8 @@ private: LWPROBE(PqReadDataReceived, Self.TxId, Self.SourceParams.GetTopicPath(), data); SINK_LOG_T("Data received: " << data); - Batch.emplace_back(NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(data.Data(), data.Size()))); - UsedSpace += data.Size(); + Batch.emplace_back(NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(data.Data(), data.Size()))); + UsedSpace += data.Size(); } Self.UpdateStateWithNewReadData(event); Self.CurrentDeferredCommit.Add(event); @@ -352,7 +352,7 @@ private: const i64 BufferSize; const THolderFactory& HolderFactory; NYdb::TDriver Driver; - std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; + std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; const NPq::NProto::TDqPqTopicSource SourceParams; const NPq::NProto::TDqReadTaskParams ReadParams; std::unique_ptr<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; @@ -360,7 +360,7 @@ private: NThreading::TFuture<void> EventFuture; THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event. TInstant StartingMessageTimestamp; - ICallbacks* const Callbacks; + ICallbacks* const Callbacks; std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits; NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit; bool SubscribedOnEvent = false; @@ -373,11 +373,11 @@ std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - IDqSourceActor::ICallbacks* callbacks, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + IDqSourceActor::ICallbacks* callbacks, const NKikimr::NMiniKQL::THolderFactory& holderFactory, - i64 bufferSize - ) + i64 bufferSize + ) { auto taskParamsIt = taskParams.find("pq"); YQL_ENSURE(taskParamsIt != taskParams.end(), "Failed to get pq task params"); @@ -385,22 +385,22 @@ std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor( NPq::NProto::TDqReadTaskParams readTaskParamsMsg; YQL_ENSURE(readTaskParamsMsg.ParseFromString(taskParamsIt->second), "Failed to parse DqPqRead task params"); - const TString& tokenName = settings.GetToken().GetName(); - const TString token = secureParams.Value(tokenName, TString()); + const TString& tokenName = settings.GetToken().GetName(); + const TString token = secureParams.Value(tokenName, TString()); const bool addBearerToToken = settings.GetAddBearerToToken(); - + TDqPqReadActor* actor = new TDqPqReadActor( inputIndex, std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), - holderFactory, + holderFactory, std::move(settings), std::move(readTaskParamsMsg), - std::move(driver), + std::move(driver), CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), - callbacks, - bufferSize - ); - + callbacks, + bufferSize + ); + return {actor, actor}; } @@ -421,8 +421,8 @@ void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver drive credentialsFactory, args.Callback, args.HolderFactory); - }); - -} - + }); + +} + } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h index d7e2c82dd0..7030f1b9d3 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h @@ -2,10 +2,10 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> - + #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - + #include <ydb/library/yql/providers/pq/proto/dq_io.pb.h> #include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h> @@ -18,7 +18,7 @@ namespace NYql::NDq { -class TDqSourceFactory; +class TDqSourceFactory; const i64 PQReadDefaultFreeSpace = 16_MB; @@ -29,12 +29,12 @@ std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSourceActor::ICallbacks* callback, const NKikimr::NMiniKQL::THolderFactory& holderFactory, - i64 bufferSize = PQReadDefaultFreeSpace - ); + i64 bufferSize = PQReadDefaultFreeSpace + ); void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); - + } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp index 85fbc91c01..3cb51b36f1 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp @@ -95,15 +95,15 @@ public: const TString& txId, NPq::NProto::TDqPqTopicSink&& sinkParams, NYdb::TDriver driver, - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, IDqSinkActor::ICallbacks* callbacks, i64 freeSpace) : TActor<TDqPqWriteActor>(&TDqPqWriteActor::StateFunc) , OutputIndex(outputIndex) , TxId(txId) , SinkParams(std::move(sinkParams)) - , Driver(std::move(driver)) - , CredentialsProviderFactory(credentialsProviderFactory) + , Driver(std::move(driver)) + , CredentialsProviderFactory(credentialsProviderFactory) , Callbacks(callbacks) , FreeSpace(freeSpace) , PersQueueClient(Driver, GetPersQueueClientSettings()) @@ -233,12 +233,12 @@ private: : NYdb::NPersQueue::ECodec::GZIP); } - NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() { - return NYdb::NPersQueue::TPersQueueClientSettings() + NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() { + return NYdb::NPersQueue::TPersQueueClientSettings() .Database(SinkParams.GetDatabase()) .DiscoveryEndpoint(SinkParams.GetEndpoint()) .EnableSsl(SinkParams.GetUseSsl()) - .CredentialsProviderFactory(CredentialsProviderFactory); + .CredentialsProviderFactory(CredentialsProviderFactory); } static i64 GetItemSize(const TString& item) { @@ -370,7 +370,7 @@ private: const TString TxId; const NPq::NProto::TDqPqTopicSink SinkParams; NYdb::TDriver Driver; - std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; + std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; IDqSinkActor::ICallbacks* const Callbacks; i64 FreeSpace = 0; @@ -394,19 +394,19 @@ std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor( TTxId txId, const THashMap<TString, TString>& secureParams, NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSinkActor::ICallbacks* callbacks, i64 freeSpace) { - const TString& tokenName = settings.GetToken().GetName(); - const TString token = secureParams.Value(tokenName, TString()); + const TString& tokenName = settings.GetToken().GetName(); + const TString token = secureParams.Value(tokenName, TString()); const bool addBearerToToken = settings.GetAddBearerToToken(); - + TDqPqWriteActor* actor = new TDqPqWriteActor( outputIndex, std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), std::move(settings), - std::move(driver), + std::move(driver), CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), callbacks, freeSpace); @@ -415,7 +415,7 @@ std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor( void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NPq::NProto::TDqPqTopicSink>("PqSink", - [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)]( + [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)]( NPq::NProto::TDqPqTopicSink&& settings, IDqSinkActorFactory::TArguments&& args) { @@ -425,10 +425,10 @@ void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver args.OutputIndex, args.TxId, args.SecureParams, - driver, - credentialsFactory, + driver, + credentialsFactory, args.Callback - ); + ); }); } diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h index 6736c8188b..8ae3a38635 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h @@ -2,10 +2,10 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> - + #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - + #include <ydb/library/yql/providers/pq/proto/dq_io.pb.h> #include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h> @@ -26,7 +26,7 @@ std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor( TTxId txId, const THashMap<TString, TString>& secureParams, NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSinkActor::ICallbacks* callbacks, i64 freeSpace = DqPqDefaultFreeSpace); diff --git a/ydb/library/yql/providers/pq/cm_client/interface/client.h b/ydb/library/yql/providers/pq/cm_client/interface/client.h index 368d2f2039..0520712b36 100644 --- a/ydb/library/yql/providers/pq/cm_client/interface/client.h +++ b/ydb/library/yql/providers/pq/cm_client/interface/client.h @@ -161,7 +161,7 @@ using TAsyncDescribePathResult = NThreading::TFuture<TDescribePathResult>; struct IClient : public TThrRefBase { using TPtr = TIntrusivePtr<IClient>; - virtual TAsyncDescribePathResult DescribePath(const TString& path) const = 0; + virtual TAsyncDescribePathResult DescribePath(const TString& path) const = 0; // TODO: If you need other methods, add them. }; diff --git a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp index 71d878f5e5..f476612a93 100644 --- a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp +++ b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp @@ -229,9 +229,9 @@ public: }; } - TAsyncDescribePathResult DescribePath(const TString& path) const override { + TAsyncDescribePathResult DescribePath(const TString& path) const override { NLogBroker::DescribePathRequest describeRequest; - describeRequest.set_token(Options.GetCredentialsProviderFactory()->CreateProvider()->GetAuthInfo()); + describeRequest.set_token(Options.GetCredentialsProviderFactory()->CreateProvider()->GetAuthInfo()); describeRequest.mutable_path()->set_path(path); NThreading::TPromise<TDescribePathResult> promise = NThreading::NewPromise<TDescribePathResult>(); diff --git a/ydb/library/yql/providers/pq/cm_client/lib/client.h b/ydb/library/yql/providers/pq/cm_client/lib/client.h index 1cc01b1a96..fb98901205 100644 --- a/ydb/library/yql/providers/pq/cm_client/lib/client.h +++ b/ydb/library/yql/providers/pq/cm_client/lib/client.h @@ -1,5 +1,5 @@ #pragma once - + #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> namespace NPq::NConfigurationManager { diff --git a/ydb/library/yql/providers/pq/common/yql_names.h b/ydb/library/yql/providers/pq/common/yql_names.h index e4ac47c5cd..d4d8ffc529 100644 --- a/ydb/library/yql/providers/pq/common/yql_names.h +++ b/ydb/library/yql/providers/pq/common/yql_names.h @@ -8,6 +8,6 @@ constexpr TStringBuf PartitionsCountProp = "PartitionsCount"; constexpr TStringBuf ConsumerSetting = "Consumer"; constexpr TStringBuf EndpointSetting = "Endpoint"; constexpr TStringBuf UseSslSetting = "UseSsl"; -constexpr TStringBuf AddBearerToTokenSetting = "AddBearerToToken"; +constexpr TStringBuf AddBearerToTokenSetting = "AddBearerToToken"; } // namespace NYql diff --git a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h index be54a2c2ae..a455606505 100644 --- a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h +++ b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h @@ -34,27 +34,27 @@ public: } }; -class TPqDataSink : public NGenerated::TPqDataSinkStub<TExprBase, TCallable, TCoAtom> { -public: - explicit TPqDataSink(const TExprNode* node) - : TPqDataSinkStub(node) {} - - explicit TPqDataSink(const TExprNode::TPtr& node) - : TPqDataSinkStub(node) {} - - static bool Match(const TExprNode* node) { - if (!TPqDataSinkStub::Match(node)) { - return false; - } - - if (node->Head().Content() != PqProviderName) { - return false; - } - - return true; - } -}; - +class TPqDataSink : public NGenerated::TPqDataSinkStub<TExprBase, TCallable, TCoAtom> { +public: + explicit TPqDataSink(const TExprNode* node) + : TPqDataSinkStub(node) {} + + explicit TPqDataSink(const TExprNode::TPtr& node) + : TPqDataSinkStub(node) {} + + static bool Match(const TExprNode* node) { + if (!TPqDataSinkStub::Match(node)) { + return false; + } + + if (node->Head().Content() != PqProviderName) { + return false; + } + + return true; + } +}; + #include <ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.defs.inl.h> } // namespace NNodes diff --git a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json index bad47e9d70..13340d1003 100644 --- a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json +++ b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json @@ -16,16 +16,16 @@ ] }, { - "Name": "TPqDataSink", - "Base": "TCallable", - "Definition": "Custom", - "Match": {"Type": "Callable", "Name": "DataSink"}, - "Children": [ - {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} - ] - }, - { + "Name": "TPqDataSink", + "Base": "TCallable", + "Definition": "Custom", + "Match": {"Type": "Callable", "Name": "DataSink"}, + "Children": [ + {"Index": 0, "Name": "Category", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"} + ] + }, + { "Name": "TPqTopic", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "PqTopic"}, @@ -34,8 +34,8 @@ {"Index": 1, "Name": "Database", "Type": "TCoAtom"}, {"Index": 2, "Name": "Path", "Type": "TCoAtom"}, {"Index": 3, "Name": "Props", "Type": "TCoNameValueTupleList"}, - {"Index": 4, "Name": "Metadata", "Type": "TCoNameValueTupleList"}, - {"Index": 5, "Name": "RowSpec", "Type": "TExprBase"} + {"Index": 4, "Name": "Metadata", "Type": "TCoNameValueTupleList"}, + {"Index": 5, "Name": "RowSpec", "Type": "TExprBase"} ] }, { @@ -56,12 +56,12 @@ {"Index": 1, "Name": "DataSource", "Type": "TPqDataSource"}, {"Index": 2, "Name": "Topic", "Type": "TPqTopic"}, {"Index": 3, "Name": "Columns", "Type": "TExprBase"}, - {"Index": 4, "Name": "Format", "Type": "TCoAtom"}, - {"Index": 5, "Name": "Compression", "Type": "TCoAtom"}, - {"Index": 6, "Name": "LimitHint", "Type": "TExprBase", "Optional": true} + {"Index": 4, "Name": "Format", "Type": "TCoAtom"}, + {"Index": 5, "Name": "Compression", "Type": "TCoAtom"}, + {"Index": 6, "Name": "LimitHint", "Type": "TExprBase", "Optional": true} ] - }, - { + }, + { "Name": "TDqPqTopicSource", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "DqPqTopicSource"}, @@ -69,7 +69,7 @@ {"Index": 0, "Name": "Topic", "Type": "TPqTopic"}, {"Index": 1, "Name": "Columns", "Type": "TExprBase"}, {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 3, "Name": "Token", "Type": "TCoSecureParam"} + {"Index": 3, "Name": "Token", "Type": "TCoSecureParam"} ] }, { @@ -79,7 +79,7 @@ "Children": [ {"Index": 0, "Name": "Topic", "Type": "TPqTopic"}, {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"} + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"} ] }, { @@ -92,26 +92,26 @@ ] }, { - "Name": "TPqWriteTopic", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "PqWriteTopic!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TPqDataSink"}, + "Name": "TPqWriteTopic", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "PqWriteTopic!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TPqDataSink"}, {"Index": 2, "Name": "Topic", "Type": "TPqTopic"}, - {"Index": 3, "Name": "Input", "Type": "TExprBase"}, - {"Index": 4, "Name": "Mode", "Type": "TCoAtom"}, - {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"} - ] - }, - { - "Name": "TPqClusterConfig", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "PqClusterConfig"}, - "Children": [ - {"Index": 0, "Name": "Endpoint", "Type": "TCoAtom"}, - {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"} - ] + {"Index": 3, "Name": "Input", "Type": "TExprBase"}, + {"Index": 4, "Name": "Mode", "Type": "TCoAtom"}, + {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TPqClusterConfig", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "PqClusterConfig"}, + "Children": [ + {"Index": 0, "Name": "Endpoint", "Type": "TCoAtom"}, + {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"} + ] } ] } diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp index b1d64befb3..a37f5c5578 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp @@ -7,7 +7,7 @@ namespace NYql { -NThreading::TFuture<void> TDummyPqGateway::OpenSession(const TString& sessionId, const TString& username) { +NThreading::TFuture<void> TDummyPqGateway::OpenSession(const TString& sessionId, const TString& username) { with_lock (Mutex) { Y_ENSURE(sessionId); Y_ENSURE(username); @@ -25,9 +25,9 @@ void TDummyPqGateway::CloseSession(const TString& sessionId) { } } -NPq::NConfigurationManager::TAsyncDescribePathResult TDummyPqGateway::DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) { +NPq::NConfigurationManager::TAsyncDescribePathResult TDummyPqGateway::DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) { Y_UNUSED(database); - Y_UNUSED(token); + Y_UNUSED(token); with_lock (Mutex) { Y_ENSURE(IsIn(OpenedSessions, sessionId), "Session " << sessionId << " is not opened in pq gateway"); const auto key = std::make_pair(cluster, path); diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h index 753c849c20..c2fce502c6 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h @@ -31,10 +31,10 @@ public: TDummyPqGateway& AddDummyTopic(const TDummyTopic& topic); public: - NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override; + NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override; void CloseSession(const TString& sessionId) override; - NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) override; + NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) override; void UpdateClusterConfigs( const TString& clusterName, @@ -43,7 +43,7 @@ public: bool secure) override; private: - mutable TMutex Mutex; + mutable TMutex Mutex; THashMap<std::pair<TString, TString>, TDummyTopic> Topics; THashSet<TString> OpenedSessions; }; diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp index bd89565050..1fe1aa287c 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp @@ -16,7 +16,7 @@ public: explicit TPqNativeGateway(const TPqGatewayServices& services); ~TPqNativeGateway(); - NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override; + NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override; void CloseSession(const TString& sessionId) override; NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath( @@ -25,7 +25,7 @@ public: const TString& database, const TString& path, const TString& token) override; - + void UpdateClusterConfigs( const TString& clusterName, const TString& endpoint, @@ -34,14 +34,14 @@ public: private: void InitClusterConfigs(); - TPqSession::TPtr GetExistingSession(const TString& sessionId) const; + TPqSession::TPtr GetExistingSession(const TString& sessionId) const; private: - mutable TMutex Mutex; + mutable TMutex Mutex; const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; TPqGatewayConfigPtr Config; IMetricsRegistryPtr Metrics; - ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; ::NPq::NConfigurationManager::IConnections::TPtr CmConnections; NYdb::TDriver YdbDriver; TPqClusterConfigsMapPtr ClusterConfigs; @@ -52,7 +52,7 @@ TPqNativeGateway::TPqNativeGateway(const TPqGatewayServices& services) : FunctionRegistry(services.FunctionRegistry) , Config(services.Config) , Metrics(services.Metrics) - , CredentialsFactory(services.CredentialsFactory) + , CredentialsFactory(services.CredentialsFactory) , CmConnections(services.CmConnections) , YdbDriver(services.YdbDriver) { @@ -84,7 +84,7 @@ void TPqNativeGateway::UpdateClusterConfigs( } } -NThreading::TFuture<void> TPqNativeGateway::OpenSession(const TString& sessionId, const TString& username) { +NThreading::TFuture<void> TPqNativeGateway::OpenSession(const TString& sessionId, const TString& username) { with_lock (Mutex) { auto [sessionIt, isNewSession] = Sessions.emplace(sessionId, MakeIntrusive<TPqSession>(sessionId, @@ -92,7 +92,7 @@ NThreading::TFuture<void> TPqNativeGateway::OpenSession(const TString& sessionId CmConnections, YdbDriver, ClusterConfigs, - CredentialsFactory)); + CredentialsFactory)); if (!isNewSession) { YQL_LOG_CTX_THROW yexception() << "Session already exists: " << sessionId; } @@ -104,20 +104,20 @@ void TPqNativeGateway::CloseSession(const TString& sessionId) { with_lock (Mutex) { Sessions.erase(sessionId); } -} - -TPqSession::TPtr TPqNativeGateway::GetExistingSession(const TString& sessionId) const { +} + +TPqSession::TPtr TPqNativeGateway::GetExistingSession(const TString& sessionId) const { with_lock (Mutex) { auto sessionIt = Sessions.find(sessionId); if (sessionIt == Sessions.end()) { YQL_LOG_CTX_THROW yexception() << "Pq gateway session was not found: " << sessionId; } return sessionIt->second; - } + } } -NPq::NConfigurationManager::TAsyncDescribePathResult TPqNativeGateway::DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) { - return GetExistingSession(sessionId)->DescribePath(cluster, database, path, token); +NPq::NConfigurationManager::TAsyncDescribePathResult TPqNativeGateway::DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) { + return GetExistingSession(sessionId)->DescribePath(cluster, database, path, token); } IPqGateway::TPtr CreatePqNativeGateway(const TPqGatewayServices& services) { diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h index 29f05d1e95..b839f89d63 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h @@ -1,11 +1,11 @@ #pragma once - + #include <ydb/library/yql/providers/common/metrics/metrics_registry.h> #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> #include <ydb/library/yql/providers/pq/provider/yql_pq_gateway.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> - + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <util/generic/ptr.h> @@ -23,7 +23,7 @@ struct TPqGatewayServices { const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; TPqGatewayConfigPtr Config; IMetricsRegistryPtr Metrics; - ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; ::NPq::NConfigurationManager::IConnections::TPtr CmConnections; NYdb::TDriver YdbDriver; diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp index cd536762e4..143b50f256 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp @@ -4,38 +4,38 @@ namespace NYql { -namespace { -NPq::NConfigurationManager::TClientOptions GetCmClientOptions(const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { +namespace { +NPq::NConfigurationManager::TClientOptions GetCmClientOptions(const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { NPq::NConfigurationManager::TClientOptions opts; - opts - .SetEndpoint(cfg.GetConfigManagerEndpoint()) + opts + .SetEndpoint(cfg.GetConfigManagerEndpoint()) .SetCredentialsProviderFactory(credentialsProviderFactory) .SetEnableSsl(cfg.GetUseSsl()); - + return opts; } -NYdb::NPersQueue::TPersQueueClientSettings GetYdbPqClientOptions(const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { +NYdb::NPersQueue::TPersQueueClientSettings GetYdbPqClientOptions(const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { NYdb::NPersQueue::TPersQueueClientSettings opts; opts .DiscoveryEndpoint(cfg.GetEndpoint()) .Database(database) - .EnableSsl(cfg.GetUseSsl()) - .CredentialsProviderFactory(credentialsProviderFactory); - + .EnableSsl(cfg.GetUseSsl()) + .CredentialsProviderFactory(credentialsProviderFactory); + return opts; } -} +} -const NPq::NConfigurationManager::IClient::TPtr& TPqSession::GetConfigManagerClient(const TString& cluster, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { - auto& client = ClusterCmClients[cluster]; +const NPq::NConfigurationManager::IClient::TPtr& TPqSession::GetConfigManagerClient(const TString& cluster, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { + auto& client = ClusterCmClients[cluster]; if (!client && CmConnections) { - client = CmConnections->GetClient(GetCmClientOptions(cfg, credentialsProviderFactory)); - } - return client; -} - -NYdb::NPersQueue::TPersQueueClient& TPqSession::GetYdbPqClient(const TString& cluster, const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { + client = CmConnections->GetClient(GetCmClientOptions(cfg, credentialsProviderFactory)); + } + return client; +} + +NYdb::NPersQueue::TPersQueueClient& TPqSession::GetYdbPqClient(const TString& cluster, const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory) { const auto clientIt = ClusterYdbPqClients.find(cluster); if (clientIt != ClusterYdbPqClients.end()) { return clientIt->second; @@ -43,17 +43,17 @@ NYdb::NPersQueue::TPersQueueClient& TPqSession::GetYdbPqClient(const TString& cl return ClusterYdbPqClients.emplace(cluster, NYdb::NPersQueue::TPersQueueClient(YdbDriver, GetYdbPqClientOptions(database, cfg, credentialsProviderFactory))).first->second; } -NPq::NConfigurationManager::TAsyncDescribePathResult TPqSession::DescribePath(const TString& cluster, const TString& database, const TString& path, const TString& token) { - const auto* config = ClusterConfigs->FindPtr(cluster); - if (!config) { +NPq::NConfigurationManager::TAsyncDescribePathResult TPqSession::DescribePath(const TString& cluster, const TString& database, const TString& path, const TString& token) { + const auto* config = ClusterConfigs->FindPtr(cluster); + if (!config) { ythrow yexception() << "Pq cluster `" << cluster << "` does not exist"; - } - + } + YQL_ENSURE(config->GetEndpoint(), "Can't describe topic `" << cluster << "`.`" << path << "`: no endpoint"); - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, token, config->GetAddBearerToToken()); + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, token, config->GetAddBearerToToken()); with_lock (Mutex) { - if (config->GetClusterType() == TPqClusterConfig::CT_PERS_QUEUE) { + if (config->GetClusterType() == TPqClusterConfig::CT_PERS_QUEUE) { const NPq::NConfigurationManager::IClient::TPtr& client = GetConfigManagerClient(cluster, *config, credentialsProviderFactory); if (!client) { NThreading::TPromise<::NPq::NConfigurationManager::TDescribePathResult> result = NThreading::NewPromise<::NPq::NConfigurationManager::TDescribePathResult>(); @@ -65,16 +65,16 @@ NPq::NConfigurationManager::TAsyncDescribePathResult TPqSession::DescribePath(co } return client->DescribePath(path); } - + return GetYdbPqClient(cluster, database, *config, credentialsProviderFactory).DescribeTopic(path).Apply([cluster, path](const NYdb::NPersQueue::TAsyncDescribeTopicResult& describeTopicResultFuture) { - const NYdb::NPersQueue::TDescribeTopicResult& describeTopicResult = describeTopicResultFuture.GetValue(); - if (!describeTopicResult.IsSuccess()) { + const NYdb::NPersQueue::TDescribeTopicResult& describeTopicResult = describeTopicResultFuture.GetValue(); + if (!describeTopicResult.IsSuccess()) { throw yexception() << "Failed to describe topic `" << cluster << "`.`" << path << "`: " << describeTopicResult.GetIssues().ToString(); } - NPq::NConfigurationManager::TTopicDescription desc(path); - desc.PartitionsCount = describeTopicResult.TopicSettings().PartitionsCount(); - return NPq::NConfigurationManager::TDescribePathResult::Make<NPq::NConfigurationManager::TTopicDescription>(std::move(desc)); - }); + NPq::NConfigurationManager::TTopicDescription desc(path); + desc.PartitionsCount = describeTopicResult.TopicSettings().PartitionsCount(); + return NPq::NConfigurationManager::TDescribePathResult::Make<NPq::NConfigurationManager::TTopicDescription>(std::move(desc)); + }); } } diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h index 23129837f0..2e386a122c 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.h @@ -1,7 +1,7 @@ #pragma once - + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - + #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> @@ -23,21 +23,21 @@ public: const NPq::NConfigurationManager::IConnections::TPtr& cmConnections, const NYdb::TDriver& ydbDriver, const TPqClusterConfigsMapPtr& clusterConfigs, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) : SessionId(sessionId) , UserName(username) , CmConnections(cmConnections) , YdbDriver(ydbDriver) , ClusterConfigs(clusterConfigs) - , CredentialsFactory(credentialsFactory) + , CredentialsFactory(credentialsFactory) { } - NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& cluster, const TString& database, const TString& path, const TString& token); + NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& cluster, const TString& database, const TString& path, const TString& token); private: - const NPq::NConfigurationManager::IClient::TPtr& GetConfigManagerClient(const TString& cluster, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory); - NYdb::NPersQueue::TPersQueueClient& GetYdbPqClient(const TString& cluster, const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory); + const NPq::NConfigurationManager::IClient::TPtr& GetConfigManagerClient(const TString& cluster, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory); + NYdb::NPersQueue::TPersQueueClient& GetYdbPqClient(const TString& cluster, const TString& database, const NYql::TPqClusterConfig& cfg, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory); private: const TString SessionId; @@ -45,11 +45,11 @@ private: const NPq::NConfigurationManager::IConnections::TPtr CmConnections; const NYdb::TDriver YdbDriver; const TPqClusterConfigsMapPtr ClusterConfigs; - const ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; - - TMutex Mutex; + const ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + + TMutex Mutex; THashMap<TString, NPq::NConfigurationManager::IClient::TPtr> ClusterCmClients; // Cluster -> CM Client. - THashMap<TString, NYdb::NPersQueue::TPersQueueClient> ClusterYdbPqClients; // Cluster -> PQ Client. + THashMap<TString, NYdb::NPersQueue::TPersQueueClient> ClusterYdbPqClients; // Cluster -> PQ Client. }; } // namespace NYql diff --git a/ydb/library/yql/providers/pq/proto/dq_io.proto b/ydb/library/yql/providers/pq/proto/dq_io.proto index e87eb72755..997f1e7046 100644 --- a/ydb/library/yql/providers/pq/proto/dq_io.proto +++ b/ydb/library/yql/providers/pq/proto/dq_io.proto @@ -22,7 +22,7 @@ message TDqPqTopicSource { string Database = 5; EClusterType ClusterType = 6; bool UseSsl = 7; - bool AddBearerToToken = 8; + bool AddBearerToToken = 8; string DatabaseId = 9; } @@ -32,6 +32,6 @@ message TDqPqTopicSink { TToken Token = 3; string Database = 4; bool UseSsl = 5; - bool AddBearerToToken = 6; + bool AddBearerToToken = 6; EClusterType ClusterType = 7; } diff --git a/ydb/library/yql/providers/pq/provider/ut/ya.make b/ydb/library/yql/providers/pq/provider/ut/ya.make index 66015a4f05..dd47181ab4 100644 --- a/ydb/library/yql/providers/pq/provider/ut/ya.make +++ b/ydb/library/yql/providers/pq/provider/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(ydb/library/yql/providers/pq/provider) - + OWNER(g:yql) - -SRCS( - yql_pq_ut.cpp -) - -PEERDIR( + +SRCS( + yql_pq_ut.cpp +) + +PEERDIR( ydb/core/yq/libs/db_resolver ydb/library/yql/core/facade ydb/library/yql/core/file_storage @@ -23,17 +23,17 @@ PEERDIR( ydb/library/yql/providers/pq/provider ydb/library/yql/providers/solomon/gateway ydb/library/yql/providers/solomon/provider -) - -YQL_LAST_ABI_VERSION() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(1800) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -END() +) + +YQL_LAST_ABI_VERSION() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +END() diff --git a/ydb/library/yql/providers/pq/provider/ya.make b/ydb/library/yql/providers/pq/provider/ya.make index ead04a48fc..b9107dd21e 100644 --- a/ydb/library/yql/providers/pq/provider/ya.make +++ b/ydb/library/yql/providers/pq/provider/ya.make @@ -14,13 +14,13 @@ SRCS( yql_pq_datasource.cpp yql_pq_datasource_type_ann.cpp yql_pq_dq_integration.cpp - yql_pq_io_discovery.cpp + yql_pq_io_discovery.cpp yql_pq_load_meta.cpp yql_pq_logical_opt.cpp yql_pq_mkql_compiler.cpp yql_pq_physical_optimize.cpp yql_pq_provider.cpp - yql_pq_provider_impl.cpp + yql_pq_provider_impl.cpp yql_pq_settings.cpp yql_pq_topic_key_parser.cpp yql_pq_helpers.cpp @@ -60,7 +60,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() - + IF (NOT OPENSOURCE) RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp index 380df33d52..a8ba7ee83a 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp @@ -17,22 +17,22 @@ using namespace NNodes; namespace { -void ScanPlanDependencies(const TExprNode::TPtr& input, TExprNode::TListType& children) { - VisitExpr(input, [&children](const TExprNode::TPtr& node) { - if (node->IsCallable("DqCnResult")) { - children.push_back(node->Child(0)); - return false; - } - - return true; - }); -} - -class TPqDataSinkProvider : public TDataProviderBase { +void ScanPlanDependencies(const TExprNode::TPtr& input, TExprNode::TListType& children) { + VisitExpr(input, [&children](const TExprNode::TPtr& node) { + if (node->IsCallable("DqCnResult")) { + children.push_back(node->Child(0)); + return false; + } + + return true; + }); +} + +class TPqDataSinkProvider : public TDataProviderBase { public: - TPqDataSinkProvider(TPqState::TPtr state, IPqGateway::TPtr gateway) + TPqDataSinkProvider(TPqState::TPtr state, IPqGateway::TPtr gateway) : State_(state) - , Gateway_(gateway) + , Gateway_(gateway) , IODiscoveryTransformer_(CreatePqDataSinkIODiscoveryTransformer(State_)) , TypeAnnotationTransformer_(CreatePqDataSinkTypeAnnotationTransformer(State_)) , ExecutionTransformer_(CreatePqDataSinkExecTransformer(State_)) @@ -46,10 +46,10 @@ public: } bool CanParse(const TExprNode& node) override { - if (node.IsCallable(TCoWrite::CallableName())) { + if (node.IsCallable(TCoWrite::CallableName())) { return TPqDataSink::Match(node.Child(1)); - } - + } + return TypeAnnotationTransformer_->CanParse(node); } @@ -95,15 +95,15 @@ public: return *PhysicalOptProposalTransformer_; } - TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { + TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { auto maybePqWrite = TMaybeNode<TPqWrite>(node); YQL_ENSURE(maybePqWrite.DataSink(), "Expected Write!, got: " << node->Content()); - YQL_CLOG(INFO, ProviderPq) << "Rewrite " << node->Content(); - const TCoWrite write(node); - TTopicKeyParser key; - YQL_ENSURE(key.Parse(*node->Child(2), nullptr, ctx), "Failed to extract topic name."); - const auto settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); + YQL_CLOG(INFO, ProviderPq) << "Rewrite " << node->Content(); + const TCoWrite write(node); + TTopicKeyParser key; + YQL_ENSURE(key.Parse(*node->Child(2), nullptr, ctx), "Failed to extract topic name."); + const auto settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); YQL_ENSURE(settings.Mode.Cast() == "append", "Only append write mode is supported for writing into topic"); const auto cluster = TString(maybePqWrite.Cast().DataSink().Cluster().Value()); @@ -116,64 +116,64 @@ public: auto topicNode = Build<TPqTopic>(ctx, write.Pos()) .Cluster().Value(cluster).Build() .Database().Value(State_->Configuration->GetDatabaseForTopic(cluster)).Build() - .RowSpec(found->RowSpec) + .RowSpec(found->RowSpec) .Path().Value(key.GetTopicPath()).Build() .Props(BuildTopicPropsList(*found, write.Pos(), ctx)) .Metadata().Build() .Done(); return Build<TPqWriteTopic>(ctx, node->Pos()) - .World(write.World()) - .DataSink(write.DataSink().Ptr()) + .World(write.World()) + .DataSink(write.DataSink().Ptr()) .Topic(topicNode) - .Input(node->Child(3)) - .Mode(settings.Mode.Cast()) - .Settings(settings.Other) - .Done().Ptr(); - } - - TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { + .Input(node->Child(3)) + .Mode(settings.Mode.Cast()) + .Settings(settings.Other) + .Done().Ptr(); + } + + TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override { const auto* config = State_->Configuration->ClustersConfigurationSettings.FindPtr(cluster); - if (!config) { - return {}; - } - - TPositionHandle pos; - return Build<NNodes::TPqClusterConfig>(ctx, pos) + if (!config) { + return {}; + } + + TPositionHandle pos; + return Build<NNodes::TPqClusterConfig>(ctx, pos) .Endpoint<TCoAtom>().Build(config->Endpoint) .TvmId<TCoAtom>().Build(ToString(config->TvmId)) - .Done().Ptr(); - } - - void GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) override { - if (auto maybeOp = TMaybeNode<TPqWriteTopic>(&node)) { - auto op = maybeOp.Cast(); - outputs.push_back(TPinInfo(nullptr, op.DataSink().Raw(), op.Topic().Raw(), MakeTopicDisplayName(op.Topic().Cluster().Value(), op.Topic().Path().Value()), false)); - } - } - - bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { - Y_UNUSED(compact); - if (CanExecute(node)) { - children.push_back(node.ChildPtr(0)); - - if (TMaybeNode<TPqWriteTopic>(&node)) { - ScanPlanDependencies(node.ChildPtr(TPqWriteTopic::idx_Input), children); - } - - return true; - } - - return false; - } - + .Done().Ptr(); + } + + void GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) override { + if (auto maybeOp = TMaybeNode<TPqWriteTopic>(&node)) { + auto op = maybeOp.Cast(); + outputs.push_back(TPinInfo(nullptr, op.DataSink().Raw(), op.Topic().Raw(), MakeTopicDisplayName(op.Topic().Cluster().Value(), op.Topic().Path().Value()), false)); + } + } + + bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { + Y_UNUSED(compact); + if (CanExecute(node)) { + children.push_back(node.ChildPtr(0)); + + if (TMaybeNode<TPqWriteTopic>(&node)) { + ScanPlanDependencies(node.ChildPtr(TPqWriteTopic::idx_Input), children); + } + + return true; + } + + return false; + } + IDqIntegration* GetDqIntegration() override { return State_->DqIntegration.Get(); } private: TPqState::TPtr State_; - IPqGateway::TPtr Gateway_; + IPqGateway::TPtr Gateway_; THolder<IGraphTransformer> IODiscoveryTransformer_; THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; THolder<TExecTransformerBase> ExecutionTransformer_; @@ -183,8 +183,8 @@ private: } -TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway::TPtr gateway) { - return new TPqDataSinkProvider(state, gateway); +TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway::TPtr gateway) { + return new TPqDataSinkProvider(state, gateway); } } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_type_ann.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_type_ann.cpp index 6109bbe42e..8a79954bef 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_type_ann.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_type_ann.cpp @@ -15,33 +15,33 @@ namespace NYql { using namespace NNodes; namespace { -bool EnsureStructTypeWithSingleStringMember(const TTypeAnnotationNode* input, TPositionHandle pos, TExprContext& ctx) { - YQL_ENSURE(input); - auto itemSchema = input->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - if (itemSchema->GetSize() != 1) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "only struct with single string, yson or json field is accepted, but has struct with " << itemSchema->GetSize() << " members")); - return false; - } - - auto column = itemSchema->GetItems()[0]; - auto columnType = column->GetItemType(); - if (columnType->GetKind() != ETypeAnnotationKind::Data) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Column " << column->GetName() << " must have a data type, but has " << columnType->GetKind())); - return false; - } - - auto columnDataType = columnType->Cast<TDataExprType>(); - auto dataSlot = columnDataType->GetSlot(); - - if (dataSlot != NUdf::EDataSlot::String && - dataSlot != NUdf::EDataSlot::Yson && - dataSlot != NUdf::EDataSlot::Json) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Column " << column->GetName() << " is not a string, yson or json, but " << NUdf::GetDataTypeInfo(dataSlot).Name)); - return false; - } - return true; -} - +bool EnsureStructTypeWithSingleStringMember(const TTypeAnnotationNode* input, TPositionHandle pos, TExprContext& ctx) { + YQL_ENSURE(input); + auto itemSchema = input->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + if (itemSchema->GetSize() != 1) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "only struct with single string, yson or json field is accepted, but has struct with " << itemSchema->GetSize() << " members")); + return false; + } + + auto column = itemSchema->GetItems()[0]; + auto columnType = column->GetItemType(); + if (columnType->GetKind() != ETypeAnnotationKind::Data) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Column " << column->GetName() << " must have a data type, but has " << columnType->GetKind())); + return false; + } + + auto columnDataType = columnType->Cast<TDataExprType>(); + auto dataSlot = columnDataType->GetSlot(); + + if (dataSlot != NUdf::EDataSlot::String && + dataSlot != NUdf::EDataSlot::Yson && + dataSlot != NUdf::EDataSlot::Json) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Column " << column->GetName() << " is not a string, yson or json, but " << NUdf::GetDataTypeInfo(dataSlot).Name)); + return false; + } + return true; +} + class TPqDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase { public: TPqDataSinkTypeAnnotationTransformer(TPqState::TPtr state) @@ -50,8 +50,8 @@ public: { using TSelf = TPqDataSinkTypeAnnotationTransformer; AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit)); - AddHandler({TPqWriteTopic::CallableName() }, Hndl(&TSelf::HandleWriteTopic)); - AddHandler({NNodes::TPqClusterConfig::CallableName() }, Hndl(&TSelf::HandleClusterConfig)); + AddHandler({TPqWriteTopic::CallableName() }, Hndl(&TSelf::HandleWriteTopic)); + AddHandler({NNodes::TPqClusterConfig::CallableName() }, Hndl(&TSelf::HandleClusterConfig)); AddHandler({TDqPqTopicSink::CallableName()}, Hndl(&TSelf::HandleDqPqTopicSink)); } @@ -61,33 +61,33 @@ public: return TStatus::Ok; } - TStatus HandleWriteTopic(TExprBase input, TExprContext& ctx) { - const auto write = input.Cast<TPqWriteTopic>(); - const auto& writeInput = write.Input().Ref(); - if (!EnsureStructTypeWithSingleStringMember(writeInput.GetTypeAnn(), writeInput.Pos(), ctx)) { - return TStatus::Error; - } - - input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - TStatus HandleClusterConfig(TExprBase input, TExprContext& ctx) { - const auto config = input.Cast<NNodes::TPqClusterConfig>(); - if (!EnsureAtom(config.Endpoint().Ref(), ctx)) { - return TStatus::Error; - } - - if (!EnsureAtom(config.TvmId().Ref(), ctx)) { - return TStatus::Error; - } - - input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - return TStatus::Ok; - } - + TStatus HandleWriteTopic(TExprBase input, TExprContext& ctx) { + const auto write = input.Cast<TPqWriteTopic>(); + const auto& writeInput = write.Input().Ref(); + if (!EnsureStructTypeWithSingleStringMember(writeInput.GetTypeAnn(), writeInput.Pos(), ctx)) { + return TStatus::Error; + } + + input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + TStatus HandleClusterConfig(TExprBase input, TExprContext& ctx) { + const auto config = input.Cast<NNodes::TPqClusterConfig>(); + if (!EnsureAtom(config.Endpoint().Ref(), ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(config.TvmId().Ref(), ctx)) { + return TStatus::Error; + } + + input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); + return TStatus::Ok; + } + TStatus HandleDqPqTopicSink(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 3, ctx)) { + if (!EnsureArgsCount(*input, 3, ctx)) { return TStatus::Error; } input->SetTypeAnn(ctx.MakeType<TVoidExprType>()); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp index 4d43adc769..2e234700d0 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp @@ -19,17 +19,17 @@ namespace NYql { using namespace NNodes; -class TPqDataSourceProvider : public TDataProviderBase { +class TPqDataSourceProvider : public TDataProviderBase { public: - TPqDataSourceProvider(TPqState::TPtr state, IPqGateway::TPtr gateway) + TPqDataSourceProvider(TPqState::TPtr state, IPqGateway::TPtr gateway) : State_(state) - , Gateway_(gateway) + , Gateway_(gateway) , ConfigurationTransformer_([this]() { - return MakeHolder<NCommon::TProviderConfigurationTransformer>(State_->Configuration, *State_->Types, TString{ PqProviderName }); - }) + return MakeHolder<NCommon::TProviderConfigurationTransformer>(State_->Configuration, *State_->Types, TString{ PqProviderName }); + }) , LoadMetaDataTransformer_(CreatePqLoadTopicMetadataTransformer(State_)) , TypeAnnotationTransformer_(CreatePqDataSourceTypeAnnotationTransformer(State_)) - , IODiscoveryTransformer_(CreatePqIODiscoveryTransformer(State_)) { + , IODiscoveryTransformer_(CreatePqIODiscoveryTransformer(State_)) { } TStringBuf GetName() const override { @@ -43,7 +43,7 @@ public: const auto& clusterSettings = State_->Configuration->ClustersConfigurationSettings; if (clusterName != NCommon::ALL_CLUSTERS && !clusterSettings.FindPtr(clusterName)) { ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << - "Unknown cluster name: " << clusterName)); + "Unknown cluster name: " << clusterName)); return false; } cluster = clusterName; @@ -61,10 +61,10 @@ public: return TypeAnnotationTransformer_->CanParse(node); } - IGraphTransformer& GetIODiscoveryTransformer() override { - return *IODiscoveryTransformer_; - } - + IGraphTransformer& GetIODiscoveryTransformer() override { + return *IODiscoveryTransformer_; + } + IGraphTransformer& GetConfigurationTransformer() override { return *ConfigurationTransformer_; } @@ -83,7 +83,7 @@ public: } TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override { - Y_UNUSED(ctx); + Y_UNUSED(ctx); YQL_CLOG(INFO, ProviderPq) << "RewriteIO"; if (auto left = TMaybeNode<TCoLeft>(node)) { return left.Input().Maybe<TPqRead>().World().Cast().Ptr(); @@ -94,7 +94,7 @@ public: return MakeIntrusive<TIssue>(ctx.GetPosition(read.Pos()), TStringBuilder() << "At function: " << TCoRead::CallableName()); }); - TTopicKeyParser topicKeyParser(read.FreeArgs().Get(2).Ref(), read.Ref().Child(4), ctx); + TTopicKeyParser topicKeyParser(read.FreeArgs().Get(2).Ref(), read.Ref().Child(4), ctx); const TString cluster(read.DataSource().Cluster().Value()); const auto* topicMeta = State_->FindTopicMeta(cluster, topicKeyParser.GetTopicPath()); if (!topicMeta) { @@ -106,27 +106,27 @@ public: .Cluster().Value(cluster).Build() .Database().Value(State_->Configuration->GetDatabaseForTopic(cluster)).Build() .Path().Value(topicKeyParser.GetTopicPath()).Build() - .RowSpec(topicMeta->RowSpec) + .RowSpec(topicMeta->RowSpec) .Props(BuildTopicPropsList(*topicMeta, read.Pos(), ctx)) .Metadata().Build() .Done(); - auto builder = Build<TPqReadTopic>(ctx, read.Pos()) - .World(read.World()) - .DataSource(read.DataSource()) - .Topic(std::move(topicNode)) - .Format().Value(topicKeyParser.GetFormat()).Build() - .Compression().Value(topicKeyParser.GetCompression()).Build(); + auto builder = Build<TPqReadTopic>(ctx, read.Pos()) + .World(read.World()) + .DataSource(read.DataSource()) + .Topic(std::move(topicNode)) + .Format().Value(topicKeyParser.GetFormat()).Build() + .Compression().Value(topicKeyParser.GetCompression()).Build(); - if (topicKeyParser.GetColumnOrder()) { - builder.Columns(topicKeyParser.GetColumnOrder()); + if (topicKeyParser.GetColumnOrder()) { + builder.Columns(topicKeyParser.GetColumnOrder()); } else { - builder.Columns<TCoVoid>().Build(); - } + builder.Columns<TCoVoid>().Build(); + } - return Build<TCoRight>(ctx, read.Pos()) - .Input(builder.Done()) - .Done().Ptr(); + return Build<TCoRight>(ctx, read.Pos()) + .Input(builder.Done()) + .Done().Ptr(); } const THashMap<TString, TString>* GetClusterTokens() override { @@ -149,12 +149,12 @@ public: void GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) override { if (auto maybeRead = TMaybeNode<TPqReadTopic>(&node)) { if (auto maybeTopic = maybeRead.Topic()) { - TStringBuf cluster; + TStringBuf cluster; if (auto dataSource = maybeRead.DataSource().Maybe<TPqDataSource>()) { - cluster = dataSource.Cast().Cluster().Value(); + cluster = dataSource.Cast().Cluster().Value(); } - auto topicDisplayName = MakeTopicDisplayName(cluster, maybeTopic.Cast().Path().Value()); - inputs.push_back(TPinInfo(maybeRead.DataSource().Raw(), nullptr, maybeTopic.Cast().Raw(), topicDisplayName, false)); + auto topicDisplayName = MakeTopicDisplayName(cluster, maybeTopic.Cast().Path().Value()); + inputs.push_back(TPinInfo(maybeRead.DataSource().Raw(), nullptr, maybeTopic.Cast().Raw(), topicDisplayName, false)); } } } @@ -165,15 +165,15 @@ public: private: TPqState::TPtr State_; - IPqGateway::TPtr Gateway_; + IPqGateway::TPtr Gateway_; TLazyInitHolder<IGraphTransformer> ConfigurationTransformer_; THolder<IGraphTransformer> LoadMetaDataTransformer_; THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; - THolder<IGraphTransformer> IODiscoveryTransformer_; + THolder<IGraphTransformer> IODiscoveryTransformer_; }; -TIntrusivePtr<IDataProvider> CreatePqDataSource(TPqState::TPtr state, IPqGateway::TPtr gateway) { - return new TPqDataSourceProvider(state, gateway); +TIntrusivePtr<IDataProvider> CreatePqDataSource(TPqState::TPtr state, IPqGateway::TPtr gateway) { + return new TPqDataSourceProvider(state, gateway); } } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp index 1f884336ec..c893c8839c 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp @@ -17,7 +17,7 @@ namespace { class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { public: - explicit TPqDataSourceTypeAnnotationTransformer(TPqState::TPtr state) + explicit TPqDataSourceTypeAnnotationTransformer(TPqState::TPtr state) : TVisitorTransformerBase(true) , State_(state) { @@ -45,12 +45,12 @@ public: return TStatus::Ok; } - const TTypeAnnotationNode* GetReadTopicSchema(TPqTopic topic, TMaybeNode<TCoAtomList> columns, TExprBase input, TExprContext& ctx, TVector<TString>& columnOrder) { + const TTypeAnnotationNode* GetReadTopicSchema(TPqTopic topic, TMaybeNode<TCoAtomList> columns, TExprBase input, TExprContext& ctx, TVector<TString>& columnOrder) { auto schema = topic.Ref().GetTypeAnn(); if (columns) { TVector<const TItemExprType*> items; items.reserve(columns.Cast().Ref().ChildrenSize()); - columnOrder.reserve(items.capacity()); + columnOrder.reserve(items.capacity()); auto itemSchema = topic.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); for (auto c : columns.Cast().Ref().ChildrenList()) { @@ -62,7 +62,7 @@ public: ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unable to find column: " << c->Content())); return nullptr; } - columnOrder.push_back(TString(c->Content())); + columnOrder.push_back(TString(c->Content())); items.push_back(itemSchema->GetItems()[*index]); } schema = ctx.MakeType<TListExprType>(ctx.MakeType<TStructExprType>(items)); @@ -71,7 +71,7 @@ public: } TStatus HandleReadTopic(TExprBase input, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(input.Ref(), 6, 7, ctx)) { + if (!EnsureMinMaxArgsCount(input.Ref(), 6, 7, ctx)) { return TStatus::Error; } @@ -90,21 +90,21 @@ public: return TStatus::Error; } - TVector<TString> columnOrder; - auto schema = GetReadTopicSchema(topic, read.Columns().Maybe<TCoAtomList>(), input, ctx, columnOrder); + TVector<TString> columnOrder; + auto schema = GetReadTopicSchema(topic, read.Columns().Maybe<TCoAtomList>(), input, ctx, columnOrder); if (!schema) { return TStatus::Error; - } - + } + input.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{ read.World().Ref().GetTypeAnn(), - schema + schema })); - return State_->Types->SetColumnOrder(input.Ref(), columnOrder, ctx); + return State_->Types->SetColumnOrder(input.Ref(), columnOrder, ctx); } TStatus HandleDqTopicSource(TExprBase input, TExprContext& ctx) { - if (!EnsureArgsCount(input.Ref(), 4, ctx)) { + if (!EnsureArgsCount(input.Ref(), 4, ctx)) { return TStatus::Error; } @@ -115,42 +115,42 @@ public: return TStatus::Error; } - const auto cluster = TString(topic.Cluster().Value()); - const auto topicPath = TString(topic.Path().Value()); - const auto* meta = State_->FindTopicMeta(cluster, topicPath); - if (!meta) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unknown topic `" << cluster << "`.`" << topicPath << "`")); + const auto cluster = TString(topic.Cluster().Value()); + const auto topicPath = TString(topic.Path().Value()); + const auto* meta = State_->FindTopicMeta(cluster, topicPath); + if (!meta) { + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unknown topic `" << cluster << "`.`" << topicPath << "`")); return TStatus::Error; } - input.Ptr()->SetTypeAnn(ctx.MakeType<TStreamExprType>(ctx.MakeType<TDataExprType>(EDataSlot::String))); + input.Ptr()->SetTypeAnn(ctx.MakeType<TStreamExprType>(ctx.MakeType<TDataExprType>(EDataSlot::String))); return TStatus::Ok; } - TStatus HandleTopic(const TExprNode::TPtr& input, TExprContext& ctx) { + TStatus HandleTopic(const TExprNode::TPtr& input, TExprContext& ctx) { if (State_->IsRtmrMode()) { - return HandleTopicInRtmrMode(input, ctx); - } - + return HandleTopicInRtmrMode(input, ctx); + } + TPqTopic topic(input); - input->SetTypeAnn(ctx.MakeType<TListExprType>(topic.RowSpec().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>())); - return TStatus::Ok; - } - -private: - TStatus HandleTopicInRtmrMode(const TExprNode::TPtr& input, TExprContext& ctx) { - TVector<const TItemExprType*> items; - auto stringType = ctx.MakeType<TDataExprType>(EDataSlot::String); - items.push_back(ctx.MakeType<TItemExprType>("key", stringType)); - items.push_back(ctx.MakeType<TItemExprType>("subkey", stringType)); - items.push_back(ctx.MakeType<TItemExprType>("value", stringType)); - auto itemType = ctx.MakeType<TStructExprType>(items); - - input->SetTypeAnn(ctx.MakeType<TListExprType>(itemType)); + input->SetTypeAnn(ctx.MakeType<TListExprType>(topic.RowSpec().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>())); return TStatus::Ok; } private: + TStatus HandleTopicInRtmrMode(const TExprNode::TPtr& input, TExprContext& ctx) { + TVector<const TItemExprType*> items; + auto stringType = ctx.MakeType<TDataExprType>(EDataSlot::String); + items.push_back(ctx.MakeType<TItemExprType>("key", stringType)); + items.push_back(ctx.MakeType<TItemExprType>("subkey", stringType)); + items.push_back(ctx.MakeType<TItemExprType>("value", stringType)); + auto itemType = ctx.MakeType<TStructExprType>(items); + + input->SetTypeAnn(ctx.MakeType<TListExprType>(itemType)); + return TStatus::Ok; + } + +private: TPqState::TPtr State_; }; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp index ea67ffd422..e7ab82b136 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp @@ -1,5 +1,5 @@ #include "yql_pq_dq_integration.h" -#include "yql_pq_helpers.h" +#include "yql_pq_helpers.h" #include "yql_pq_mkql_compiler.h" #include <ydb/library/yql/ast/yql_expr.h> @@ -23,7 +23,7 @@ namespace { class TPqDqIntegration: public TDqIntegrationBase { public: - explicit TPqDqIntegration(const TPqState::TPtr& state) + explicit TPqDqIntegration(const TPqState::TPtr& state) : State_(state.Get()) { } @@ -56,9 +56,9 @@ public: } ui64 Partition(const TDqSettings&, size_t maxPartitions, const TExprNode& node, TVector<TString>& partitions, TString*, TExprContext&, bool) override { - if (auto maybePqRead = TMaybeNode<TPqReadTopic>(&node)) { - return PartitionTopicRead(maybePqRead.Cast().Topic(), maxPartitions, partitions); - } + if (auto maybePqRead = TMaybeNode<TPqReadTopic>(&node)) { + return PartitionTopicRead(maybePqRead.Cast().Topic(), maxPartitions, partitions); + } if (auto maybeDqSource = TMaybeNode<TDqSource>(&node)) { auto settings = maybeDqSource.Cast().Settings(); if (auto topicSource = TMaybeNode<TDqPqTopicSource>(settings.Raw())) { @@ -69,52 +69,52 @@ public: } TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) override { - if (const auto& maybePqReadTopic = TMaybeNode<TPqReadTopic>(read)) { - const auto& pqReadTopic = maybePqReadTopic.Cast(); - - const auto rowType = pqReadTopic.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems().back()->Cast<TListExprType>()->GetItemType(); - const auto& clusterName = pqReadTopic.DataSource().Cluster().StringValue(); - const auto settings = Build<TCoNameValueTupleList>(ctx, pqReadTopic.Topic().Pos()) - .Add() - .Name().Value("format").Build() - .Value(pqReadTopic.Format()) - .Build() - .Done(); - - const auto token = "cluster:default_" + clusterName; - auto columns = pqReadTopic.Columns().Ptr(); - if (!columns->IsList()) { - const auto pos = columns->Pos(); - const auto& items = rowType->Cast<TStructExprType>()->GetItems(); - TExprNode::TListType cols; - cols.reserve(items.size()); - std::transform(items.cbegin(), items.cend(), std::back_inserter(cols), [&](const TItemExprType* item) { return ctx.NewAtom(pos, item->GetName()); }); - columns = ctx.NewList(pos, std::move(cols)); - } - - return Build<TDqSourceWrap>(ctx, read->Pos()) - .Input<TDqPqTopicSource>() - .Topic(pqReadTopic.Topic()) - .Columns(std::move(columns)) - .Settings(BuildTopicReadSettings(clusterName, read->Pos(), ctx)) - .Token<TCoSecureParam>() - .Name().Build(token) - .Build() - .Build() - .RowType(ExpandType(pqReadTopic.Pos(), *rowType, ctx)) - .DataSource(pqReadTopic.DataSource().Cast<TCoDataSource>()) - .Settings(settings) - .Done().Ptr(); - } + if (const auto& maybePqReadTopic = TMaybeNode<TPqReadTopic>(read)) { + const auto& pqReadTopic = maybePqReadTopic.Cast(); + + const auto rowType = pqReadTopic.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems().back()->Cast<TListExprType>()->GetItemType(); + const auto& clusterName = pqReadTopic.DataSource().Cluster().StringValue(); + const auto settings = Build<TCoNameValueTupleList>(ctx, pqReadTopic.Topic().Pos()) + .Add() + .Name().Value("format").Build() + .Value(pqReadTopic.Format()) + .Build() + .Done(); + + const auto token = "cluster:default_" + clusterName; + auto columns = pqReadTopic.Columns().Ptr(); + if (!columns->IsList()) { + const auto pos = columns->Pos(); + const auto& items = rowType->Cast<TStructExprType>()->GetItems(); + TExprNode::TListType cols; + cols.reserve(items.size()); + std::transform(items.cbegin(), items.cend(), std::back_inserter(cols), [&](const TItemExprType* item) { return ctx.NewAtom(pos, item->GetName()); }); + columns = ctx.NewList(pos, std::move(cols)); + } + + return Build<TDqSourceWrap>(ctx, read->Pos()) + .Input<TDqPqTopicSource>() + .Topic(pqReadTopic.Topic()) + .Columns(std::move(columns)) + .Settings(BuildTopicReadSettings(clusterName, read->Pos(), ctx)) + .Token<TCoSecureParam>() + .Name().Build(token) + .Build() + .Build() + .RowType(ExpandType(pqReadTopic.Pos(), *rowType, ctx)) + .DataSource(pqReadTopic.DataSource().Cast<TCoDataSource>()) + .Settings(settings) + .Done().Ptr(); + } return read; } TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override { - YQL_ENSURE(false, "Unimplemented"); + YQL_ENSURE(false, "Unimplemented"); } void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override { - RegisterDqPqMkqlCompilers(compiler); + RegisterDqPqMkqlCompilers(compiler); } static TStringBuf Name(const TCoNameValueTuple& nameValue) { @@ -127,17 +127,17 @@ public: YQL_ENSURE(value.IsAtom()); return value.Content(); } - - return {}; + + return {}; } static NPq::NProto::EClusterType ToClusterType(NYql::TPqClusterConfig::EClusterType t) { switch (t) { - case NYql::TPqClusterConfig::CT_UNSPECIFIED: + case NYql::TPqClusterConfig::CT_UNSPECIFIED: return NPq::NProto::Unspecified; - case NYql::TPqClusterConfig::CT_PERS_QUEUE: + case NYql::TPqClusterConfig::CT_PERS_QUEUE: return NPq::NProto::PersQueue; - case NYql::TPqClusterConfig::CT_DATA_STREAMS: + case NYql::TPqClusterConfig::CT_DATA_STREAMS: return NPq::NProto::DataStreams; } } @@ -168,12 +168,12 @@ public: srcDesc.SetEndpoint(TString(Value(setting))); } else if (name == UseSslSetting) { srcDesc.SetUseSsl(FromString<bool>(Value(setting))); - } else if (name == AddBearerToTokenSetting) { - srcDesc.SetAddBearerToToken(FromString<bool>(Value(setting))); + } else if (name == AddBearerToTokenSetting) { + srcDesc.SetAddBearerToToken(FromString<bool>(Value(setting))); } } - if (auto maybeToken = TMaybeNode<TCoSecureParam>(topicSource.Token().Raw())) { + if (auto maybeToken = TMaybeNode<TCoSecureParam>(topicSource.Token().Raw())) { srcDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value())); } @@ -210,12 +210,12 @@ public: sinkDesc.SetEndpoint(TString(Value(setting))); } else if (name == UseSslSetting) { sinkDesc.SetUseSsl(FromString<bool>(Value(setting))); - } else if (name == AddBearerToTokenSetting) { - sinkDesc.SetAddBearerToToken(FromString<bool>(Value(setting))); + } else if (name == AddBearerToTokenSetting) { + sinkDesc.SetAddBearerToToken(FromString<bool>(Value(setting))); } } - if (auto maybeToken = TMaybeNode<TCoSecureParam>(topicSink.Token().Raw())) { + if (auto maybeToken = TMaybeNode<TCoSecureParam>(topicSink.Token().Raw())) { sinkDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value())); } @@ -225,35 +225,35 @@ public: } } - NNodes::TCoNameValueTupleList BuildTopicReadSettings(const TString& cluster, TPositionHandle pos, TExprContext& ctx) const { - TVector<TCoNameValueTuple> props; - - { - TMaybe<TString> consumer = State_->Configuration->Consumer.Get(); + NNodes::TCoNameValueTupleList BuildTopicReadSettings(const TString& cluster, TPositionHandle pos, TExprContext& ctx) const { + TVector<TCoNameValueTuple> props; + + { + TMaybe<TString> consumer = State_->Configuration->Consumer.Get(); if (consumer) { Add(props, ConsumerSetting, *consumer, pos, ctx); - } - } - - auto clusterConfiguration = State_->Configuration->ClustersConfigurationSettings.FindPtr(cluster); - if (!clusterConfiguration) { - ythrow yexception() << "Unknown pq cluster \"" << cluster << "\""; - } - - Add(props, EndpointSetting, clusterConfiguration->Endpoint, pos, ctx); - if (clusterConfiguration->UseSsl) { - Add(props, UseSslSetting, "1", pos, ctx); - } - - if (clusterConfiguration->AddBearerToToken) { - Add(props, AddBearerToTokenSetting, "1", pos, ctx); - } - - return Build<TCoNameValueTupleList>(ctx, pos) - .Add(props) - .Done(); - } - + } + } + + auto clusterConfiguration = State_->Configuration->ClustersConfigurationSettings.FindPtr(cluster); + if (!clusterConfiguration) { + ythrow yexception() << "Unknown pq cluster \"" << cluster << "\""; + } + + Add(props, EndpointSetting, clusterConfiguration->Endpoint, pos, ctx); + if (clusterConfiguration->UseSsl) { + Add(props, UseSslSetting, "1", pos, ctx); + } + + if (clusterConfiguration->AddBearerToToken) { + Add(props, AddBearerToTokenSetting, "1", pos, ctx); + } + + return Build<TCoNameValueTupleList>(ctx, pos) + .Add(props) + .Done(); + } + private: TPqState* State_; // State owns dq integration, so back reference must be not smart. }; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h index 118b37c012..7f6c422005 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h @@ -12,11 +12,11 @@ namespace NYql { struct IPqGateway : public TThrRefBase { using TPtr = TIntrusivePtr<IPqGateway>; - virtual NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) = 0; + virtual NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) = 0; virtual void CloseSession(const TString& sessionId) = 0; // CM API. - virtual ::NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) = 0; + virtual ::NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) = 0; virtual void UpdateClusterConfigs( const TString& clusterName, diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp index f2e347cf73..b84c68edff 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp @@ -1,29 +1,29 @@ -#include "yql_pq_provider_impl.h" +#include "yql_pq_provider_impl.h" #include "yql_pq_helpers.h" - + #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h> - -namespace NYql { - -namespace { - -using namespace NNodes; - -class TPqIODiscoveryTransformer : public TGraphTransformerBase { + +namespace NYql { + +namespace { + +using namespace NNodes; + +class TPqIODiscoveryTransformer : public TGraphTransformerBase { using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; -public: - explicit TPqIODiscoveryTransformer(TPqState::TPtr state) - : State_(state) - { - } - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - Y_UNUSED(ctx); - output = input; +public: + explicit TPqIODiscoveryTransformer(TPqState::TPtr state) + : State_(state) + { + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + Y_UNUSED(ctx); + output = input; if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) return TStatus::Ok; @@ -40,32 +40,32 @@ public: *resolvedIds_ = future.GetValue(); }); return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { + } + + NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { return AsyncFuture_; - } - - TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { - output = input; + } + + TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { + output = input; AsyncFuture_.GetValue(); FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); ResolvedIds_->clear(); FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); - return TStatus::Ok; - } - -private: - const TPqState::TPtr State_; + return TStatus::Ok; + } + +private: + const TPqState::TPtr State_; NThreading::TFuture<void> AsyncFuture_; TDbId2Endpoint FullResolvedIds_; std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); -}; - -} - -THolder<IGraphTransformer> CreatePqIODiscoveryTransformer(TPqState::TPtr state) { - return THolder(new TPqIODiscoveryTransformer(state)); -} - -} +}; + +} + +THolder<IGraphTransformer> CreatePqIODiscoveryTransformer(TPqState::TPtr state) { + return THolder(new TPqIODiscoveryTransformer(state)); +} + +} diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_load_meta.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_load_meta.cpp index 51f2a1ce17..28c87e5ff8 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_load_meta.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_load_meta.cpp @@ -19,23 +19,23 @@ namespace { class TPqLoadTopicMetadataTransformer : public TGraphTransformerBase { public: - explicit TPqLoadTopicMetadataTransformer(TPqState::TPtr state) + explicit TPqLoadTopicMetadataTransformer(TPqState::TPtr state) : State_(std::move(state)) {} - void AddToPendingTopics(const TString& cluster, const TString& topicPath, TPositionHandle pos, TExprNode::TPtr rowSpec, TExprNode::TPtr columnOrder) { + void AddToPendingTopics(const TString& cluster, const TString& topicPath, TPositionHandle pos, TExprNode::TPtr rowSpec, TExprNode::TPtr columnOrder) { const auto topicKey = std::make_pair(cluster, topicPath); const auto found = State_->Topics.FindPtr(topicKey); - if (found) { - return; + if (found) { + return; } - - YQL_CLOG(INFO, ProviderPq) << "Load topic meta for: `" << cluster << "`.`" << topicPath << "`"; - TPqState::TTopicMeta m; - m.Pos = pos; - m.RowSpec = rowSpec; - m.ColumnOrder = columnOrder; - PendingTopics_.emplace(topicKey, m); + + YQL_CLOG(INFO, ProviderPq) << "Load topic meta for: `" << cluster << "`.`" << topicPath << "`"; + TPqState::TTopicMeta m; + m.Pos = pos; + m.RowSpec = rowSpec; + m.ColumnOrder = columnOrder; + PendingTopics_.emplace(topicKey, m); } TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { @@ -48,32 +48,32 @@ public: VisitExpr(input, [&](const TExprNode::TPtr& node) { if (auto maybePqRead = TMaybeNode<TPqRead>(node)) { TPqRead read = maybePqRead.Cast(); - if (read.DataSource().Category().Value() != PqProviderName) { - return true; + if (read.DataSource().Category().Value() != PqProviderName) { + return true; } - - TTopicKeyParser topicParser(read.Arg(2).Ref(), read.Ref().Child(4), ctx); - AddToPendingTopics(read.DataSource().Cluster().StringValue(), topicParser.GetTopicPath(), node->Pos(), topicParser.GetUserSchema(), topicParser.GetColumnOrder()); + + TTopicKeyParser topicParser(read.Arg(2).Ref(), read.Ref().Child(4), ctx); + AddToPendingTopics(read.DataSource().Cluster().StringValue(), topicParser.GetTopicPath(), node->Pos(), topicParser.GetUserSchema(), topicParser.GetColumnOrder()); } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) { TPqWrite write = maybePqWrite.Cast(); if (write.DataSink().Category().Value() == PqProviderName) { - TTopicKeyParser topicParser(write.Arg(2).Ref(), nullptr, ctx); - AddToPendingTopics(write.DataSink().Cluster().StringValue(), topicParser.GetTopicPath(), node->Pos(), {}, {}); + TTopicKeyParser topicParser(write.Arg(2).Ref(), nullptr, ctx); + AddToPendingTopics(write.DataSink().Cluster().StringValue(), topicParser.GetTopicPath(), node->Pos(), {}, {}); } } return true; }); - for (auto& [x, meta] : PendingTopics_) { - auto itemType = LoadTopicMeta(x.first, x.second, ctx, meta); - if (!itemType) { + for (auto& [x, meta] : PendingTopics_) { + auto itemType = LoadTopicMeta(x.first, x.second, ctx, meta); + if (!itemType) { return TStatus::Error; } - meta.RawFormat = (meta.RowSpec == nullptr); - if (!meta.RowSpec) { - meta.RowSpec = ExpandType(meta.Pos, *itemType, ctx); - } + meta.RawFormat = (meta.RowSpec == nullptr); + if (!meta.RowSpec) { + meta.RowSpec = ExpandType(meta.Pos, *itemType, ctx); + } State_->Topics.emplace(x, meta); } @@ -94,7 +94,7 @@ public: } private: - static const TStructExprType* CreateDefaultItemType(TExprContext& ctx) { + static const TStructExprType* CreateDefaultItemType(TExprContext& ctx) { // Schema for topic: // { // Data:String @@ -111,29 +111,29 @@ private: return ctx.MakeType<TStructExprType>(items); } - const TStructExprType* LoadTopicMeta(const TString& cluster, const TString& topic, TExprContext& ctx, TPqState::TTopicMeta& meta) { - // todo: return TFuture - try { - auto future = State_->Gateway->DescribePath(State_->SessionId, cluster, State_->Configuration->GetDatabaseForTopic(cluster), topic, State_->Configuration->Tokens.at(cluster)); - NPq::NConfigurationManager::TDescribePathResult description = future.GetValueSync(); - if (!description.IsTopic()) { - ctx.IssueManager.RaiseIssue(TIssue{TStringBuilder() << "Path '" << topic << "' is not a topic"}); - return {}; - } - meta.Description = description.GetTopicDescription(); - return CreateDefaultItemType(ctx); + const TStructExprType* LoadTopicMeta(const TString& cluster, const TString& topic, TExprContext& ctx, TPqState::TTopicMeta& meta) { + // todo: return TFuture + try { + auto future = State_->Gateway->DescribePath(State_->SessionId, cluster, State_->Configuration->GetDatabaseForTopic(cluster), topic, State_->Configuration->Tokens.at(cluster)); + NPq::NConfigurationManager::TDescribePathResult description = future.GetValueSync(); + if (!description.IsTopic()) { + ctx.IssueManager.RaiseIssue(TIssue{TStringBuilder() << "Path '" << topic << "' is not a topic"}); + return {}; + } + meta.Description = description.GetTopicDescription(); + return CreateDefaultItemType(ctx); } catch (const std::exception& ex) { TIssues issues; issues.AddIssue(ex.what()); ctx.IssueManager.AddIssues(issues); - return nullptr; - } - } - + return nullptr; + } + } + private: TPqState::TPtr State_; - // (cluster, topic) -> meta - THashMap<std::pair<TString, TString>, TPqState::TTopicMeta> PendingTopics_; + // (cluster, topic) -> meta + THashMap<std::pair<TString, TString>, TPqState::TTopicMeta> PendingTopics_; NThreading::TFuture<void> AsyncFuture_; }; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp index 6e8a6462d9..665dcf08bf 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp @@ -24,8 +24,8 @@ public: { #define HNDL(name) "LogicalOptimizer-"#name, Hndl(&TPqLogicalOptProposalTransformer::name) AddHandler(0, &TCoLeft::Match, HNDL(TrimReadWorld)); - // AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembers)); - // AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqWrap)); + // AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembers)); + // AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqWrap)); #undef HNDL } @@ -37,7 +37,7 @@ public: return TExprBase(ctx.NewWorld(node.Pos())); } - /* + /* TMaybeNode<TExprBase> ExtractMembers(TExprBase node, TExprContext& ctx) const { const auto& extract = node.Cast<TCoExtractMembers>(); const auto& input = extract.Input(); @@ -73,7 +73,7 @@ public: .Columns(extract.Members()) .Build() .Done(); - }*/ + }*/ private: TPqState::TPtr State_; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp index 8886ddb83e..b4f4a39e8d 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp @@ -9,25 +9,25 @@ namespace NYql { using namespace NKikimr::NMiniKQL; using namespace NNodes; -void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) { - compiler.ChainCallable(TDqSourceWideWrap::CallableName(), - [](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { - if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == PqProviderName) { +void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) { + compiler.ChainCallable(TDqSourceWideWrap::CallableName(), + [](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { + if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == PqProviderName) { const auto wrapped = TryWrapWithParser(wrapper, ctx); if (wrapped) { return *wrapped; - } - + } + const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); - auto flow = ctx.ProgramBuilder.ToFlow(input); - return ctx.ProgramBuilder.ExpandMap(flow, + auto flow = ctx.ProgramBuilder.ToFlow(input); + return ctx.ProgramBuilder.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }); - } - - return TRuntimeNode(); - }); -} - -} + } + + return TRuntimeNode(); + }); +} + +} diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.h b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.h index 5cebebcca7..8545bfb53e 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.h @@ -6,6 +6,6 @@ namespace NYql { -void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler); +void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler); } diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_physical_optimize.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_physical_optimize.cpp index a9351eb654..60acbce848 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_physical_optimize.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_physical_optimize.cpp @@ -35,20 +35,20 @@ public: NNodes::TCoNameValueTupleList BuildTopicWriteSettings(const TString& cluster, TPositionHandle pos, TExprContext& ctx) const { TVector<TCoNameValueTuple> props; - auto clusterConfiguration = State_->Configuration->ClustersConfigurationSettings.FindPtr(cluster); - if (!clusterConfiguration) { + auto clusterConfiguration = State_->Configuration->ClustersConfigurationSettings.FindPtr(cluster); + if (!clusterConfiguration) { ythrow yexception() << "Unknown pq cluster \"" << cluster << "\""; } - Add(props, EndpointSetting, clusterConfiguration->Endpoint, pos, ctx); - if (clusterConfiguration->UseSsl) { + Add(props, EndpointSetting, clusterConfiguration->Endpoint, pos, ctx); + if (clusterConfiguration->UseSsl) { Add(props, UseSslSetting, "1", pos, ctx); } - if (clusterConfiguration->AddBearerToToken) { - Add(props, AddBearerToTokenSetting, "1", pos, ctx); - } - + if (clusterConfiguration->AddBearerToToken) { + Add(props, AddBearerToTokenSetting, "1", pos, ctx); + } + return Build<TCoNameValueTupleList>(ctx, pos) .Add(props) .Done(); @@ -69,11 +69,11 @@ public: return node; } - const auto* topicMeta = State_->FindTopicMeta(topicNode); - if (!topicMeta) { - ctx.AddError(TIssue(ctx.GetPosition(write.Pos()), TStringBuilder() << "Unknown topic `" << topicNode.Cluster().StringValue() << "`.`" - << topicNode.Path().StringValue() << "`")); - return nullptr; + const auto* topicMeta = State_->FindTopicMeta(topicNode); + if (!topicMeta) { + ctx.AddError(TIssue(ctx.GetPosition(write.Pos()), TStringBuilder() << "Unknown topic `" << topicNode.Cluster().StringValue() << "`.`" + << topicNode.Path().StringValue() << "`")); + return nullptr; } YQL_CLOG(INFO, ProviderPq) << "Optimize PqWriteTopic `" << topicNode.Cluster().StringValue() << "`.`" << topicNode.Path().StringValue() << "`"; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp index b3f700c147..43415cb311 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp @@ -30,7 +30,7 @@ TDataProviderInitializer GetPqDataProviderInitializer( Y_UNUSED(operationOptions); auto state = MakeIntrusive<TPqState>(sessionId); - state->SupportRtmrMode = supportRtmrMode; + state->SupportRtmrMode = supportRtmrMode; state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; state->DbResolver = dbResolverWithMeta; @@ -43,10 +43,10 @@ TDataProviderInitializer GetPqDataProviderInitializer( TDataProviderInfo info; info.Names.insert({TString{PqProviderName}}); - info.Source = CreatePqDataSource(state, gateway); - info.Sink = CreatePqDataSink(state, gateway); + info.Source = CreatePqDataSource(state, gateway); + info.Sink = CreatePqDataSink(state, gateway); - info.OpenSession = [gateway](const TString& sessionId, const TString& username, + info.OpenSession = [gateway](const TString& sessionId, const TString& username, const TOperationProgressWriter& progressWriter, const TYqlOperationOptions& operationOptions, TIntrusivePtr<IRandomProvider> randomProvider, TIntrusivePtr<ITimeProvider> timeProvider) { Y_UNUSED(progressWriter); @@ -54,7 +54,7 @@ TDataProviderInitializer GetPqDataProviderInitializer( Y_UNUSED(randomProvider); Y_UNUSED(timeProvider); - return gateway->OpenSession(sessionId, username); + return gateway->OpenSession(sessionId, username); }; info.CloseSession = [gateway](const TString& sessionId) { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h index a890e32b69..e2fdd051d7 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h @@ -17,10 +17,10 @@ struct TPqState : public TThrRefBase { using TPtr = TIntrusivePtr<TPqState>; struct TTopicMeta { - TPositionHandle Pos; - bool RawFormat = true; - TExprNode::TPtr RowSpec; - TExprNode::TPtr ColumnOrder; + TPositionHandle Pos; + bool RawFormat = true; + TExprNode::TPtr RowSpec; + TExprNode::TPtr ColumnOrder; TMaybe<::NPq::NConfigurationManager::TTopicDescription> Description; }; @@ -42,11 +42,11 @@ public: return Configuration->PqReadByRtmrCluster_.Get() != "dq"; } -public: - bool SupportRtmrMode = false; - const TString SessionId; - THashMap<std::pair<TString, TString>, TTopicMeta> Topics; - +public: + bool SupportRtmrMode = false; + const TString SessionId; + THashMap<std::pair<TString, TString>, TTopicMeta> Topics; + TTypeAnnotationContext* Types = nullptr; TPqConfiguration::TPtr Configuration = MakeIntrusive<TPqConfiguration>(); const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.cpp index ed8c90156d..a335abaf45 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.cpp @@ -1,9 +1,9 @@ -#include "yql_pq_provider_impl.h" - -namespace NYql { - -TString MakeTopicDisplayName(TStringBuf cluster, TStringBuf path) { - return TString::Join(cluster, ".`", path, "`"); -} - -} // namespace NYql +#include "yql_pq_provider_impl.h" + +namespace NYql { + +TString MakeTopicDisplayName(TStringBuf cluster, TStringBuf path) { + return TString::Join(cluster, ".`", path, "`"); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h index 619eb468e2..d66857ab17 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h @@ -10,8 +10,8 @@ namespace NYql { -TIntrusivePtr<IDataProvider> CreatePqDataSource(TPqState::TPtr state, IPqGateway::TPtr gateway); -TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway::TPtr gateway); +TIntrusivePtr<IDataProvider> CreatePqDataSource(TPqState::TPtr state, IPqGateway::TPtr gateway); +TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway::TPtr gateway); THolder<IGraphTransformer> CreatePqLoadTopicMetadataTransformer(TPqState::TPtr state); @@ -25,8 +25,8 @@ THolder<IGraphTransformer> CreatePqLogicalOptProposalTransformer(TPqState::TPtr THolder<IGraphTransformer> CreatePqPhysicalOptProposalTransformer(TPqState::TPtr state); -THolder<IGraphTransformer> CreatePqIODiscoveryTransformer(TPqState::TPtr state); - -TString MakeTopicDisplayName(TStringBuf cluster, TStringBuf path); - +THolder<IGraphTransformer> CreatePqIODiscoveryTransformer(TPqState::TPtr state); + +TString MakeTopicDisplayName(TStringBuf cluster, TStringBuf path); + } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp index 278db980f4..60609bd2d4 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp @@ -7,7 +7,7 @@ using namespace NCommon; TPqConfiguration::TPqConfiguration() { REGISTER_SETTING(*this, Consumer); REGISTER_SETTING(*this, Database); - REGISTER_SETTING(*this, PqReadByRtmrCluster_); + REGISTER_SETTING(*this, PqReadByRtmrCluster_); } TPqSettings::TConstPtr TPqConfiguration::Snapshot() const { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h index 5ed40d07d9..ca1c7ffdb6 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h @@ -1,5 +1,5 @@ #pragma once - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> #include <ydb/library/yql/providers/common/config/yql_dispatch.h> @@ -17,20 +17,20 @@ struct TPqSettings { NCommon::TConfSetting<TString, false> Consumer; NCommon::TConfSetting<TString, false> Database; // It is needed in case of Cloud.LB for external users, but can be taken from config for internal LB. - NCommon::TConfSetting<TString, false> PqReadByRtmrCluster_; + NCommon::TConfSetting<TString, false> PqReadByRtmrCluster_; }; struct TPqClusterConfigurationSettings { TString ClusterName; - NYql::TPqClusterConfig::EClusterType ClusterType = NYql::TPqClusterConfig::CT_UNSPECIFIED; + NYql::TPqClusterConfig::EClusterType ClusterType = NYql::TPqClusterConfig::CT_UNSPECIFIED; TString Endpoint; TString ConfigManagerEndpoint; - bool UseSsl = false; + bool UseSsl = false; TString Database; TString DatabaseId; ui32 TvmId = 0; - TString AuthToken; - bool AddBearerToToken = false; + TString AuthToken; + bool AddBearerToToken = false; }; struct TPqConfiguration : public TPqSettings, public NCommon::TSettingDispatcher { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp index 2b99e52484..a20c9f6adf 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp @@ -4,40 +4,40 @@ namespace NYql { -namespace { -std::pair<TExprNode::TPtr, TExprNode::TPtr> GetSchema(const TExprNode& settings) { - for (auto i = 0U; i < settings.ChildrenSize(); ++i) { - if (settings.Child(i)->Head().IsAtom("userschema")) { - return { settings.Child(i)->ChildPtr(1), settings.Child(i)->ChildrenSize() > 2 ? settings.Child(i)->TailPtr() : TExprNode::TPtr() }; - } - } - - return {}; -} +namespace { +std::pair<TExprNode::TPtr, TExprNode::TPtr> GetSchema(const TExprNode& settings) { + for (auto i = 0U; i < settings.ChildrenSize(); ++i) { + if (settings.Child(i)->Head().IsAtom("userschema")) { + return { settings.Child(i)->ChildPtr(1), settings.Child(i)->ChildrenSize() > 2 ? settings.Child(i)->TailPtr() : TExprNode::TPtr() }; + } + } + + return {}; } +} -TTopicKeyParser::TTopicKeyParser(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx) { - YQL_ENSURE(Parse(expr, readSettings, ctx), "Failed to parse topic info"); -} - -bool TTopicKeyParser::Parse(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx) { - if (expr.IsCallable("MrTableConcat")) { - return TryParseKey(expr.Head(), ctx); +TTopicKeyParser::TTopicKeyParser(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx) { + YQL_ENSURE(Parse(expr, readSettings, ctx), "Failed to parse topic info"); +} + +bool TTopicKeyParser::Parse(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx) { + if (expr.IsCallable("MrTableConcat")) { + return TryParseKey(expr.Head(), ctx); } - - if (expr.IsCallable(NNodes::TCoKey::CallableName())) { - return TryParseKey(expr, ctx); - } - - if (readSettings && expr.IsCallable("MrObject")) { - return TryParseObject(expr, readSettings); - } - ctx.AddError(TIssue(ctx.GetPosition(expr.Pos()), "Expected MrTableConcat or Key or MrObject")); - return false; + + if (expr.IsCallable(NNodes::TCoKey::CallableName())) { + return TryParseKey(expr, ctx); + } + + if (readSettings && expr.IsCallable("MrObject")) { + return TryParseObject(expr, readSettings); + } + ctx.AddError(TIssue(ctx.GetPosition(expr.Pos()), "Expected MrTableConcat or Key or MrObject")); + return false; } -bool TTopicKeyParser::TryParseKey(const TExprNode& expr, TExprContext& ctx) { - const auto maybeKey = NNodes::TExprBase(&expr).Maybe<NNodes::TCoKey>(); +bool TTopicKeyParser::TryParseKey(const TExprNode& expr, TExprContext& ctx) { + const auto maybeKey = NNodes::TExprBase(&expr).Maybe<NNodes::TCoKey>(); if (!maybeKey) { ctx.AddError(TIssue(ctx.GetPosition(expr.Pos()), "Expected Key")); return false; @@ -54,11 +54,11 @@ bool TTopicKeyParser::TryParseKey(const TExprNode& expr, TExprContext& ctx) { return true; } -bool TTopicKeyParser::TryParseObject(const TExprNode& expr, TExprNode::TPtr readSettings) { - std::tie(UserSchema, ColumnOrder) = GetSchema(*readSettings); - TopicPath = TString(expr.Child(0)->Content()); - Format = TString(expr.Child(1)->Content()); - Compression = TString(expr.Child(2)->Content()); - return true; -} +bool TTopicKeyParser::TryParseObject(const TExprNode& expr, TExprNode::TPtr readSettings) { + std::tie(UserSchema, ColumnOrder) = GetSchema(*readSettings); + TopicPath = TString(expr.Child(0)->Content()); + Format = TString(expr.Child(1)->Content()); + Compression = TString(expr.Child(2)->Content()); + return true; +} } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h index b22f61125f..a22e42caf1 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h @@ -7,41 +7,41 @@ namespace NYql { class TTopicKeyParser { public: - TTopicKeyParser() {} - TTopicKeyParser(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx); + TTopicKeyParser() {} + TTopicKeyParser(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx); const TString& GetTopicPath() const { return TopicPath; } - TExprNode::TPtr GetUserSchema() const { - return UserSchema; - } - - TExprNode::TPtr GetColumnOrder() const { - return ColumnOrder; - } - - const TString& GetFormat() const { - return Format; - } - - const TString& GetCompression() const { - return Compression; - } - - bool Parse(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx); - + TExprNode::TPtr GetUserSchema() const { + return UserSchema; + } + + TExprNode::TPtr GetColumnOrder() const { + return ColumnOrder; + } + + const TString& GetFormat() const { + return Format; + } + + const TString& GetCompression() const { + return Compression; + } + + bool Parse(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx); + private: - bool TryParseKey(const TExprNode& expr, TExprContext& ctx); - bool TryParseObject(const TExprNode& expr, TExprNode::TPtr readSettings); + bool TryParseKey(const TExprNode& expr, TExprContext& ctx); + bool TryParseObject(const TExprNode& expr, TExprNode::TPtr readSettings); private: TString TopicPath; - TString Format; - TString Compression; - TExprNode::TPtr UserSchema; - TExprNode::TPtr ColumnOrder; + TString Format; + TString Compression; + TExprNode::TPtr UserSchema; + TExprNode::TPtr ColumnOrder; }; } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp index 4127452d12..4083269deb 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp @@ -1,221 +1,221 @@ -#include <library/cpp/testing/unittest/registar.h> - +#include <library/cpp/testing/unittest/registar.h> + #include <ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.h> #include <ydb/library/yql/dq/comp_nodes/yql_common_dq_transform.h> - + #include <ydb/library/yql/providers/common/comp_nodes/yql_factory.h> - + #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_provider.h> - + #include <ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h> #include <ydb/library/yql/providers/pq/provider/yql_pq_provider.h> - + #include <ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h> #include <ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h> - + #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> - + #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - + #include <ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h> #include <ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h> - + #include <ydb/library/yql/core/facade/yql_facade.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/services/mounts/yql_mounts.h> - + #include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h> #include <ydb/library/yql/core/file_storage/file_storage.h> - + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -#include <util/stream/tee.h> -#include <util/string/cast.h> - -namespace NYql { - -bool RunPqProgram( - const TString& code, - bool optimizeOnly, - bool printExpr = false, - bool printTrace = false, - TString* errorsMessage = nullptr) { - NLog::YqlLoggerScope logger("cerr", false); - NLog::YqlLogger().SetComponentLevel(NLog::EComponent::Core, NLog::ELevel::DEBUG); - NLog::YqlLogger().SetComponentLevel(NLog::EComponent::ProviderRtmr, NLog::ELevel::DEBUG); - - IOutputStream* errorsOutput = &Cerr; - TMaybe<TStringOutput> errorsMessageOutput; - TMaybe<TTeeOutput> tee; - if (errorsMessage) { - errorsMessageOutput.ConstructInPlace(*errorsMessage); - tee.ConstructInPlace(&*errorsMessageOutput, &Cerr); - errorsOutput = &*tee; - } - - // Gateways config. - TGatewaysConfig gatewaysConfig; - // pq - { - auto& pqClusterConfig = *gatewaysConfig.MutablePq()->MutableClusterMapping()->Add(); - pqClusterConfig.SetName("lb"); - pqClusterConfig.SetClusterType(NYql::TPqClusterConfig::CT_PERS_QUEUE); - pqClusterConfig.SetEndpoint("lb.ru"); - pqClusterConfig.SetConfigManagerEndpoint("cm.lb.ru"); - pqClusterConfig.SetTvmId(777); - } - - // solomon - { - auto& solomonClusterConfig = *gatewaysConfig.MutableSolomon()->MutableClusterMapping()->Add(); - solomonClusterConfig.SetName("sol"); - solomonClusterConfig.SetCluster("sol.ru"); - } - - // dq - { - auto& dqCfg = *gatewaysConfig.MutableDq(); - auto* setting = dqCfg.AddDefaultSettings(); - setting->SetName("EnableComputeActor"); - setting->SetValue("1"); - } - - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); - TVector<TDataProviderInitializer> dataProvidersInit; - - // pq - auto pqGateway = MakeIntrusive<TDummyPqGateway>(); - pqGateway->AddDummyTopic(TDummyTopic("lb", "my_in_topic")); - pqGateway->AddDummyTopic(TDummyTopic("lb", "my_out_topic")); - dataProvidersInit.push_back(GetPqDataProviderInitializer(std::move(pqGateway))); - - // solomon - auto solomonGateway = CreateSolomonGateway(gatewaysConfig.GetSolomon()); - dataProvidersInit.push_back(GetSolomonDataProviderInitializer(std::move(solomonGateway))); - - // dq - auto dqCompFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory({ - NYql::GetCommonDqFactory(), - NKikimr::NMiniKQL::GetYqlFactory() - }); - - auto dqTaskTransformFactory = NYql::CreateCompositeTaskTransformFactory({ - NYql::CreateCommonDqTaskTransformFactory() - }); - + +#include <util/stream/tee.h> +#include <util/string/cast.h> + +namespace NYql { + +bool RunPqProgram( + const TString& code, + bool optimizeOnly, + bool printExpr = false, + bool printTrace = false, + TString* errorsMessage = nullptr) { + NLog::YqlLoggerScope logger("cerr", false); + NLog::YqlLogger().SetComponentLevel(NLog::EComponent::Core, NLog::ELevel::DEBUG); + NLog::YqlLogger().SetComponentLevel(NLog::EComponent::ProviderRtmr, NLog::ELevel::DEBUG); + + IOutputStream* errorsOutput = &Cerr; + TMaybe<TStringOutput> errorsMessageOutput; + TMaybe<TTeeOutput> tee; + if (errorsMessage) { + errorsMessageOutput.ConstructInPlace(*errorsMessage); + tee.ConstructInPlace(&*errorsMessageOutput, &Cerr); + errorsOutput = &*tee; + } + + // Gateways config. + TGatewaysConfig gatewaysConfig; + // pq + { + auto& pqClusterConfig = *gatewaysConfig.MutablePq()->MutableClusterMapping()->Add(); + pqClusterConfig.SetName("lb"); + pqClusterConfig.SetClusterType(NYql::TPqClusterConfig::CT_PERS_QUEUE); + pqClusterConfig.SetEndpoint("lb.ru"); + pqClusterConfig.SetConfigManagerEndpoint("cm.lb.ru"); + pqClusterConfig.SetTvmId(777); + } + + // solomon + { + auto& solomonClusterConfig = *gatewaysConfig.MutableSolomon()->MutableClusterMapping()->Add(); + solomonClusterConfig.SetName("sol"); + solomonClusterConfig.SetCluster("sol.ru"); + } + + // dq + { + auto& dqCfg = *gatewaysConfig.MutableDq(); + auto* setting = dqCfg.AddDefaultSettings(); + setting->SetName("EnableComputeActor"); + setting->SetValue("1"); + } + + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); + TVector<TDataProviderInitializer> dataProvidersInit; + + // pq + auto pqGateway = MakeIntrusive<TDummyPqGateway>(); + pqGateway->AddDummyTopic(TDummyTopic("lb", "my_in_topic")); + pqGateway->AddDummyTopic(TDummyTopic("lb", "my_out_topic")); + dataProvidersInit.push_back(GetPqDataProviderInitializer(std::move(pqGateway))); + + // solomon + auto solomonGateway = CreateSolomonGateway(gatewaysConfig.GetSolomon()); + dataProvidersInit.push_back(GetSolomonDataProviderInitializer(std::move(solomonGateway))); + + // dq + auto dqCompFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory({ + NYql::GetCommonDqFactory(), + NKikimr::NMiniKQL::GetYqlFactory() + }); + + auto dqTaskTransformFactory = NYql::CreateCompositeTaskTransformFactory({ + NYql::CreateCommonDqTaskTransformFactory() + }); + const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")); NYdb::TDriver driver(driverConfig); auto dqGateway = CreateLocalDqGateway(driver, functionRegistry.Get(), dqCompFactory, dqTaskTransformFactory, {}); - - auto storage = NYql::CreateFileStorage({}); + + auto storage = NYql::CreateFileStorage({}); dataProvidersInit.push_back(NYql::GetDqDataProviderInitializer(&CreateInMemoryExecTransformer, dqGateway, dqCompFactory, {}, storage)); - - TExprContext moduleCtx; - IModuleResolver::TPtr moduleResolver; - YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); - - TProgramFactory factory(true, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut"); - - factory.SetGatewaysConfig(&gatewaysConfig); - factory.SetModules(moduleResolver); - - TProgramPtr program = factory.Create("program", code); + + TExprContext moduleCtx; + IModuleResolver::TPtr moduleResolver; + YQL_ENSURE(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); + + TProgramFactory factory(true, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut"); + + factory.SetGatewaysConfig(&gatewaysConfig); + factory.SetModules(moduleResolver); + + TProgramPtr program = factory.Create("program", code); program->ConfigureYsonResultFormat(NYson::EYsonFormat::Text); - - Cerr << "Parse SQL..." << Endl; - NSQLTranslation::TTranslationSettings sqlSettings; - sqlSettings.SyntaxVersion = 1; - sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; - sqlSettings.Flags.insert("DqEngineEnable"); - sqlSettings.Flags.insert("DqEngineForce"); - - sqlSettings.ClusterMapping["lb"] = PqProviderName; - sqlSettings.ClusterMapping["sol"] = SolomonProviderName; - if (!program->ParseSql(sqlSettings)) { - program->PrintErrorsTo(*errorsOutput); - return false; - } - program->AstRoot()->PrettyPrintTo(Cerr, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); - - - Cerr << "Compile..." << Endl; - if (!program->Compile("user")) { - program->PrintErrorsTo(*errorsOutput); - return false; - } - - auto exprOut = printExpr ? &Cout : nullptr; - auto traceOpt = printTrace ? &Cerr : nullptr; - - TProgram::TStatus status = TProgram::TStatus::Error; - if (optimizeOnly) { - Cerr << "Optimize..." << Endl; - status = program->Optimize("user", traceOpt, nullptr, exprOut); - } else { - Cerr << "Run..." << Endl; - status = program->Run("user", traceOpt, nullptr, exprOut); - } - - if (status == TProgram::TStatus::Error) { - if (printTrace) { - program->Print(traceOpt, nullptr); - } - program->PrintErrorsTo(*errorsOutput); - return false; - } - + + Cerr << "Parse SQL..." << Endl; + NSQLTranslation::TTranslationSettings sqlSettings; + sqlSettings.SyntaxVersion = 1; + sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; + sqlSettings.Flags.insert("DqEngineEnable"); + sqlSettings.Flags.insert("DqEngineForce"); + + sqlSettings.ClusterMapping["lb"] = PqProviderName; + sqlSettings.ClusterMapping["sol"] = SolomonProviderName; + if (!program->ParseSql(sqlSettings)) { + program->PrintErrorsTo(*errorsOutput); + return false; + } + program->AstRoot()->PrettyPrintTo(Cerr, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); + + + Cerr << "Compile..." << Endl; + if (!program->Compile("user")) { + program->PrintErrorsTo(*errorsOutput); + return false; + } + + auto exprOut = printExpr ? &Cout : nullptr; + auto traceOpt = printTrace ? &Cerr : nullptr; + + TProgram::TStatus status = TProgram::TStatus::Error; + if (optimizeOnly) { + Cerr << "Optimize..." << Endl; + status = program->Optimize("user", traceOpt, nullptr, exprOut); + } else { + Cerr << "Run..." << Endl; + status = program->Run("user", traceOpt, nullptr, exprOut); + } + + if (status == TProgram::TStatus::Error) { + if (printTrace) { + program->Print(traceOpt, nullptr); + } + program->PrintErrorsTo(*errorsOutput); + return false; + } + driver.Stop(true); - Cerr << "Done." << Endl; - return true; -} - -Y_UNIT_TEST_SUITE(YqlPqSimpleTests) { - - Y_UNIT_TEST(SelectWithNoSchema) { - auto code = R"( -USE lb; -PRAGMA pq.Consumer="my_test_consumer"; -INSERT INTO my_out_topic -SELECT Data FROM my_in_topic WHERE Data < "100"; - )"; - TString errorMessage; - auto res = RunPqProgram(code, true, true, true, &errorMessage); - UNIT_ASSERT_C(res, errorMessage); - } - - Y_UNIT_TEST(SelectWithSchema) { - auto code = R"( -USE lb; -PRAGMA pq.Consumer="my_test_consumer"; - -INSERT INTO my_out_topic -SELECT CAST(y as string) || x FROM lb.object(my_in_topic, "json") WITH SCHEMA (Int32 as y, String as x) - )"; - TString errorMessage; - auto res = RunPqProgram(code, true, true, true, &errorMessage); - UNIT_ASSERT_C(res, errorMessage); - } - - Y_UNIT_TEST(SelectStarWithSchema) { - auto code = R"( -USE lb; -PRAGMA pq.Consumer="my_test_consumer"; - -$q = SELECT * FROM lb.object(my_in_topic, "json") WITH SCHEMA (Int32 as y, String as x); -INSERT INTO my_out_topic -SELECT x FROM $q - )"; - TString errorMessage; - auto res = RunPqProgram(code, true, true, true, &errorMessage); - UNIT_ASSERT_C(res, errorMessage); - } - -} - -} // NYql + Cerr << "Done." << Endl; + return true; +} + +Y_UNIT_TEST_SUITE(YqlPqSimpleTests) { + + Y_UNIT_TEST(SelectWithNoSchema) { + auto code = R"( +USE lb; +PRAGMA pq.Consumer="my_test_consumer"; +INSERT INTO my_out_topic +SELECT Data FROM my_in_topic WHERE Data < "100"; + )"; + TString errorMessage; + auto res = RunPqProgram(code, true, true, true, &errorMessage); + UNIT_ASSERT_C(res, errorMessage); + } + + Y_UNIT_TEST(SelectWithSchema) { + auto code = R"( +USE lb; +PRAGMA pq.Consumer="my_test_consumer"; + +INSERT INTO my_out_topic +SELECT CAST(y as string) || x FROM lb.object(my_in_topic, "json") WITH SCHEMA (Int32 as y, String as x) + )"; + TString errorMessage; + auto res = RunPqProgram(code, true, true, true, &errorMessage); + UNIT_ASSERT_C(res, errorMessage); + } + + Y_UNIT_TEST(SelectStarWithSchema) { + auto code = R"( +USE lb; +PRAGMA pq.Consumer="my_test_consumer"; + +$q = SELECT * FROM lb.object(my_in_topic, "json") WITH SCHEMA (Int32 as y, String as x); +INSERT INTO my_out_topic +SELECT x FROM $q + )"; + TString errorMessage; + auto res = RunPqProgram(code, true, true, true, &errorMessage); + UNIT_ASSERT_C(res, errorMessage); + } + +} + +} // NYql diff --git a/ydb/library/yql/providers/s3/actors/ya.make b/ydb/library/yql/providers/s3/actors/ya.make index e23702313d..051ef515ed 100644 --- a/ydb/library/yql/providers/s3/actors/ya.make +++ b/ydb/library/yql/providers/s3/actors/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER( - g:yq + g:yq g:yql ) diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index eb5410a24e..77e944c42b 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -229,7 +229,7 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor( ui64 inputIndex, const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, - NYql::NDq::IDqSourceActor::ICallbacks* callback, + NYql::NDq::IDqSourceActor::ICallbacks* callback, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, const std::shared_ptr<NS3::TRetryConfig>& retryConfig) { diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h index 9280e9ba2e..5b0723ffb9 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h @@ -15,7 +15,7 @@ std::pair<NYql::NDq::IDqSourceActor*, NActors::IActor*> CreateS3ReadActor( ui64 inputIndex, const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, - IDqSourceActor::ICallbacks* callback, + IDqSourceActor::ICallbacks* callback, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr); diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h index 2df0a312b1..2d808507ed 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h +++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h @@ -7,7 +7,7 @@ #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> #include <ydb/library/yql/providers/s3/proto/retry_config.pb.h> - + namespace NYql::NDq { void RegisterS3ReadActorFactory( diff --git a/ydb/library/yql/providers/s3/expr_nodes/ya.make b/ydb/library/yql/providers/s3/expr_nodes/ya.make index 6d3003f148..35e2822a8f 100644 --- a/ydb/library/yql/providers/s3/expr_nodes/ya.make +++ b/ydb/library/yql/providers/s3/expr_nodes/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER( - g:yq + g:yq g:yql ) diff --git a/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json b/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json index ffb5fb8035..33de86a757 100644 --- a/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json +++ b/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json @@ -44,7 +44,7 @@ "Match": {"Type": "Callable", "Name": "S3SourceSettings"}, "Children": [ {"Index": 0, "Name": "Paths", "Type": "TS3Paths"}, - {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"} + {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"} ] }, { diff --git a/ydb/library/yql/providers/s3/proto/ya.make b/ydb/library/yql/providers/s3/proto/ya.make index 48215f2ab5..783f2e0447 100644 --- a/ydb/library/yql/providers/s3/proto/ya.make +++ b/ydb/library/yql/providers/s3/proto/ya.make @@ -1,7 +1,7 @@ -OWNER( - g:yq - g:yql -) +OWNER( + g:yq + g:yql +) PROTO_LIBRARY() diff --git a/ydb/library/yql/providers/s3/provider/ya.make b/ydb/library/yql/providers/s3/provider/ya.make index 075bad7b0b..4381d08a79 100644 --- a/ydb/library/yql/providers/s3/provider/ya.make +++ b/ydb/library/yql/providers/s3/provider/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER( - g:yq + g:yq g:yql ) diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index 539272add6..96e03972c3 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -101,14 +101,14 @@ public: ); } - const auto token = "cluster:default_" + clusterName; - YQL_CLOG(INFO, ProviderS3) << "Wrap " << read->Content() << " with token: " << token; + const auto token = "cluster:default_" + clusterName; + YQL_CLOG(INFO, ProviderS3) << "Wrap " << read->Content() << " with token: " << token; return Build<TDqSourceWrap>(ctx, read->Pos()) .Input<TS3SourceSettings>() .Paths(s3ReadObject.Object().Paths()) .Token<TCoSecureParam>() - .Name().Build(token) + .Name().Build(token) .Build() .Build() .RowType(ExpandType(s3ReadObject.Pos(), *rowType, ctx)) @@ -124,7 +124,7 @@ public: if (const auto maySettings = source.Settings().Maybe<TS3SourceSettings>()) { const auto settings = maySettings.Cast(); const auto& cluster = source.DataSource().Cast<TS3DataSource>().Cluster().StringValue(); - const auto& connect = State_->Configuration->Clusters.at(cluster); + const auto& connect = State_->Configuration->Clusters.at(cluster); NS3::TSource srcDesc; srcDesc.SetUrl(connect.Url); diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp index 3e0384c9d7..7d171d5cd3 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp @@ -63,13 +63,13 @@ void OnDiscovery( auto& items = std::get<TItemsMap>(output); if (items.size() + contents.size() > 9000ULL) { std::get<TIssues>(output) = { TIssue(pos, TStringBuilder() << "It's over nine thousand items under '" << std::get<0U>(keys) << std::get<1U>(keys) << "'!")}; - break; + break; } for (const auto& content : contents) { items.emplace(content.Node("s3:Key", false, nss).Value<TString>(), content.Node("s3:Size", false, nss).Value<unsigned>()); } - + if (root.Node("s3:IsTruncated", false, nss).Value<bool>()) { if (const auto g = gateway.lock()) { const auto& next = root.Node("s3:NextContinuationToken", false, nss).Value<TString>(); @@ -81,11 +81,11 @@ void OnDiscovery( headers.emplace_back(token); return g->Download(std::get<0U>(keys) + "?list-type=2&prefix=" + prefix + "&continuation-token=" + next + "&max-keys=" + maxKeys, std::move(headers), 0U, std::bind(&OnDiscovery, gateway, pos, std::placeholders::_1, std::cref(keys), std::ref(output), std::move(promise), pendingBucketsWPtr, promiseInd)); - } + } YQL_CLOG(INFO, ProviderS3) << "Gateway disappeared."; - } + } } - + break; } catch (const std::exception& ex) { logMsg += TStringBuilder() << "Exception occurred: " << ex.what(); @@ -182,11 +182,11 @@ public: const auto& object = read.Arg(2).Ref(); const std::string_view& path = object.Head().Content(); const auto& prefix = path.substr(0U, path.find_first_of("*?{")); - const auto& connect = State_->Configuration->Clusters.at(read.DataSource().Cluster().StringValue()); - const auto& token = State_->Configuration->Tokens.at(read.DataSource().Cluster().StringValue()); - const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token); - const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); - + const auto& connect = State_->Configuration->Clusters.at(read.DataSource().Cluster().StringValue()); + const auto& token = State_->Configuration->Tokens.at(read.DataSource().Cluster().StringValue()); + const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token); + const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); + std::get<TNodeSet>((*PendingBuckets_)[std::make_tuple(connect.Url, TString(prefix), authToken.empty() ? TString() : TString("X-YaCloud-SubjectToken:") += authToken)]).emplace(read.Raw()); } } @@ -223,9 +223,9 @@ public: } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - // Raise errors if any - AllFuture_.GetValue(); - + // Raise errors if any + AllFuture_.GetValue(); + TNodeOnNodeOwnedMap replaces(PendingBuckets_->size()); auto buckets = std::move(*PendingBuckets_); auto count = 0ULL; @@ -234,11 +234,11 @@ public: if (const auto issues = std::move(std::get<TIssues>(bucket.second))) { YQL_CLOG(INFO, ProviderS3) << "Discovery " << std::get<0U>(bucket.first) << std::get<1U>(bucket.first) << " error " << issues.ToString(); std::for_each(issues.begin(), issues.end(), std::bind(&TExprContext::AddError, std::ref(ctx), std::placeholders::_1)); - return TStatus::Error; - } - - const auto nodes = std::move(std::get<TNodeSet>(bucket.second)); - for (const auto r : nodes) { + return TStatus::Error; + } + + const auto nodes = std::move(std::get<TNodeSet>(bucket.second)); + for (const auto r : nodes) { const TS3Read read(r); const auto& object = read.Arg(2).Ref(); const std::string_view& path = object.Head().Content(); @@ -278,7 +278,7 @@ public: } YQL_CLOG(INFO, ProviderS3) << "Object " << path << " has " << paths.size() << " items with total size " << total; } else if (const auto f = items.find(TString(path)); items.cend() == f) { - ctx.AddError(TIssue(ctx.GetPosition(object.Pos()), TStringBuilder() << "Object " << path << " doesn't exist.")); + ctx.AddError(TIssue(ctx.GetPosition(object.Pos()), TStringBuilder() << "Object " << path << " doesn't exist.")); return TStatus::Error; } else if (const auto size = f->second; size > State_->Configuration->FileSizeLimit) { ctx.AddError(TIssue(ctx.GetPosition(object.Pos()), TStringBuilder() << "Object " << path << " size " << size << " is too large.")); diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_provider.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_provider.cpp index 4d70add3c0..aa8d0e7d99 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_provider.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_provider.cpp @@ -26,9 +26,9 @@ TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; - state->CredentialsFactory = credentialsFactory; + state->CredentialsFactory = credentialsFactory; if (gatewaysConfig) { - state->Configuration->Init(gatewaysConfig->GetS3(), typeCtx); + state->Configuration->Init(gatewaysConfig->GetS3(), typeCtx); } TDataProviderInfo info; diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_provider.h b/ydb/library/yql/providers/s3/provider/yql_s3_provider.h index 12cd74f805..35ef6ba15e 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_provider.h +++ b/ydb/library/yql/providers/s3/provider/yql_s3_provider.h @@ -26,7 +26,7 @@ struct TS3State : public TThrRefBase TTypeAnnotationContext* Types = nullptr; TS3Configuration::TPtr Configuration = MakeIntrusive<TS3Configuration>(); const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; - ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; }; TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr); diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp index 99c308eaf3..504800e677 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp @@ -19,7 +19,7 @@ bool TS3Configuration::HasCluster(TStringBuf cluster) const { return ValidClusters.contains(cluster); } -void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx) +void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx) { FileSizeLimit = config.HasFileSizeLimit() ? config.GetFileSizeLimit() : 2_GB; MaxFilesPerQuery = config.HasMaxFilesPerQuery() ? config.GetMaxFilesPerQuery() : 7000; @@ -37,11 +37,11 @@ void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeA this->Dispatch(cluster.GetName(), cluster.GetSettings()); auto& settings = Clusters[cluster.GetName()]; settings.Url = cluster.GetUrl(); - TString authToken; + TString authToken; if (const auto& token = cluster.GetToken()) { authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "", token); - } - Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); + } + Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); } this->FreezeDefaults(); } diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h index 9bbcb72c87..268c1648a9 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h @@ -23,7 +23,7 @@ struct TS3Configuration : public TS3Settings, public NCommon::TSettingDispatcher TS3Configuration(); TS3Configuration(const TS3Configuration&) = delete; - void Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx); + void Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx); bool HasCluster(TStringBuf cluster) const; diff --git a/ydb/library/yql/providers/s3/ya.make b/ydb/library/yql/providers/s3/ya.make index f97084d1aa..7323d161bc 100644 --- a/ydb/library/yql/providers/s3/ya.make +++ b/ydb/library/yql/providers/s3/ya.make @@ -1,11 +1,11 @@ -OWNER( - g:yq - g:yql -) - -RECURSE( - actors - expr_nodes - proto - provider -) +OWNER( + g:yq + g:yql +) + +RECURSE( + actors + expr_nodes + proto + provider +) diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp index fc07635176..03f3e76350 100644 --- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp +++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp @@ -82,8 +82,8 @@ public: ui64 outputIndex, TDqSolomonWriteParams&& writeParams, NYql::NDq::IDqSinkActor::ICallbacks* callbacks, - const NMonitoring::TDynamicCounterPtr& counters, - std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider, + const NMonitoring::TDynamicCounterPtr& counters, + std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider, i64 freeSpace) : TActor<TDqSolomonWriteActor>(&TDqSolomonWriteActor::StateFunc) , OutputIndex(outputIndex) @@ -95,7 +95,7 @@ public: , UserMetricsEncoder( WriteParams.Shard.GetScheme(), WriteParams.Shard.GetClusterType() == NSo::NProto::ESolomonClusterType::CT_MONITORING) - , CredentialsProvider(credentialsProvider) + , CredentialsProvider(credentialsProvider) { SINK_LOG_D("Init"); } @@ -246,7 +246,7 @@ private: TString GetUrl() const { TStringBuilder builder; - builder << (WriteParams.Shard.GetUseSsl() ? "https://" : "http://"); + builder << (WriteParams.Shard.GetUseSsl() ? "https://" : "http://"); builder << WriteParams.Shard.GetEndpoint(); switch (WriteParams.Shard.GetClusterType()) { @@ -295,14 +295,14 @@ private: void FillAuth(NHttp::THttpOutgoingRequestPtr& httpRequest) { const TString authorizationHeader = "Authorization"; - const TString authToken = CredentialsProvider->GetAuthInfo(); + const TString authToken = CredentialsProvider->GetAuthInfo(); switch (WriteParams.Shard.GetClusterType()) { case NSo::NProto::ESolomonClusterType::CT_SOLOMON: - httpRequest->Set(authorizationHeader, "OAuth " + authToken); + httpRequest->Set(authorizationHeader, "OAuth " + authToken); break; case NSo::NProto::ESolomonClusterType::CT_MONITORING: - httpRequest->Set(authorizationHeader, "Bearer " + authToken); + httpRequest->Set(authorizationHeader, "Bearer " + authToken); break; default: Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); @@ -430,7 +430,7 @@ private: THashMap<NHttp::THttpOutgoingRequestPtr, TMetricsInflight> InflightBuffer; TMetricsEncoder UserMetricsEncoder; - std::shared_ptr<NYdb::ICredentialsProvider> CredentialsProvider; + std::shared_ptr<NYdb::ICredentialsProvider> CredentialsProvider; }; std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( @@ -438,33 +438,33 @@ std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( ui64 outputIndex, const THashMap<TString, TString>& secureParams, NYql::NDq::IDqSinkActor::ICallbacks* callbacks, - const NMonitoring::TDynamicCounterPtr& counters, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + const NMonitoring::TDynamicCounterPtr& counters, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, i64 freeSpace) { - const TString& tokenName = settings.GetToken().GetName(); - const TString token = secureParams.Value(tokenName, TString()); + const TString& tokenName = settings.GetToken().GetName(); + const TString token = secureParams.Value(tokenName, TString()); TDqSolomonWriteParams params { .Shard = std::move(settings), }; - auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); - auto credentialsProvider = credentialsProviderFactory->CreateProvider(); - + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); + auto credentialsProvider = credentialsProviderFactory->CreateProvider(); + TDqSolomonWriteActor* actor = new TDqSolomonWriteActor( outputIndex, std::move(params), callbacks, counters, - credentialsProvider, + credentialsProvider, freeSpace); return {actor, actor}; } -void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { +void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NSo::NProto::TDqSolomonShard>("SolomonSink", - [credentialsFactory]( + [credentialsFactory]( NYql::NSo::NProto::TDqSolomonShard&& settings, IDqSinkActorFactory::TArguments&& args) { @@ -475,8 +475,8 @@ void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredService args.OutputIndex, args.SecureParams, args.Callback, - counters, - credentialsFactory); + counters, + credentialsFactory); }); } diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h index 56c34cbb25..b7209ee24f 100644 --- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h +++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h @@ -23,10 +23,10 @@ std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( ui64 outputIndex, const THashMap<TString, TString>& secureParams, NYql::NDq::IDqSinkActor::ICallbacks* callbacks, - const NMonitoring::TDynamicCounterPtr& counters, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + const NMonitoring::TDynamicCounterPtr& counters, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, i64 freeSpace = DqSolomonDefaultFreeSpace); -void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); +void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp index a97b2af780..d1cfad8700 100644 --- a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp +++ b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp @@ -51,7 +51,7 @@ void InitSink( secureParams, &actor.GetSinkCallbacks(), counters, - nullptr, + nullptr, freeSpace); actor.InitSink(dqSink, dqSinkAsActor); @@ -98,7 +98,7 @@ NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud) { } settings.SetClusterType(isCloud ? NSo::NProto::ESolomonClusterType::CT_MONITORING : NSo::NProto::ESolomonClusterType::CT_SOLOMON); - settings.SetUseSsl(false); + settings.SetUseSsl(false); FillDqSolomonScheme(*settings.MutableScheme()); diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json index d064daa1b4..2413c39dba 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json @@ -73,7 +73,7 @@ {"Index": 1, "Name": "Project", "Type": "TCoAtom"}, {"Index": 2, "Name": "Cluster", "Type": "TCoAtom"}, {"Index": 3, "Name": "Service", "Type": "TCoAtom"}, - {"Index": 4, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} + {"Index": 4, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} ] }, { diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto index 3bb9d19d51..71e0c5596b 100644 --- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto +++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto @@ -9,7 +9,7 @@ enum ESolomonClusterType { CT_MONITORING = 2; } -message TToken { +message TToken { string Name = 1; } @@ -32,10 +32,10 @@ message TDqSolomonShard { string Service = 4; ESolomonClusterType ClusterType = 20; - bool UseSsl = 21; + bool UseSsl = 21; TDqSolomonShardScheme Scheme = 30; string ServiceAccount = 40; - TToken Token = 41; + TToken Token = 41; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index 941b2adc30..5badaab7b6 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -28,8 +28,8 @@ struct TSolomonConfiguration clusters.push_back(cluster.GetName()); ClusterConfigs[cluster.GetName()] = cluster; - const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken()); - Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); + const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken()); + Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); } this->SetValidClusters(clusters); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 773c960991..3a8a2112c7 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -32,17 +32,17 @@ NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonC } } -const TTypeAnnotationNode* GetItemType(const TExprNode& node) { +const TTypeAnnotationNode* GetItemType(const TExprNode& node) { const TTypeAnnotationNode* typeAnn = node.GetTypeAnn(); switch (typeAnn->GetKind()) { case ETypeAnnotationKind::Flow: - return typeAnn->Cast<TFlowExprType>()->GetItemType(); + return typeAnn->Cast<TFlowExprType>()->GetItemType(); case ETypeAnnotationKind::Stream: - return typeAnn->Cast<TStreamExprType>()->GetItemType(); + return typeAnn->Cast<TStreamExprType>()->GetItemType(); default: break; } YQL_ENSURE(false, "Invalid solomon sink type " << typeAnn->GetKind()); - return nullptr; + return nullptr; } void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) { @@ -132,13 +132,13 @@ public: shardDesc.SetService(shard.Service().StringValue()); shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); - shardDesc.SetUseSsl(clusterDesc->GetUseSsl()); + shardDesc.SetUseSsl(clusterDesc->GetUseSsl()); - const TTypeAnnotationNode* itemType = GetItemType(node); + const TTypeAnnotationNode* itemType = GetItemType(node); FillScheme(*itemType, *shardDesc.MutableScheme()); - if (auto maybeToken = shard.Token()) { - shardDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value())); + if (auto maybeToken = shard.Token()) { + shardDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value())); } protoSettings.PackFrom(shardDesc); diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp index 7ab55cfcfc..b9542c1679 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp @@ -74,7 +74,7 @@ public: ui64 inputIndex, const TString& database, const TString& endpoint, - std::shared_ptr<::NYdb::ICredentialsProviderFactory> credentialsProviderFactory, + std::shared_ptr<::NYdb::ICredentialsProviderFactory> credentialsProviderFactory, bool secure, const TString& path, ::NYdb::TDriver driver, @@ -88,7 +88,7 @@ public: , Path(path), Columns(columns), KeyColumnTypes(keyColumnTypes) , MaxRows(maxRowsInRequest), MaxBytes(maxBytesInRequest) , EndKey(keyTo) - , Connection(driver, ::NYdb::TCommonClientSettings().Database(database).DiscoveryEndpoint(endpoint).CredentialsProviderFactory(credentialsProviderFactory).DiscoveryMode(::NYdb::EDiscoveryMode::Async).EnableSsl(secure)) + , Connection(driver, ::NYdb::TCommonClientSettings().Database(database).DiscoveryEndpoint(endpoint).CredentialsProviderFactory(credentialsProviderFactory).DiscoveryMode(::NYdb::EDiscoveryMode::Async).EnableSsl(secure)) , LastReadKey(keyFrom.empty() ? NKikimr::TSerializedCellVec::Serialize(TVector<NKikimr::TCell>(KeyColumnTypes.size())) : keyFrom) , LastReadKeyInclusive(false) , Retried(0U) @@ -235,8 +235,8 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateYdbReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, NYql::NDq::IDqSourceActor::ICallbacks* callback, - ::NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) + ::NYdb::TDriver driver, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { TString keyFrom, keyTo; if (const auto taskParamsIt = taskParams.find("ydb"); taskParamsIt != taskParams.cend()) { @@ -247,8 +247,8 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateYdbReadActor( keyTo = range.Getto_key(); } - auto token = secureParams.Value(params.GetToken(), TString{}); - auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, params.GetAddBearerToToken()); + auto token = secureParams.Value(params.GetToken(), TString{}); + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, params.GetAddBearerToToken()); TVector<TString> columns; columns.reserve(params.GetColumns().size()); for (auto i = 0; i < params.GetColumns().size(); ++i) @@ -261,7 +261,7 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateYdbReadActor( ui64 maxRowsInRequest = 0ULL; ui64 maxBytesInRequest = 0ULL; - const auto actor = new TYdbReadActor(inputIndex, params.GetDatabase(), params.GetEndpoint(), credentialsProviderFactory, params.GetSecure(), params.GetTable(), std::move(driver), callback, columns, keyColumnTypes, maxRowsInRequest, maxBytesInRequest, keyFrom, keyTo); + const auto actor = new TYdbReadActor(inputIndex, params.GetDatabase(), params.GetEndpoint(), credentialsProviderFactory, params.GetSecure(), params.GetTable(), std::move(driver), callback, columns, keyColumnTypes, maxRowsInRequest, maxBytesInRequest, keyFrom, keyTo); return {actor, actor}; } diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h index 67ff7ed699..7a0ff94573 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.h @@ -3,7 +3,7 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> #include <ydb/library/yql/providers/ydb/proto/source.pb.h> - + #include <library/cpp/actors/core/actor.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> @@ -15,7 +15,7 @@ std::pair<NYql::NDq::IDqSourceActor*, NActors::IActor*> CreateYdbReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, NYql::NDq::IDqSourceActor::ICallbacks* callback, - ::NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); + ::NYdb::TDriver driver, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp index 022df27783..903c0788c5 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp @@ -5,7 +5,7 @@ namespace NYql::NDq { -void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, ::NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { +void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, ::NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NYql::NYdb::TSource>("YdbSource", [driver, credentialsFactory](NYql::NYdb::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { return CreateYdbReadActor(std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, driver, credentialsFactory); diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h index 6bf59880dd..0a56daad5c 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h @@ -1,14 +1,14 @@ #pragma once #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> - + namespace NYql::NDq { -void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); +void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); } diff --git a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp index 43343ab228..0515a9323e 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp +++ b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp @@ -258,13 +258,13 @@ public: UdfTerminate((TString("Unable to get token ") += Token).c_str()); } - auto parser = CreateStructuredTokenParser(token); - if (!parser.HasIAMToken()) { - UdfTerminate("Structured token does not contan YDB token"); - } - - token = parser.GetIAMToken(); - + auto parser = CreateStructuredTokenParser(token); + if (!parser.HasIAMToken()) { + UdfTerminate("Structured token does not contan YDB token"); + } + + token = parser.GetIAMToken(); + const auto settings = NYdb::NClickhouseInternal::TScanSettings().SnapshotId(Snapshot); const auto rows = Rows->GetValue(ctx).GetOrDefault<ui64>(0ULL); diff --git a/ydb/library/yql/providers/ydb/proto/source.proto b/ydb/library/yql/providers/ydb/proto/source.proto index 7e6c3e68b7..603243a4eb 100644 --- a/ydb/library/yql/providers/ydb/proto/source.proto +++ b/ydb/library/yql/providers/ydb/proto/source.proto @@ -11,5 +11,5 @@ message TSource { bool Secure = 6; repeated string Columns = 7; repeated uint32 KeyColumnTypes = 8; - bool AddBearerToToken = 9; + bool AddBearerToToken = 9; } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp index ecb85e824b..d2012ad398 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp @@ -81,8 +81,8 @@ public: if (const auto& maybeYdbReadTable = TMaybeNode<TYdbReadTable>(read)) { const auto& ydbReadTable = maybeYdbReadTable.Cast(); const auto& clusterName = ydbReadTable.DataSource().Cluster().Value(); - const auto token = "cluster:default_" + TString(clusterName); - YQL_CLOG(INFO, ProviderYdb) << "Wrap " << read->Content() << " with token: " << token; + const auto token = "cluster:default_" + TString(clusterName); + YQL_CLOG(INFO, ProviderYdb) << "Wrap " << read->Content() << " with token: " << token; const auto rowType = ydbReadTable.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems().back()->Cast<TListExprType>()->GetItemType(); auto columns = ydbReadTable.Columns().Ptr(); @@ -99,7 +99,7 @@ public: .Input<TYdbSourceSettings>() .Table(ydbReadTable.Table()) .Token<TCoSecureParam>() - .Name().Build(token) + .Name().Build(token) .Build() .Columns(std::move(columns)) .Build() @@ -126,7 +126,7 @@ public: srcDesc.SetDatabase(connect.Database); srcDesc.SetEndpoint(connect.Endpoint); srcDesc.SetSecure(connect.Secure); - srcDesc.SetAddBearerToToken(connect.AddBearerToToken); + srcDesc.SetAddBearerToToken(connect.AddBearerToToken); srcDesc.SetToken(token); const auto& columns = settings.Columns(); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp index 607b3ba7a8..682d4a7953 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp @@ -146,16 +146,16 @@ public: for (const auto& item : tablesFromCluster) { const auto& cluster = item.first; - TString token = State_->Configuration->Tokens.at(cluster); + TString token = State_->Configuration->Tokens.at(cluster); const auto& config = State_->Configuration->Clusters[cluster]; - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token, config.AddBearerToToken); + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token, config.AddBearerToToken); const auto ins = Clients_->emplace(cluster, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>{{Driver_, NYdb::TCommonClientSettings() .Database(config.Database) .DiscoveryEndpoint(config.Endpoint) .EnableSsl(config.Secure) - .CredentialsProviderFactory(credentialsProviderFactory) + .CredentialsProviderFactory(credentialsProviderFactory) .DiscoveryMode(NYdb::EDiscoveryMode::Async)}, std::nullopt}); YQL_CLOG(INFO, ProviderYdb) << "Take snapshot for: `" << cluster << "` from " << config.Endpoint; diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp index d17e353d3e..5916c637eb 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp @@ -29,7 +29,7 @@ TDataProviderInitializer GetYdbDataProviderInitializer( state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; - state->CredentialsFactory = credentialsFactory; + state->CredentialsFactory = credentialsFactory; state->DbResolver = dbResolverWithMeta; if (gatewaysConfig) { state->Configuration->Init(gatewaysConfig->GetYdb(), typeCtx, state->DbResolver, state->DatabaseIds); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h index ac9d00d145..890fad7034 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h @@ -32,7 +32,7 @@ struct TYdbState : public TThrRefBase TTypeAnnotationContext* Types = nullptr; TYdbConfiguration::TPtr Configuration = MakeIntrusive<TYdbConfiguration>(); const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; - ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; }; diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp index 06d1919027..ab8b642a45 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp @@ -34,7 +34,7 @@ void TYdbConfiguration::Init( const auto& endpoint = config.GetDefaultEndpoint(); - for (const auto& cluster : config.GetClusterMapping()) { + for (const auto& cluster : config.GetClusterMapping()) { this->Dispatch(cluster.GetName(), cluster.GetSettings()); if (dbResolver) { @@ -51,11 +51,11 @@ void TYdbConfiguration::Init( settings.DatabaseId = cluster.GetId(); if (cluster.HasSecure()) settings.Secure = cluster.GetSecure(); - if (cluster.HasAddBearerToToken()) - settings.AddBearerToToken = cluster.GetAddBearerToToken(); - - const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_ydb", cluster.GetToken()); - Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); + if (cluster.HasAddBearerToToken()) + settings.AddBearerToToken = cluster.GetAddBearerToToken(); + + const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_ydb", cluster.GetToken()); + Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); settings.Raw = cluster; } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h index fc6fb03ced..1dc66146d5 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h @@ -21,7 +21,7 @@ struct TClusterMainSettings { TString Database; TString DatabaseId; bool Secure = false; - bool AddBearerToToken = false; + bool AddBearerToToken = false; TYdbClusterConfig Raw; }; diff --git a/ydb/library/yql/providers/ydb/ya.make b/ydb/library/yql/providers/ydb/ya.make index a79e0f4cfc..f6530e1a37 100644 --- a/ydb/library/yql/providers/ydb/ya.make +++ b/ydb/library/yql/providers/ydb/ya.make @@ -1,7 +1,7 @@ RECURSE( - actors - comp_nodes + actors + comp_nodes expr_nodes - proto + proto provider ) diff --git a/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp b/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp index 9c263c2b0b..203096d033 100644 --- a/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp +++ b/ydb/library/yql/public/decimal/ut/yql_decimal_ut.cpp @@ -1,23 +1,23 @@ #include <ydb/library/yql/public/decimal/yql_decimal.h> #include <ydb/library/yql/public/decimal/yql_decimal_serialize.h> - + #include <library/cpp/testing/unittest/registar.h> - + namespace NYql { -namespace NDecimal { +namespace NDecimal { Y_UNIT_TEST_SUITE(TYqlDecimalTest) { void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const TString& expected) { - TString result = ToString(v, precision, scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); + TString result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); TInt128 parsed = FromString(result, precision, scale); UNIT_ASSERT(parsed == v); - } - + } + void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { - TString result = ToString(v, precision, scale); - UNIT_ASSERT_VALUES_EQUAL(result, ""); - } - + TString result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, ""); + } + void SimpleSerializeAndDeserialize(TInt128 v, size_t expectedSize) { char buff[sizeof(TInt128)]; const auto s = Serialize(v, buff); @@ -54,95 +54,95 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { UNIT_ASSERT_VALUES_EQUAL(result, expected); } - Y_UNIT_TEST(TestZeroFormat) { - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 0), "0"); - } - - Y_UNIT_TEST(TestZeroScale) { - SimplePositiveTest(1, 5, 0, "1"); - SimplePositiveTest(10, 5, 0, "10"); - SimplePositiveTest(100, 5, 0, "100"); - SimplePositiveTest(1000, 5, 0, "1000"); - SimplePositiveTest(10000, 5, 0, "10000"); - SimpleNegativeFormatTest(100000, 5, 0); - SimpleNegativeFormatTest(1000000, 5, 0); - - // negative numbers - SimplePositiveTest(-1, 5, 0, "-1"); - SimplePositiveTest(-10, 5, 0, "-10"); - SimplePositiveTest(-100, 5, 0, "-100"); - SimplePositiveTest(-1000, 5, 0, "-1000"); - SimplePositiveTest(-10000, 5, 0, "-10000"); - SimpleNegativeFormatTest(-100000, 5, 0); - } - - Y_UNIT_TEST(TestFormats) { - // we have no trailing zeros - SimplePositiveTest(1, 15, 6, "0.000001"); - SimplePositiveTest(10, 15, 6, "0.00001"); - SimplePositiveTest(100, 15, 6, "0.0001"); - SimplePositiveTest(1000, 15, 6, "0.001"); - SimplePositiveTest(10000, 15, 6, "0.01"); - SimplePositiveTest(100000, 15, 6, "0.1"); - SimplePositiveTest(1000000, 15, 6, "1"); - SimplePositiveTest(10000000, 15, 6, "10"); - SimplePositiveTest(100000000, 15, 6, "100"); - + Y_UNIT_TEST(TestZeroFormat) { + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 0), "0"); + } + + Y_UNIT_TEST(TestZeroScale) { + SimplePositiveTest(1, 5, 0, "1"); + SimplePositiveTest(10, 5, 0, "10"); + SimplePositiveTest(100, 5, 0, "100"); + SimplePositiveTest(1000, 5, 0, "1000"); + SimplePositiveTest(10000, 5, 0, "10000"); + SimpleNegativeFormatTest(100000, 5, 0); + SimpleNegativeFormatTest(1000000, 5, 0); + + // negative numbers + SimplePositiveTest(-1, 5, 0, "-1"); + SimplePositiveTest(-10, 5, 0, "-10"); + SimplePositiveTest(-100, 5, 0, "-100"); + SimplePositiveTest(-1000, 5, 0, "-1000"); + SimplePositiveTest(-10000, 5, 0, "-10000"); + SimpleNegativeFormatTest(-100000, 5, 0); + } + + Y_UNIT_TEST(TestFormats) { + // we have no trailing zeros + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(10, 15, 6, "0.00001"); + SimplePositiveTest(100, 15, 6, "0.0001"); + SimplePositiveTest(1000, 15, 6, "0.001"); + SimplePositiveTest(10000, 15, 6, "0.01"); + SimplePositiveTest(100000, 15, 6, "0.1"); + SimplePositiveTest(1000000, 15, 6, "1"); + SimplePositiveTest(10000000, 15, 6, "10"); + SimplePositiveTest(100000000, 15, 6, "100"); + SimplePositiveTest(2020000, 15, 6, "2.02"); SimplePositiveTest(3003000, 15, 6, "3.003"); - // negative numbers - SimplePositiveTest(-1, 15, 6, "-0.000001"); - SimplePositiveTest(-10, 15, 6, "-0.00001"); - SimplePositiveTest(-100, 15, 6, "-0.0001"); - SimplePositiveTest(-1000, 15, 6, "-0.001"); - SimplePositiveTest(-10000, 15, 6, "-0.01"); - SimplePositiveTest(-100000, 15, 6, "-0.1"); - SimplePositiveTest(-1000000, 15, 6, "-1"); - SimplePositiveTest(-10000000, 15, 6, "-10"); - SimplePositiveTest(-100000000, 15, 6, "-100"); - + // negative numbers + SimplePositiveTest(-1, 15, 6, "-0.000001"); + SimplePositiveTest(-10, 15, 6, "-0.00001"); + SimplePositiveTest(-100, 15, 6, "-0.0001"); + SimplePositiveTest(-1000, 15, 6, "-0.001"); + SimplePositiveTest(-10000, 15, 6, "-0.01"); + SimplePositiveTest(-100000, 15, 6, "-0.1"); + SimplePositiveTest(-1000000, 15, 6, "-1"); + SimplePositiveTest(-10000000, 15, 6, "-10"); + SimplePositiveTest(-100000000, 15, 6, "-100"); + SimplePositiveTest(-2020000, 15, 6, "-2.02"); SimplePositiveTest(-3003000, 15, 6, "-3.003"); - SimplePositiveTest(1, 15, 6, "0.000001"); - SimplePositiveTest(12, 15, 6, "0.000012"); - SimplePositiveTest(123, 15, 6, "0.000123"); - SimplePositiveTest(1234, 15, 6, "0.001234"); - SimplePositiveTest(12345, 15, 6, "0.012345"); - SimplePositiveTest(123456, 15, 6, "0.123456"); - SimplePositiveTest(1234567, 15, 6, "1.234567"); - SimplePositiveTest(12345678, 15, 6, "12.345678"); - SimplePositiveTest(123456789, 15, 6, "123.456789"); - SimplePositiveTest(1234567898, 15, 6, "1234.567898"); - SimplePositiveTest(12345678987ll, 15, 6, "12345.678987"); - SimplePositiveTest(123456789876ll, 15, 6, "123456.789876"); - } - - Y_UNIT_TEST(TestHugeNumberFormat) { + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(12, 15, 6, "0.000012"); + SimplePositiveTest(123, 15, 6, "0.000123"); + SimplePositiveTest(1234, 15, 6, "0.001234"); + SimplePositiveTest(12345, 15, 6, "0.012345"); + SimplePositiveTest(123456, 15, 6, "0.123456"); + SimplePositiveTest(1234567, 15, 6, "1.234567"); + SimplePositiveTest(12345678, 15, 6, "12.345678"); + SimplePositiveTest(123456789, 15, 6, "123.456789"); + SimplePositiveTest(1234567898, 15, 6, "1234.567898"); + SimplePositiveTest(12345678987ll, 15, 6, "12345.678987"); + SimplePositiveTest(123456789876ll, 15, 6, "123456.789876"); + } + + Y_UNIT_TEST(TestHugeNumberFormat) { TInt128 max120 = Inf() - 1; const char max120String[] = "99999999999999999999999999999999999"; // 35 digits static_assert(sizeof(max120String) == 36, "sizeof(max120String) == 36"); SimplePositiveTest(max120, MaxPrecision, 0, max120String); SimplePositiveTest(max120 + 1, MaxPrecision, 0, "inf"); - + TInt128 min120 = -Inf() + 1; const char min120String[] = "-99999999999999999999999999999999999"; static_assert(sizeof(min120String) == 37, "sizeof(min120String) == 37"); SimplePositiveTest(min120, MaxPrecision, 0, min120String); SimplePositiveTest(min120 - 1, MaxPrecision, 0, "-inf"); - - // take spot for sign and zero before dot + + // take spot for sign and zero before dot const char min120StringAfterDot[] = "-0.99999999999999999999999999999999999"; // 35 by nine + leading zero static_assert(sizeof(min120StringAfterDot) == 39, "sizeof(min120StringAfterDot) == 39"); SimplePositiveTest(min120, MaxPrecision, MaxPrecision, min120StringAfterDot); - + SimpleNegativeFormatTest(1, MaxPrecision + 1, MaxPrecision + 1); - SimpleNegativeFormatTest(1, MaxPrecision + 1, 0); + SimpleNegativeFormatTest(1, MaxPrecision + 1, 0); SimpleNegativeFormatTest(1, 2, 3); - } + } Y_UNIT_TEST(TestFormStringRoundToEven) { UNIT_ASSERT(FromString(".51", 1, 0) == 1); @@ -340,7 +340,7 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { CheckMul<35>("inf", "-inf", "-inf"); CheckMul<35>("inf", "nan", "nan"); } -} - -} -} +} + +} +} diff --git a/ydb/library/yql/public/decimal/yql_decimal.cpp b/ydb/library/yql/public/decimal/yql_decimal.cpp index 89fc92cd49..dc0615caba 100644 --- a/ydb/library/yql/public/decimal/yql_decimal.cpp +++ b/ydb/library/yql/public/decimal/yql_decimal.cpp @@ -38,9 +38,9 @@ bool IsComparable(TInt128 v) { const char* ToString(TInt128 val, ui8 precision, ui8 scale) { if (!precision || precision > MaxPrecision || scale > precision) { - return nullptr; - } - + return nullptr; + } + if (val == Inf()) return "inf"; if (val == -Inf()) @@ -51,9 +51,9 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { return "-nan"; if (!IsNormal(val)) { - return nullptr; - } - + return nullptr; + } + if (!val) { return "0"; } @@ -61,8 +61,8 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { const bool neg = val < 0; TUint128 v = neg ? -val : val; - // log_{10}(2^120) ~= 36.12, 37 decimal places - // plus dot, zero before dot, sign and zero byte at the end + // log_{10}(2^120) ~= 36.12, 37 decimal places + // plus dot, zero before dot, sign and zero byte at the end static thread_local char str[40]; auto end = str + sizeof(str); *--end = 0; @@ -70,41 +70,41 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { auto s = end; do { - if (!precision--) { + if (!precision--) { return nullptr; - } - + } + const auto digit = ui8(v % Ten); if (digit || !scale || s != end) { *--s = "0123456789"[digit]; - } - + } + if (scale && !--scale && s != end) { *--s = '.'; - } + } } while (v /= Ten); - if (scale) { + if (scale) { do { - if (!precision--) { - return nullptr; - } - - *--s = '0'; + if (!precision--) { + return nullptr; + } + + *--s = '0'; } while (--scale); - - *--s = '.'; - } - - if (*s == '.') { - *--s = '0'; - } - - if (neg) { + + *--s = '.'; + } + + if (*s == '.') { + *--s = '0'; + } + + if (neg) { *--s = '-'; - } - + } + return s; } diff --git a/ydb/library/yql/public/issue/yql_issue.h b/ydb/library/yql/public/issue/yql_issue.h index 7b97674c9d..e64b1f86f2 100644 --- a/ydb/library/yql/public/issue/yql_issue.h +++ b/ydb/library/yql/public/issue/yql_issue.h @@ -163,10 +163,10 @@ public: (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message))); } - TIssue& SetCode(TIssueCode id, ESeverity severity) { + TIssue& SetCode(TIssueCode id, ESeverity severity) { IssueCode = id; Severity = severity; - return *this; + return *this; } ESeverity GetSeverity() const { @@ -177,10 +177,10 @@ public: return IssueCode; } - TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) { + TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) { Severity = (ESeverity)Min((ui32)issue->GetSeverity(), (ui32)Severity); Children_.push_back(issue); - return *this; + return *this; } const TVector<TIntrusivePtr<TIssue>>& GetSubIssues() const { @@ -225,12 +225,12 @@ public: return *this; } - inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) + inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) { } inline TIssues& operator=(TIssues&& rhs) { - Issues_ = std::move(rhs.Issues_); + Issues_ = std::move(rhs.Issues_); return *this; } diff --git a/ydb/library/yql/public/issue/yql_issue_manager.cpp b/ydb/library/yql/public/issue/yql_issue_manager.cpp index 7375d2d30f..3914d3b299 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager.cpp +++ b/ydb/library/yql/public/issue/yql_issue_manager.cpp @@ -168,8 +168,8 @@ TIssues TIssueManager::GetCompletedIssues() const { } res.AddIssues(CompletedIssues_); return res; -} - +} + void TIssueManager::AddIssues(const TIssues& issues) { for (auto& issue: issues) { if (auto p = CheckUniqAndLimit(issue)) { diff --git a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp index 287f63fedd..58d9f52f0a 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp +++ b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp @@ -13,8 +13,8 @@ static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 co Y_UNIT_TEST_SUITE(TIssueManagerTest) { Y_UNIT_TEST(NoErrorNoLevelTest) { TIssueManager issueManager; - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); auto issues = issueManager.GetIssues(); UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); } @@ -22,8 +22,8 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) { Y_UNIT_TEST(NoErrorOneLevelTest) { TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); issueManager.LeaveScope(); auto issues = issueManager.GetIssues(); UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); @@ -34,8 +34,8 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) { issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); issueManager.LeaveScope(); - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); issueManager.LeaveScope(); auto issues = issueManager.GetIssues(); UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); @@ -44,15 +44,15 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) { Y_UNIT_TEST(OneErrorOneLevelTest) { TIssueManager issueManager; issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - auto completedIssues1 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); + auto completedIssues1 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - auto completedIssues2 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); + auto completedIssues2 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); issueManager.LeaveScope(); - auto completedIssues3 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); - + auto completedIssues3 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); + auto issues = issueManager.GetIssues(); UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); auto scopeIssue = issues.begin(); diff --git a/ydb/library/yql/public/udf/udf_helpers.h b/ydb/library/yql/public/udf/udf_helpers.h index 2be9443559..b3c46453c7 100644 --- a/ydb/library/yql/public/udf/udf_helpers.h +++ b/ydb/library/yql/public/udf/udf_helpers.h @@ -42,7 +42,7 @@ namespace NUdf { } } -#define UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, create_impl) \ +#define UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, create_impl) \ class udfName: public ::NYql::NUdf::TBoxedValue { \ public: \ explicit udfName(::NYql::NUdf::IFunctionTypeInfoBuilder& builder) \ @@ -78,7 +78,7 @@ namespace NUdf { Y_UNUSED(userType); \ if (Name() == name) { \ typeBody if (!typesOnly) { \ - create_impl \ + create_impl \ SetIRImplementation(builder, irResourceId, irFunctionName); \ } \ return true; \ @@ -94,9 +94,9 @@ namespace NUdf { const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const -#define UDF_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \ - UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, builder.Implementation(new udfName(builder));) - +#define UDF_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \ + UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, builder.Implementation(new udfName(builder));) + #define UDF(udfName, typeBody) UDF_IMPL(udfName, typeBody, ;, ;, "", "") #define UDF_RUN_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \ @@ -182,9 +182,9 @@ namespace NUdf { #define SIMPLE_UDF_WITH_IR(udfName, signature, irResourceId, irFunctionName) \ UDF_IMPL(udfName, builder.SimpleSignature<signature>();, ;, ;, irResourceId, irFunctionName) -#define SIMPLE_UDF_WITH_CREATE_IMPL(udfName, signature, create_impl) \ - UDF_IMPL_EX(udfName, builder.SimpleSignature<signature>();, ;, ;, "", "", create_impl) - +#define SIMPLE_UDF_WITH_CREATE_IMPL(udfName, signature, create_impl) \ + UDF_IMPL_EX(udfName, builder.SimpleSignature<signature>();, ;, ;, "", "", create_impl) + #define SIMPLE_UDF_OPTIONS(udfName, signature, options) \ UDF(udfName, builder.SimpleSignature<signature>(); options;) @@ -214,9 +214,9 @@ namespace NUdf { template<bool CheckOptional, const char* TFuncName, template<class> class TFunc, typename... TUserTypes> class TUserDataTypeFuncFactory : public ::NYql::NUdf::TBoxedValue { public: - typedef bool TTypeAwareMarker; - -public: + typedef bool TTypeAwareMarker; + +public: static const ::NYql::NUdf::TStringRef& Name() { static auto name = ::NYql::NUdf::TStringRef(TFuncName, std::strlen(TFuncName)); return name; @@ -255,23 +255,23 @@ public: ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, bool typesOnly) { - if (Name() != name) { - // the only case when we return false + if (Name() != name) { + // the only case when we return false return false; } - - if (!userType) { - builder.SetError("User type is not specified"); - return true; - } - + + if (!userType) { + builder.SetError("User type is not specified"); + return true; + } + auto typeHelper = builder.TypeInfoHelper(); auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) { builder.SetError("Missing or invalid user type"); - return true; + return true; } - + auto argsTypeInspector = TTupleTypeInspector(*typeHelper, userTypeInspector.GetElementType(0)); if (!argsTypeInspector || argsTypeInspector.GetElementsCount() < 1) { builder.SetError("Missing or invalid user type arguments"); @@ -296,30 +296,30 @@ public: auto typeId = dataTypeInspector.GetTypeId(); if (!DeclareSignatureImpl<TUserTypes...>(typeId, userType, builder, typesOnly)) { - TStringBuilder sb; + TStringBuilder sb; sb << "User type " << NYql::NUdf::GetDataTypeInfo(NYql::NUdf::GetDataSlot(typeId)).Name << " is not supported"; - builder.SetError(sb); - } - - return true; + builder.SetError(sb); + } + + return true; } }; template<typename... TUdfs> class TSimpleUdfModuleHelper : public IUdfModule { - Y_HAS_SUBTYPE(TTypeAwareMarker); - + Y_HAS_SUBTYPE(TTypeAwareMarker); + public: void CleanupOnTerminate() const override { } template<typename TUdfType> void GetAllFunctionsImpl(IFunctionNamesSink& names) const { - auto r = names.Add(TUdfType::Name()); + auto r = names.Add(TUdfType::Name()); if (THasTTypeAwareMarker<TUdfType>::value) { - r->SetTypeAwareness(); - } + r->SetTypeAwareness(); + } } template<typename THead1, typename THead2, typename... TTail> diff --git a/ydb/library/yql/sql/cluster_mapping.cpp b/ydb/library/yql/sql/cluster_mapping.cpp index 46271e5a77..4b64edfd33 100644 --- a/ydb/library/yql/sql/cluster_mapping.cpp +++ b/ydb/library/yql/sql/cluster_mapping.cpp @@ -1,34 +1,34 @@ -#include "cluster_mapping.h" +#include "cluster_mapping.h" #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - -using namespace NYql; - -namespace NSQLTranslation { -TClusterMapping::TClusterMapping(const THashMap<TString, TString>& mapping) { - for (const auto& p : mapping) { - if (p.second == KikimrProviderName) { - CaseSensitiveClusters.emplace(p); - continue; - } - - TString clusterLowerCase = to_lower(p.first); - CaseInsensitiveClusters.emplace(clusterLowerCase, p.second); - } -} - -TMaybe<TString> TClusterMapping::GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { - auto providerPtr1 = CaseSensitiveClusters.FindPtr(cluster); - if (providerPtr1) { - normalizedClusterName = cluster; - return *providerPtr1; - } - - TString clusterLowerCase = to_lower(cluster); - auto providerPtr2 = CaseInsensitiveClusters.FindPtr(clusterLowerCase); - if (providerPtr2) { - normalizedClusterName = clusterLowerCase; - return *providerPtr2; - } - return Nothing(); -} -} + +using namespace NYql; + +namespace NSQLTranslation { +TClusterMapping::TClusterMapping(const THashMap<TString, TString>& mapping) { + for (const auto& p : mapping) { + if (p.second == KikimrProviderName) { + CaseSensitiveClusters.emplace(p); + continue; + } + + TString clusterLowerCase = to_lower(p.first); + CaseInsensitiveClusters.emplace(clusterLowerCase, p.second); + } +} + +TMaybe<TString> TClusterMapping::GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { + auto providerPtr1 = CaseSensitiveClusters.FindPtr(cluster); + if (providerPtr1) { + normalizedClusterName = cluster; + return *providerPtr1; + } + + TString clusterLowerCase = to_lower(cluster); + auto providerPtr2 = CaseInsensitiveClusters.FindPtr(clusterLowerCase); + if (providerPtr2) { + normalizedClusterName = clusterLowerCase; + return *providerPtr2; + } + return Nothing(); +} +} diff --git a/ydb/library/yql/sql/cluster_mapping.h b/ydb/library/yql/sql/cluster_mapping.h index c2de216371..73328d8232 100644 --- a/ydb/library/yql/sql/cluster_mapping.h +++ b/ydb/library/yql/sql/cluster_mapping.h @@ -1,16 +1,16 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/generic/maybe.h> - -namespace NSQLTranslation { -class TClusterMapping { -public: - explicit TClusterMapping(const THashMap<TString, TString>& mapping); - TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const; - -private: - THashMap<TString, TString> CaseSensitiveClusters; - THashMap<TString, TString> CaseInsensitiveClusters; -}; -} +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/maybe.h> + +namespace NSQLTranslation { +class TClusterMapping { +public: + explicit TClusterMapping(const THashMap<TString, TString>& mapping); + TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const; + +private: + THashMap<TString, TString> CaseSensitiveClusters; + THashMap<TString, TString> CaseInsensitiveClusters; +}; +} diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h index 0b9194a249..f9a3689997 100644 --- a/ydb/library/yql/sql/settings/translation_settings.h +++ b/ydb/library/yql/sql/settings/translation_settings.h @@ -56,7 +56,7 @@ namespace NSQLTranslation { THashMap<TString, TString> ClusterMapping; TString PathPrefix; - // keys (cluster name) should be normalized + // keys (cluster name) should be normalized THashMap<TString, TString> ClusterPathPrefixes; THashMap<TString, TString> ModuleMapping; THashSet<TString> Libraries; diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g index 3b616b9c9f..f6b070895f 100644 --- a/ydb/library/yql/sql/v0/SQL.g +++ b/ydb/library/yql/sql/v0/SQL.g @@ -322,7 +322,7 @@ evaluate_if_stmt: EVALUATE IF expr do_stmt (ELSE do_stmt)?; evaluate_for_stmt: EVALUATE FOR bind_parameter IN expr do_stmt (ELSE do_stmt)?; table_ref: opt_id_prefix (table_key | id_expr LPAREN table_arg (COMMA table_arg)* RPAREN) table_hints?; -table_key: id_table_or_at (COLON id_or_string)?; +table_key: id_table_or_at (COLON id_or_string)?; table_arg: AT? expr (COLON id_or_string)?; table_hints: WITH (id_or_string | pure_column_list); @@ -414,13 +414,13 @@ id: IDENTIFIER | keyword; id_schema: IDENTIFIER | keyword_restricted; id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat | keyword_in_uncompat; in_id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat; -id_table: IDENTIFIER | keyword_restricted; -id_table_or_at: AT? id_table; +id_table: IDENTIFIER | keyword_restricted; +id_table_or_at: AT? id_table; id_or_at: AT? id; opt_id_prefix: (id_or_string DOT)?; -keyword: keyword_restricted | keyword_alter_uncompat | keyword_table_uncompat; +keyword: keyword_restricted | keyword_alter_uncompat | keyword_table_uncompat; keyword_restricted: keyword_compat | keyword_expr_uncompat | keyword_select_uncompat | keyword_in_uncompat; keyword_expr_uncompat: @@ -447,11 +447,11 @@ keyword_expr_uncompat: | WHERE ; -keyword_table_uncompat: +keyword_table_uncompat: ERASE | STREAM -; - +; + keyword_select_uncompat: ALL | AS diff --git a/ydb/library/yql/sql/v0/context.cpp b/ydb/library/yql/sql/v0/context.cpp index a3a9013f2b..131e90b765 100644 --- a/ydb/library/yql/sql/v0/context.cpp +++ b/ydb/library/yql/sql/v0/context.cpp @@ -46,7 +46,7 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, TIssues& issues) - : ClusterMapping(settings.ClusterMapping) + : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) , Settings(settings) @@ -118,8 +118,8 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP } else if (action == EWarningAction::DISABLE) { return Cnull; } - } - + } + // we have the last cell for issue, let's fill it with our internal error if (severity >= TSeverityIds::S_WARNING) { const bool aboveHalf = Issues.Size() > Settings.MaxErrors / 2; @@ -135,8 +135,8 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP if (Settings.MaxErrors <= Issues.Size()) { ythrow NProtoAST::TTooManyErrors() << "Too many issues"; } - } - + } + Issues.AddIssue(TIssue(pos, TString())); auto& curIssue = Issues.back(); curIssue.Severity = severity; @@ -156,14 +156,14 @@ bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) { return true; } - TString normalizedClusterName; - if (!GetClusterProvider(*arg, normalizedClusterName)) { - Error() << "Unknown cluster or provider: " << *arg; + TString normalizedClusterName; + if (!GetClusterProvider(*arg, normalizedClusterName)) { + Error() << "Unknown cluster or provider: " << *arg; IncrementMonCounter("sql_errors", "BadPragmaValue"); return false; } - ClusterPathPrefixes[normalizedClusterName] = value; + ClusterPathPrefixes[normalizedClusterName] = value; } else { PathPrefix = value; } @@ -172,11 +172,11 @@ bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) { } TNodePtr TContext::GetPrefixedPath(const TString& cluster, const TDeferredAtom& path) { - auto* clusterPrefix = ClusterPathPrefixes.FindPtr(cluster); + auto* clusterPrefix = ClusterPathPrefixes.FindPtr(cluster); if (clusterPrefix && !clusterPrefix->empty()) { return AddTablePathPrefix(*this, *clusterPrefix, path); } else { - auto provider = GetClusterProvider(cluster); + auto provider = GetClusterProvider(cluster); YQL_ENSURE(provider.Defined()); auto* providerPrefix = ProviderPathPrefixes.FindPtr(*provider); diff --git a/ydb/library/yql/sql/v0/context.h b/ydb/library/yql/sql/v0/context.h index 45f40fae61..16fd951ca4 100644 --- a/ydb/library/yql/sql/v0/context.h +++ b/ydb/library/yql/sql/v0/context.h @@ -20,7 +20,7 @@ namespace NSQLTranslationV0 { typedef TMap<TString, TNodePtr> TNamedNodesMap; - class TContext { + class TContext { public: TContext(const NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues); @@ -58,26 +58,26 @@ namespace NSQLTranslationV0 { } } - bool HasCluster(const TString& cluster) const { - return GetClusterProvider(cluster).Defined(); - } - + bool HasCluster(const TString& cluster) const { + return GetClusterProvider(cluster).Defined(); + } + TMaybe<TString> GetClusterProvider(const TString& cluster) const { - TString unusedNormalizedClusterName; - return GetClusterProvider(cluster, unusedNormalizedClusterName); - } - - TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { - auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); + TString unusedNormalizedClusterName; + return GetClusterProvider(cluster, unusedNormalizedClusterName); + } + + TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { + auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); if (!provider) { if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { - normalizedClusterName = cluster; + normalizedClusterName = cluster; return TString(NYql::KikimrProviderName); } return Nothing(); } - return provider; + return provider; } TString GetServiceName(const ISource& source) const; @@ -118,7 +118,7 @@ namespace NSQLTranslationV0 { private: NYql::TPosition Position; THolder<TStringOutput> IssueMsgHolder; - NSQLTranslation::TClusterMapping ClusterMapping; + NSQLTranslation::TClusterMapping ClusterMapping; TString PathPrefix; THashMap<TString, TString> ProviderPathPrefixes; THashMap<TString, TString> ClusterPathPrefixes; diff --git a/ydb/library/yql/sql/v0/list_builtin.cpp b/ydb/library/yql/sql/v0/list_builtin.cpp index 64b016610d..b3fe59db1d 100644 --- a/ydb/library/yql/sql/v0/list_builtin.cpp +++ b/ydb/library/yql/sql/v0/list_builtin.cpp @@ -207,7 +207,7 @@ TNodePtr TListHasBuiltin::GetUpdateLambda() { bool TListFold1Builtin::DoInit(TContext& ctx, ISource* src) { if (Args.size() != 1) { - ctx.Error(Pos) << "Folding list with " << OpName << " requires only one parameter"; + ctx.Error(Pos) << "Folding list with " << OpName << " requires only one parameter"; return false; } if (!Args[0]->Init(ctx, src)) { diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp index 57f0c6a561..c211b3e3c0 100644 --- a/ydb/library/yql/sql/v0/node.cpp +++ b/ydb/library/yql/sql/v0/node.cpp @@ -28,7 +28,7 @@ TString ErrorDistinctByGroupKey(const TString& column) { TTableRef::TTableRef(const TString& refName, const TString& cluster, TNodePtr keys) : RefName(refName) - , Cluster(cluster) + , Cluster(cluster) , Keys(keys) { } @@ -2196,7 +2196,7 @@ public: ui32 idx = 1; TString cluster; if (Ids.size() > 2) { - cluster = Ids[idx].Name; + cluster = Ids[idx].Name; ++idx; } @@ -2204,8 +2204,8 @@ public: return Nothing(); } - TString normalizedClusterName; - if (!cluster.empty() && !ctx.GetClusterProvider(cluster, normalizedClusterName)) { + TString normalizedClusterName; + if (!cluster.empty() && !ctx.GetClusterProvider(cluster, normalizedClusterName)) { hasErrors = true; ctx.Error() << "Unknown cluster: " << cluster; return Nothing(); @@ -2216,7 +2216,7 @@ public: return Nothing(); } - return std::make_pair(normalizedClusterName, tableName); + return std::make_pair(normalizedClusterName, tableName); } TSourcePtr TryMakeSource(TContext& ctx, const TString& view, bool& hasErrors) { diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index 4c7cfc273b..bb1394b2b2 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -984,7 +984,7 @@ namespace NSQLTranslationV0 { }; TSourcePtr BuildReduce(TPosition pos, ReduceMode mode, TSourcePtr source, TVector<TSortSpecificationPtr>&& orderBy, TVector<TNodePtr>&& keys, TVector<TNodePtr>&& args, TNodePtr udf, TNodePtr having, const TWriteSettings& settings); - TSourcePtr BuildProcess(TPosition pos, TSourcePtr source, TNodePtr with, TVector<TNodePtr>&& terms, bool listCall, bool stream, const TWriteSettings& settings); + TSourcePtr BuildProcess(TPosition pos, TSourcePtr source, TNodePtr with, TVector<TNodePtr>&& terms, bool listCall, bool stream, const TWriteSettings& settings); TNodePtr BuildSelectResult(TPosition pos, TSourcePtr source, bool writeResult, bool inSubquery); diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 7752b28b01..e086d836fc 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -53,8 +53,8 @@ public: } if (mode == ITableKeys::EBuildKeysMode::INPUT && IsQueryMode(ctx.Settings.Mode) && - ctx.GetClusterProvider(Cluster).GetRef() != "kikimr" && - ctx.GetClusterProvider(Cluster).GetRef() != "rtmr") { + ctx.GetClusterProvider(Cluster).GetRef() != "kikimr" && + ctx.GetClusterProvider(Cluster).GetRef() != "rtmr") { key = Y("MrTableConcat", key); } @@ -822,21 +822,21 @@ protected: } for (auto& cluster: *clusters) { - TString normalizedClusterName; - auto service = ctx.GetClusterProvider(cluster, normalizedClusterName); + TString normalizedClusterName; + auto service = ctx.GetClusterProvider(cluster, normalizedClusterName); if (!service) { ctx.Error(ctx.Pos()) << "Unknown cluster: " << cluster; return false; } - auto sinkName = normalizedClusterName + "_sink"; + auto sinkName = normalizedClusterName + "_sink"; auto op = ProduceOperation(ctx, sinkName, *service); if (!op) { return false; } - block = L(block, Y("let", sinkName, Y("DataSink", BuildQuotedAtom(Pos, *service), BuildQuotedAtom(Pos, normalizedClusterName)))); + block = L(block, Y("let", sinkName, Y("DataSink", BuildQuotedAtom(Pos, *service), BuildQuotedAtom(Pos, normalizedClusterName)))); block = L(block, op); } @@ -1083,7 +1083,7 @@ public: cluster = "$all"; serviceName = Prefix; } else { - serviceName = *ctx.GetClusterProvider(Prefix, cluster); + serviceName = *ctx.GetClusterProvider(Prefix, cluster); } auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName)); diff --git a/ydb/library/yql/sql/v0/select.cpp b/ydb/library/yql/sql/v0/select.cpp index ef47023be1..219f1323bc 100644 --- a/ydb/library/yql/sql/v0/select.cpp +++ b/ydb/library/yql/sql/v0/select.cpp @@ -1660,17 +1660,17 @@ public: TPosition pos, TSourcePtr source, TNodePtr with, - TVector<TNodePtr>&& terms, + TVector<TNodePtr>&& terms, bool listCall, - bool stream, + bool stream, const TWriteSettings& settings ) : IRealSource(pos) , Source(std::move(source)) , With(with) - , Terms(std::move(terms)) + , Terms(std::move(terms)) , ListCall(listCall) - , Stream(stream) + , Stream(stream) , Settings(settings) { } @@ -1689,16 +1689,16 @@ public: return false; } - if (Stream && !Source->IsStream()) { - ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources"; - return false; - } - - if (!Stream && Source->IsStream() && !ctx.PragmaDirectRead) { - ctx.Error(Pos) << "PROCESS STREAM must be used for streaming sources"; - return false; - } - + if (Stream && !Source->IsStream()) { + ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources"; + return false; + } + + if (!Stream && Source->IsStream() && !ctx.PragmaDirectRead) { + ctx.Error(Pos) << "PROCESS STREAM must be used for streaming sources"; + return false; + } + auto src = Source.Get(); if (!With) { src->AllColumns(); @@ -1820,17 +1820,17 @@ public: return false; } - bool IsStream() const override { - return Stream; - } - + bool IsStream() const override { + return Stream; + } + TWriteSettings GetWriteSettings() const override { return Settings; } TNodePtr DoClone() const final { return new TProcessSource(Pos, Source->CloneSource(), SafeClone(With), - CloneContainer(Terms), ListCall, Stream, Settings); + CloneContainer(Terms), ListCall, Stream, Settings); } private: @@ -1857,7 +1857,7 @@ private: TNodePtr TermsGround; TVector<TNodePtr> Terms; const bool ListCall; - const bool Stream; + const bool Stream; const TWriteSettings Settings; }; @@ -1867,10 +1867,10 @@ TSourcePtr BuildProcess( TNodePtr with, TVector<TNodePtr>&& terms, bool listCall, - bool stream, + bool stream, const TWriteSettings& settings ) { - return new TProcessSource(pos, std::move(source), with, std::move(terms), listCall, stream, settings); + return new TProcessSource(pos, std::move(source), with, std::move(terms), listCall, stream, settings); } class TNestedProxySource: public IProxySource { diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index b028fc0439..f812d63e72 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -73,8 +73,8 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { return GetKeywordId(ctx, node.GetAlt_keyword1().GetRule_keyword_restricted1()); case TRule_keyword::kAltKeyword2: return GetIdentifier(ctx, node.GetAlt_keyword2().GetRule_keyword_alter_uncompat1()); - case TRule_keyword::kAltKeyword3: - return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); + case TRule_keyword::kAltKeyword3: + return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); default: Y_FAIL("You should change implementation according grammar changes"); } @@ -116,24 +116,24 @@ static std::pair<bool, TString> Id(const TRule_id_or_at& node, TTranslation& ctx return std::make_pair(hasAt, Id(node.GetRule_id2(), ctx) ); } -static TString Id(const TRule_id_table& node, TTranslation& ctx) { - // id_table: IDENTIFIER | keyword_restricted; - switch (node.Alt_case()) { - case TRule_id_table::kAltIdTable1: - return ctx.Identifier(node.GetAlt_id_table1().GetToken1()); - case TRule_id_table::kAltIdTable2: - return GetKeyword(ctx, node.GetAlt_id_table2().GetRule_keyword_restricted1()); - default: - Y_FAIL("You should change implementation according grammar changes"); - } -} - -static std::pair<bool, TString> Id(const TRule_id_table_or_at& node, TTranslation& ctx) { - // id_table_or_at: AT? id_table; - bool hasAt = node.HasBlock1(); - return std::make_pair(hasAt, Id(node.GetRule_id_table2(), ctx)); -} - +static TString Id(const TRule_id_table& node, TTranslation& ctx) { + // id_table: IDENTIFIER | keyword_restricted; + switch (node.Alt_case()) { + case TRule_id_table::kAltIdTable1: + return ctx.Identifier(node.GetAlt_id_table1().GetToken1()); + case TRule_id_table::kAltIdTable2: + return GetKeyword(ctx, node.GetAlt_id_table2().GetRule_keyword_restricted1()); + default: + Y_FAIL("You should change implementation according grammar changes"); + } +} + +static std::pair<bool, TString> Id(const TRule_id_table_or_at& node, TTranslation& ctx) { + // id_table_or_at: AT? id_table; + bool hasAt = node.HasBlock1(); + return std::make_pair(hasAt, Id(node.GetRule_id_table2(), ctx)); +} + static TString Id(const TRule_id_expr& node, TTranslation& ctx) { switch (node.Alt_case()) { case TRule_id_expr::kAltIdExpr1: @@ -190,16 +190,16 @@ static TString IdOrString(const TRule_id_or_string& node, TTranslation& ctx, boo Y_FAIL("You should change implementation according grammar changes"); } -static TString IdOrStringAsCluster(const TRule_id_or_string& node, TTranslation& ctx) { - TString cluster = IdOrString(node, ctx); - TString normalizedClusterName; - if (!ctx.Context().GetClusterProvider(cluster, normalizedClusterName)) { - ctx.Error() << "Unknown cluster: " << cluster; - return {}; - } - return normalizedClusterName; -} - +static TString IdOrStringAsCluster(const TRule_id_or_string& node, TTranslation& ctx) { + TString cluster = IdOrString(node, ctx); + TString normalizedClusterName; + if (!ctx.Context().GetClusterProvider(cluster, normalizedClusterName)) { + ctx.Error() << "Unknown cluster: " << cluster; + return {}; + } + return normalizedClusterName; +} + static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& ctx, const TString& defaultStr = {}) { if (!node.HasBlock1()) { return defaultStr; @@ -207,13 +207,13 @@ static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& c return IdOrString(node.GetBlock1().GetRule_id_or_string1(), ctx); } -static TString OptIdPrefixAsClusterStr(const TRule_opt_id_prefix& node, TTranslation& ctx, const TString& defaultStr = {}) { - if (!node.HasBlock1()) { - return defaultStr; - } - return IdOrStringAsCluster(node.GetBlock1().GetRule_id_or_string1(), ctx); -} - +static TString OptIdPrefixAsClusterStr(const TRule_opt_id_prefix& node, TTranslation& ctx, const TString& defaultStr = {}) { + if (!node.HasBlock1()) { + return defaultStr; + } + return IdOrStringAsCluster(node.GetBlock1().GetRule_id_or_string1(), ctx); +} + static void PureColumnListStr(const TRule_pure_column_list& node, TTranslation& ctx, TVector<TString>& outList) { outList.push_back(IdOrString(node.GetRule_id_or_string2(), ctx)); for (auto& block: node.GetBlock3()) { @@ -388,7 +388,7 @@ static std::pair<TString, TString> TableKeyImpl(const std::pair<bool, TString>& } static std::pair<TString, TString> TableKeyImpl(const TRule_table_key& node, TTranslation& ctx) { - auto nameWithAt(Id(node.GetRule_id_table_or_at1(), ctx)); + auto nameWithAt(Id(node.GetRule_id_table_or_at1(), ctx)); TString view; if (node.HasBlock2()) { view = IdOrString(node.GetBlock2().GetRule_id_or_string2(), ctx); @@ -465,10 +465,10 @@ static TTableRef SimpleTableRefImpl(const TRule_simple_table_ref& node, NSQLTran switch (node.GetBlock1().Alt_case()) { case TRule_simple_table_ref_TBlock1::AltCase::kAlt1: { if (node.GetBlock1().GetAlt1().GetBlock1().HasRule_opt_id_prefix1()) { - cluster = OptIdPrefixAsClusterStr(node.GetBlock1().GetAlt1().GetBlock1().GetRule_opt_id_prefix1(), ctx, ctx.Context().CurrCluster); - if (!cluster && ctx.Context().CurrCluster) { - return TTableRef(ctx.Context().MakeName("table"), ctx.Context().CurrCluster, nullptr); - } + cluster = OptIdPrefixAsClusterStr(node.GetBlock1().GetAlt1().GetBlock1().GetRule_opt_id_prefix1(), ctx, ctx.Context().CurrCluster); + if (!cluster && ctx.Context().CurrCluster) { + return TTableRef(ctx.Context().MakeName("table"), ctx.Context().CurrCluster, nullptr); + } } tr.ConstructInPlace(ctx.Context().MakeName("table"), cluster.empty() ? ctx.Context().CurrCluster : cluster, nullptr); @@ -908,10 +908,10 @@ TTableRef TSqlTranslation::TableRefImpl(const TRule_table_ref& node) { Ctx.Error() << "Cluster should not be used in limited view"; return TTableRef(Ctx.MakeName("table"), Ctx.CurrCluster, nullptr); } - auto cluster = OptIdPrefixAsClusterStr(node.GetRule_opt_id_prefix1(), *this, Context().CurrCluster); - if (!cluster) { - return TTableRef(Ctx.MakeName("table"), Ctx.CurrCluster, nullptr); - } + auto cluster = OptIdPrefixAsClusterStr(node.GetRule_opt_id_prefix1(), *this, Context().CurrCluster); + if (!cluster) { + return TTableRef(Ctx.MakeName("table"), Ctx.CurrCluster, nullptr); + } TTableRef tr(Context().MakeName("table"), cluster, nullptr); TPosition pos(Context().Pos()); @@ -2881,17 +2881,17 @@ bool TSqlSelect::SortSpecificationList(const TRule_sort_specification_list& node TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteSettings& settings, TPosition& selectPos) { // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING call_expr (AS id_or_string)? - // (WHERE expr)? (HAVING expr)?)? - + // (WHERE expr)? (HAVING expr)?)? + Token(node.GetToken1()); TPosition startPos(Ctx.Pos()); - - const bool stream = node.HasBlock2(); + + const bool stream = node.HasBlock2(); if (!selectPos) { selectPos = startPos; } - TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source3())); + TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source3())); if (!source) { return nullptr; } @@ -2909,7 +2909,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS bool hasUsing = node.HasBlock5(); if (!hasUsing) { - return BuildProcess(startPos, std::move(source), nullptr, {}, true, stream, settings); + return BuildProcess(startPos, std::move(source), nullptr, {}, true, stream, settings); } const auto& block5 = node.GetBlock5(); @@ -2921,7 +2921,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS } Ctx.IncrementMonCounter("sql_features", "ProcessWhere"); } else { - Ctx.IncrementMonCounter("sql_features", stream ? "ProcessStream" : "Process"); + Ctx.IncrementMonCounter("sql_features", stream ? "ProcessStream" : "Process"); } if (block5.HasBlock5()) { @@ -2979,7 +2979,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS with->SetLabel(IdOrString(block5.GetBlock3().GetRule_id_or_string2(), *this)); } - return BuildProcess(startPos, std::move(source), with, std::move(args), listCall, stream, settings); + return BuildProcess(startPos, std::move(source), with, std::move(args), listCall, stream, settings); } TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSettings& settings, TPosition& selectPos) { @@ -4048,10 +4048,10 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { if (tableRef.HasBlock1()) { switch (tableRef.GetBlock1().Alt_case()) { case TRule_simple_table_ref_TBlock1::AltCase::kAlt1: { - cluster = OptIdPrefixAsClusterStr(tableRef.GetBlock1().GetAlt1().GetBlock1().GetRule_opt_id_prefix1(), *this, Ctx.CurrCluster); - if (!cluster && Ctx.CurrCluster) { - return nullptr; - } + cluster = OptIdPrefixAsClusterStr(tableRef.GetBlock1().GetAlt1().GetBlock1().GetRule_opt_id_prefix1(), *this, Ctx.CurrCluster); + if (!cluster && Ctx.CurrCluster) { + return nullptr; + } auto id = Id(tableRef.GetBlock1().GetAlt1().GetBlock1().GetRule_id_or_at2(), *this); nameOrAt = std::make_pair(id.first, TDeferredAtom(Ctx.Pos(), id.second)); break; @@ -4368,10 +4368,10 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& case TRule_sql_stmt_core::kAltSqlStmtCore6: { const auto& rule = core.GetAlt_sql_stmt_core6().GetRule_use_stmt1(); Token(rule.GetToken1()); - Ctx.CurrCluster = IdOrStringAsCluster(rule.GetRule_id_or_string2(), *this); - if (!Ctx.CurrCluster) { - return false; - } + Ctx.CurrCluster = IdOrStringAsCluster(rule.GetRule_id_or_string2(), *this); + if (!Ctx.CurrCluster) { + return false; + } break; } case TRule_sql_stmt_core::kAltSqlStmtCore7: { @@ -5236,7 +5236,7 @@ google::protobuf::Message* SqlAST(const TString& query, const TString& queryName #endif NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, ""); NProtoAST::TProtoASTBuilder<NALP::SQLParser, NALP::SQLLexer> builder(query, queryName, arena); - return builder.BuildAST(collector); + return builder.BuildAST(collector); } google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, google::protobuf::Arena* arena) { @@ -5256,15 +5256,15 @@ TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) } TSqlQuery query(ctx, ctx.Settings.Mode, true); TNodePtr node(query.Build(static_cast<const TSQLParserAST&>(protoAst))); - try { - if (node && node->Init(ctx, nullptr)) { - return node->Translate(ctx); - } - } catch (const NProtoAST::TTooManyErrors&) { - // do not add error issue, no room for it + try { + if (node && node->Init(ctx, nullptr)) { + return node->Translate(ctx); + } + } catch (const NProtoAST::TTooManyErrors&) { + // do not add error issue, no room for it } - - return nullptr; + + return nullptr; } void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message& protoAst, @@ -5277,7 +5277,7 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message ctx.IncrementMonCounter("sql_errors", "AstToYqlError"); } else { ctx.IncrementMonCounter("sql_errors", "AstToYqlSilentError"); - ctx.Error() << "Error occurred on parse SQL query, but no error is collected" << + ctx.Error() << "Error occurred on parse SQL query, but no error is collected" << ", please send this request over bug report into YQL interface or write on yql@ maillist"; } } diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 3a70aef93c..0287d74833 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -902,17 +902,17 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(StrayUTF8) { /// 'c' in plato is russian here - NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); + NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); UNIT_ASSERT(!res.Root); - - TString a1 = Err2Str(res); + + TString a1 = Err2Str(res); TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input... - + <main>:1:15: Error: Unexpected character : cannot match to any predicted input... - -)foo"); - - UNIT_ASSERT_NO_DIFF(a1, a2); + +)foo"); + + UNIT_ASSERT_NO_DIFF(a1, a2); } Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { @@ -999,7 +999,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(SelectNoCluster) { NYql::TAstParseResult res = SqlToYql("select foo from bar"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); } Y_UNIT_TEST(SelectDuplicateColumns) { @@ -1196,13 +1196,13 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(UpsertValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: UPSERT INTO ... VALUES requires specification of table columns\n"); } Y_UNIT_TEST(ReplaceValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:20: Error: REPLACE INTO ... VALUES requires specification of table columns\n"); } @@ -1501,7 +1501,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(InsertIntoWithTruncateKikimr) { - NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT INTO WITH TRUNCATE is not supported for kikimr tables\n"); } @@ -1513,7 +1513,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(UpsertWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:39: Error: VALUES have 2 columns, UPSERT INTO expects: 3\n"); } @@ -1666,23 +1666,23 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: IF EXISTS in DROP TABLE is not supported.\n"); } - + Y_UNIT_TEST(TooManyErrors) { - const char* q = R"( - USE plato; - $a1 = A; - $a2 = DateTime::ToDate($a1, $a1); - $a3 = DateTime::ToDate($a2, $a2); - $a4 = DateTime::ToDate($a3, $a3); - - $s = (select b from plato.abc); - - select $a4 from $s; -)"; - - NYql::TAstParseResult res = SqlToYql(q, 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), + const char* q = R"( + USE plato; + $a1 = A; + $a2 = DateTime::ToDate($a1, $a1); + $a3 = DateTime::ToDate($a2, $a2); + $a4 = DateTime::ToDate($a3, $a3); + + $s = (select b from plato.abc); + + select $a4 from $s; +)"; + + NYql::TAstParseResult res = SqlToYql(q, 10); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), R"(<main>:3:15: Error: Column A is not in source column set. Did you mean b? <main>:3:15: Error: Column A is not in source column set. Did you mean b? <main>:3:15: Error: Column A is not in source column set. Did you mean b? @@ -1693,8 +1693,8 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { <main>:3:15: Error: Column A is not in source column set. Did you mean b? <main>:3:15: Error: Column A is not in source column set. Did you mean b? <main>: Error: Too many issues, code: 1 -)"); - } +)"); + } Y_UNIT_TEST(ShouldCloneBindingForNamedParameter) { NYql::TAstParseResult res = SqlToYql(R"( @@ -1721,24 +1721,24 @@ select FormatType($f()); UNIT_ASSERT(res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); } - - Y_UNIT_TEST(YtCaseInsensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); - UNIT_ASSERT(res.Root); - - res = SqlToYql("use PlatO; select * from foo;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrCaseSensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), R"(<main>:1:15: Error: Unknown cluster: PlatO -<main>:1:15: Error: No cluster name given and no default cluster is selected -)"); - - res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); - } + + Y_UNIT_TEST(YtCaseInsensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); + UNIT_ASSERT(res.Root); + + res = SqlToYql("use PlatO; select * from foo;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(KikimrCaseSensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), R"(<main>:1:15: Error: Unknown cluster: PlatO +<main>:1:15: Error: No cluster name given and no default cluster is selected +)"); + + res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); + } } diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index ef394c7a00..89e1194497 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -798,7 +798,7 @@ keyword_expr_uncompat: | JSON_EXISTS | JSON_VALUE | JSON_QUERY - | LOCAL + | LOCAL | NOT | NULL | PROCESS @@ -811,12 +811,12 @@ keyword_expr_uncompat: | WHERE ; -keyword_table_uncompat: +keyword_table_uncompat: ANY | ERASE | STREAM -; - +; + keyword_select_uncompat: ALL | AS diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index 43ef1243f1..3e36a31c06 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -480,7 +480,7 @@ public: return false; } - Args[1] = BuildQuotedAtom(Args[1]->GetPos(), *literal); + Args[1] = BuildQuotedAtom(Args[1]->GetPos(), *literal); } else { if (Service.empty()) { ctx.Error(GetPos()) << GetOpName() << " requires either one of \"yt\"/\"kikimr\"/\"rtmr\" as second argument or current cluster name"; diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp index c08fe96cae..3ea9ea097b 100644 --- a/ydb/library/yql/sql/v1/context.cpp +++ b/ydb/library/yql/sql/v1/context.cpp @@ -67,7 +67,7 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = { TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, TIssues& issues) - : ClusterMapping(settings.ClusterMapping) + : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) , Settings(settings) @@ -171,8 +171,8 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP } else if (action == EWarningAction::DISABLE) { return Cnull; } - } - + } + // we have the last cell for issue, let's fill it with our internal error if (severity >= TSeverityIds::S_WARNING) { const bool aboveHalf = Issues.Size() > Settings.MaxErrors / 2; @@ -188,8 +188,8 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP if (Settings.MaxErrors <= Issues.Size()) { ythrow NProtoAST::TTooManyErrors() << "Too many issues"; } - } - + } + Issues.AddIssue(TIssue(pos, TString())); auto& curIssue = Issues.back(); curIssue.Severity = severity; @@ -209,14 +209,14 @@ bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) { return true; } - TString normalizedClusterName; - if (!GetClusterProvider(*arg, normalizedClusterName)) { - Error() << "Unknown cluster or provider: " << *arg; + TString normalizedClusterName; + if (!GetClusterProvider(*arg, normalizedClusterName)) { + Error() << "Unknown cluster or provider: " << *arg; IncrementMonCounter("sql_errors", "BadPragmaValue"); return false; } - ClusterPathPrefixes[normalizedClusterName] = value; + ClusterPathPrefixes[normalizedClusterName] = value; } else { PathPrefix = value; } @@ -225,7 +225,7 @@ bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) { } TNodePtr TContext::GetPrefixedPath(const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& path) { - auto* clusterPrefix = cluster.GetLiteral() ? ClusterPathPrefixes.FindPtr(*cluster.GetLiteral()) : nullptr; + auto* clusterPrefix = cluster.GetLiteral() ? ClusterPathPrefixes.FindPtr(*cluster.GetLiteral()) : nullptr; if (clusterPrefix && !clusterPrefix->empty()) { return AddTablePathPrefix(*this, *clusterPrefix, path); } else { @@ -287,10 +287,10 @@ TString TContext::AddSimpleUdf(const TString& udf) { return name; } -void TContext::SetPackageVersion(const TString& packageName, ui32 version) { - PackageVersions[packageName] = version; -} - +void TContext::SetPackageVersion(const TString& packageName, ui32 version) { + PackageVersions[packageName] = version; +} + void TScopedState::UseCluster(const TString& service, const TDeferredAtom& cluster) { YQL_ENSURE(!cluster.Empty()); if (cluster.GetLiteral()) { diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index 8951757486..21b9693495 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -21,10 +21,10 @@ namespace NSQLTranslationV1 { return name == "$_"; } - inline bool IsStreamingService(const TString& service) { - return service == NYql::RtmrProviderName || service == NYql::PqProviderName; - } - + inline bool IsStreamingService(const TString& service) { + return service == NYql::RtmrProviderName || service == NYql::PqProviderName; + } + struct TNodeWithUsageInfo : public TThrRefBase { explicit TNodeWithUsageInfo(const TNodePtr& node, TPosition namePos, int level) : Node(node) @@ -76,7 +76,7 @@ namespace NSQLTranslationV1 { AsStringLiteral, }; - class TContext { + class TContext { public: TContext(const NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues); @@ -116,26 +116,26 @@ namespace NSQLTranslationV1 { } } - bool HasCluster(const TString& cluster) const { - return GetClusterProvider(cluster).Defined(); - } - + bool HasCluster(const TString& cluster) const { + return GetClusterProvider(cluster).Defined(); + } + TMaybe<TString> GetClusterProvider(const TString& cluster) const { - TString unusedNormalizedClusterName; - return GetClusterProvider(cluster, unusedNormalizedClusterName); - } - - TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { - auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); + TString unusedNormalizedClusterName; + return GetClusterProvider(cluster, unusedNormalizedClusterName); + } + + TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { + auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); if (!provider) { if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { - normalizedClusterName = cluster; + normalizedClusterName = cluster; return TString(NYql::KikimrProviderName); } return Nothing(); } - return provider; + return provider; } bool HasNonYtProvider(const ISource& source) const; @@ -161,9 +161,9 @@ namespace NSQLTranslationV1 { bool AddExport(TPosition symbolPos, const TString& symbolName); TString AddImport(const TVector<TString>& modulePath); TString AddSimpleUdf(const TString& udf); - void SetPackageVersion(const TString& packageName, ui32 version); + void SetPackageVersion(const TString& packageName, ui32 version); - bool IsStreamingService(const TStringBuf service) const; + bool IsStreamingService(const TStringBuf service) const; bool CheckColumnReference(TPosition pos, const TString& name) { const bool allowed = GetColumnReferenceState() != EColumnRefState::Deny; @@ -188,7 +188,7 @@ namespace NSQLTranslationV1 { private: NYql::TPosition Position; THolder<TStringOutput> IssueMsgHolder; - NSQLTranslation::TClusterMapping ClusterMapping; + NSQLTranslation::TClusterMapping ClusterMapping; TString PathPrefix; THashMap<TString, TString> ProviderPathPrefixes; THashMap<TString, TString> ClusterPathPrefixes; @@ -255,9 +255,9 @@ namespace NSQLTranslationV1 { // if FlexibleTypes=true, emit TypeOrMember callable and resolve Type/Column uncertainty on type annotation stage, otherwise always emit Type bool FlexibleTypes = false; THashMap<TString, TMaybe<TString>> Libraries; // alias -> optional file - THashMap<TString, ui32> PackageVersions; + THashMap<TString, ui32> PackageVersions; NYql::TWarningPolicy WarningPolicy; - TString PqReadByRtmrCluster; + TString PqReadByRtmrCluster; }; class TColumnRefScope { diff --git a/ydb/library/yql/sql/v1/join.cpp b/ydb/library/yql/sql/v1/join.cpp index 29f5b3ec4b..9930541637 100644 --- a/ydb/library/yql/sql/v1/join.cpp +++ b/ydb/library/yql/sql/v1/join.cpp @@ -196,10 +196,10 @@ public: JoinLinkSettings.push_back(linkSettings); } - bool IsStream() const override { - return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); }); - } - + bool IsStream() const override { + return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); }); + } + protected: static TString FullColumnName(const TColumnNode& column) { auto sourceName = *column.GetSourceName(); diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index 870bbd9cc7..7e8c96dbef 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -1380,7 +1380,7 @@ bool ISource::AddExpressions(TContext& ctx, const TVector<TNodePtr>& expressions if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { if (SessionWindow) { ctx.Error(expr->GetPos()) << "Duplicate session window specification:"; - ctx.Error(SessionWindow->GetPos()) << "Previous session window is declared here"; + ctx.Error(SessionWindow->GetPos()) << "Previous session window is declared here"; return false; } SessionWindow = expr; @@ -3338,7 +3338,7 @@ TSourcePtr TryMakeSourceFromExpression(TContext& ctx, const TString& currService TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(node->GetPos(), *literal), view); TTableRef table(ctx.MakeName("table"), currService, currCluster, tableKey); table.Options = BuildInputOptions(node->GetPos(), GetContextHints(ctx)); - return BuildTableSource(node->GetPos(), table); + return BuildTableSource(node->GetPos(), table); } if (dynamic_cast<TLambdaNode*>(node.Get())) { @@ -3350,7 +3350,7 @@ TSourcePtr TryMakeSourceFromExpression(TContext& ctx, const TString& currService TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(wrappedNode, ctx), view); TTableRef table(ctx.MakeName("table"), currService, currCluster, tableKey); table.Options = BuildInputOptions(node->GetPos(), GetContextHints(ctx)); - return BuildTableSource(node->GetPos(), table); + return BuildTableSource(node->GetPos(), table); } void MakeTableFromExpression(TContext& ctx, TNodePtr node, TDeferredAtom& table) { diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index de82b45a23..e1a8a106be 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -1262,7 +1262,7 @@ namespace NSQLTranslationV1 { TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources); TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom = false); TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList = false); - TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label = TString()); + TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label = TString()); TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& service, const TDeferredAtom& cluster, const TString& label = TString()); TSourcePtr BuildRefColumnSource(TPosition pos, const TString& partExpression); TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings); @@ -1287,7 +1287,7 @@ namespace NSQLTranslationV1 { TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, - bool selectStream, + bool selectStream, const TWriteSettings& settings ); TSourcePtr BuildSelect(TPosition pos, TSourcePtr source, TNodePtr skipTake); @@ -1301,7 +1301,7 @@ namespace NSQLTranslationV1 { TVector<TNodePtr>&& keys, TVector<TNodePtr>&& args, TNodePtr udf, TNodePtr having, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy, bool listCall); TSourcePtr BuildProcess(TPosition pos, TSourcePtr source, TNodePtr with, bool withExtFunction, TVector<TNodePtr>&& terms, bool listCall, - bool prcessStream, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy); + bool prcessStream, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy); TNodePtr BuildSelectResult(TPosition pos, TSourcePtr source, bool writeResult, bool inSubquery, TScopedStatePtr scoped); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 725356b9f5..5e3f8918e3 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -1602,10 +1602,10 @@ public: Add(node); } - for (const auto& p : ctx.PackageVersions) { - Add(Y("set_package_version", BuildQuotedAtom(Pos, p.first), BuildQuotedAtom(Pos, ToString(p.second)))); - } - + for (const auto& p : ctx.PackageVersions) { + Add(Y("set_package_version", BuildQuotedAtom(Pos, p.first), BuildQuotedAtom(Pos, ToString(p.second)))); + } + Add(Y("import", "aggregate_module", BuildQuotedAtom(Pos, "/lib/yql/aggregate.yql"))); Add(Y("import", "window_module", BuildQuotedAtom(Pos, "/lib/yql/window.yql"))); for (const auto& module : ctx.Settings.ModuleMapping) { @@ -1661,15 +1661,15 @@ public: BuildQuotedAtom(Pos, "DiscoveryMode")))); } - if (ctx.DqEngineEnable) { - TString mode = "auto"; - if (ctx.PqReadByRtmrCluster && ctx.PqReadByRtmrCluster != "dq") { - mode = "disable"; - } else if (ctx.DqEngineForce) { - mode = "force"; - } + if (ctx.DqEngineEnable) { + TString mode = "auto"; + if (ctx.PqReadByRtmrCluster && ctx.PqReadByRtmrCluster != "dq") { + mode = "disable"; + } else if (ctx.DqEngineForce) { + mode = "force"; + } Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode)))); + BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode)))); } if (ctx.JsonQueryReturnsJsonDocument.Defined()) { @@ -1685,23 +1685,23 @@ public: Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "OrderedColumns")))); } - - if (ctx.PqReadByRtmrCluster) { - auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all")); - Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll, - BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); - - auto rtmrSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, "$all")); - Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll, - BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); - - if (ctx.PqReadByRtmrCluster != "dq") { - // set any dynamic settings for particular RTMR cluster for CommitAll! - auto rtmrSource = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)); - Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource, - BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "Dummy_"), BuildQuotedAtom(Pos, "1")))); - } - } + + if (ctx.PqReadByRtmrCluster) { + auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all")); + Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll, + BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); + + auto rtmrSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, "$all")); + Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll, + BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); + + if (ctx.PqReadByRtmrCluster != "dq") { + // set any dynamic settings for particular RTMR cluster for CommitAll! + auto rtmrSource = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)); + Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource, + BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "Dummy_"), BuildQuotedAtom(Pos, "1")))); + } + } } } @@ -1798,7 +1798,7 @@ public: cluster = "$all"; serviceName = Prefix; } else { - serviceName = *ctx.GetClusterProvider(Prefix, cluster); + serviceName = *ctx.GetClusterProvider(Prefix, cluster); } auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName)); diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp index 2841f05a5b..0a86cbd461 100644 --- a/ydb/library/yql/sql/v1/select.cpp +++ b/ydb/library/yql/sql/v1/select.cpp @@ -644,7 +644,7 @@ TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleInde class TTableSource: public IRealSource { public: - TTableSource(TPosition pos, const TTableRef& table, const TString& label) + TTableSource(TPosition pos, const TTableRef& table, const TString& label) : IRealSource(pos) , Table(table) , FakeSource(BuildFakeSource(pos)) @@ -740,11 +740,11 @@ public: } bool IsStream() const override { - return IsStreamingService(Table.Service); + return IsStreamingService(Table.Service); } TPtr DoClone() const final { - return new TTableSource(Pos, Table, GetLabel()); + return new TTableSource(Pos, Table, GetLabel()); } bool IsTableSource() const override { @@ -756,8 +756,8 @@ private: const TSourcePtr FakeSource; }; -TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label) { - return new TTableSource(pos, table, label); +TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label) { + return new TTableSource(pos, table, label); } class TInnerSource: public IProxySource { @@ -860,17 +860,17 @@ public: return NewSource ? NewSource->Build(ctx) : Node; } - bool IsStream() const override { - auto source = Node->GetSource(); - if (source) { - return source->IsStream(); - } - // NewSource will be built later in DoInit->TryMakeSourceFromExpression - // where Service will be used in all situations - // let's detect IsStream by Service value - return IsStreamingService(Service); - } - + bool IsStream() const override { + auto source = Node->GetSource(); + if (source) { + return source->IsStream(); + } + // NewSource will be built later in DoInit->TryMakeSourceFromExpression + // where Service will be used in all situations + // let's detect IsStream by Service value + return IsStreamingService(Service); + } + TPtr DoClone() const final { return new TInnerSource(Pos, SafeClone(Node), Service, Cluster, GetLabel()); } @@ -1404,7 +1404,7 @@ public: const TVector<TNodePtr>& terms, bool distinct, const TVector<TNodePtr>& without, - bool selectStream, + bool selectStream, const TWriteSettings& settings ) : IRealSource(pos) @@ -1420,7 +1420,7 @@ public: , Without(without) , Distinct(distinct) , HoppingWindowSpec(hoppingWindowSpec) - , SelectStream(selectStream) + , SelectStream(selectStream) , Settings(settings) { } @@ -1444,7 +1444,7 @@ public: if (!Source->Init(ctx, initSrc)) { return false; } - if (SelectStream && !Source->IsStream()) { + if (SelectStream && !Source->IsStream()) { ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources"; return false; } @@ -1477,10 +1477,10 @@ public: for (auto& expr: GroupByExpr) { if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { - if (Source->IsStream()) { - ctx.Error(Pos) << "SessionWindow is unsupported for streaming sources"; + if (Source->IsStream()) { + ctx.Error(Pos) << "SessionWindow is unsupported for streaming sources"; return false; - } + } sessionWindow->MarkValid(); } @@ -1703,7 +1703,7 @@ public: } bool IsStream() const override { - return Source->IsStream(); + return Source->IsStream(); } EOrderKind GetOrderKind() const override { @@ -1758,7 +1758,7 @@ public: return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr), CloneContainer(GroupBy), CompactGroupBy, AssumeSorted, CloneContainer(OrderBy), SafeClone(Having), newSpecs, SafeClone(HoppingWindowSpec), - CloneContainer(Terms), Distinct, Without, SelectStream, Settings); + CloneContainer(Terms), Distinct, Without, SelectStream, Settings); } private: @@ -2092,7 +2092,7 @@ private: const bool Distinct; bool OrderByInit = false; THoppingWindowSpecPtr HoppingWindowSpec; - const bool SelectStream; + const bool SelectStream; const TWriteSettings Settings; }; @@ -2103,9 +2103,9 @@ public: TSourcePtr source, TNodePtr with, bool withExtFunction, - TVector<TNodePtr>&& terms, + TVector<TNodePtr>&& terms, bool listCall, - bool processStream, + bool processStream, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy ) @@ -2113,9 +2113,9 @@ public: , Source(std::move(source)) , With(with) , WithExtFunction(withExtFunction) - , Terms(std::move(terms)) + , Terms(std::move(terms)) , ListCall(listCall) - , ProcessStream(processStream) + , ProcessStream(processStream) , Settings(settings) , AssumeOrderBy(assumeOrderBy) { @@ -2135,11 +2135,11 @@ public: return false; } - if (ProcessStream && !Source->IsStream()) { - ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources"; - return false; - } - + if (ProcessStream && !Source->IsStream()) { + ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources"; + return false; + } + auto src = Source.Get(); if (!With) { src->AllColumns(); @@ -2306,17 +2306,17 @@ public: return false; } - bool IsStream() const override { - return Source->IsStream(); - } - + bool IsStream() const override { + return Source->IsStream(); + } + TWriteSettings GetWriteSettings() const override { return Settings; } TNodePtr DoClone() const final { return new TProcessSource(Pos, Source->CloneSource(), SafeClone(With), WithExtFunction, - CloneContainer(Terms), ListCall, ProcessStream, Settings, CloneContainer(AssumeOrderBy)); + CloneContainer(Terms), ListCall, ProcessStream, Settings, CloneContainer(AssumeOrderBy)); } private: @@ -2341,7 +2341,7 @@ private: const bool WithExtFunction; TVector<TNodePtr> Terms; const bool ListCall; - const bool ProcessStream; + const bool ProcessStream; const TWriteSettings Settings; TVector<TSortSpecificationPtr> AssumeOrderBy; }; @@ -2353,7 +2353,7 @@ TSourcePtr BuildProcess( bool withExtFunction, TVector<TNodePtr>&& terms, bool listCall, - bool processStream, + bool processStream, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy ) { @@ -2465,12 +2465,12 @@ TSourcePtr DoBuildSelectCore( TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, - bool selectStream, + bool selectStream, const TWriteSettings& settings ) { if (groupBy.empty() || !groupBy.front()->ContentListPtr()) { return new TSelectCore(pos, std::move(source), groupByExpr, groupBy, compactGroupBy, assumeSorted, - orderBy, having, winSpecs, hoppingWindowSpec, terms, distinct, without, selectStream, settings); + orderBy, having, winSpecs, hoppingWindowSpec, terms, distinct, without, selectStream, settings); } if (groupBy.size() == 1) { /// actualy no big idea to use grouping function in this case (result allways 0) @@ -2478,7 +2478,7 @@ TSourcePtr DoBuildSelectCore( source = new TNestedProxySource(pos, *contentPtr, source); return DoBuildSelectCore(ctx, pos, originalSource, source, groupByExpr, *contentPtr, compactGroupBy, assumeSorted, orderBy, having, std::move(winSpecs), - hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); + hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); } /// \todo some smart merge logic, generalize common part of grouping (expr, flatten, etc)? TIntrusivePtr<TCompositeSelect> compositeSelect = new TCompositeSelect(pos, std::move(source), originalSource->CloneSource(), settings); @@ -2505,7 +2505,7 @@ TSourcePtr DoBuildSelectCore( totalGroups += contentPtr->size(); TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr), CloneContainer(*contentPtr), compactGroupBy, assumeSorted, orderBy, SafeClone(having), winSpecs, - hoppingWindowSpec, terms, distinct, without, selectStream, settings); + hoppingWindowSpec, terms, distinct, without, selectStream, settings); subselects.emplace_back(selectCore); } if (totalGroups > ctx.PragmaGroupByLimit) { @@ -2533,11 +2533,11 @@ TSourcePtr BuildSelectCore( TVector<TNodePtr>&& terms, bool distinct, TVector<TNodePtr>&& without, - bool selectStream, + bool selectStream, const TWriteSettings& settings) { return DoBuildSelectCore(ctx, pos, source, source, groupByExpr, groupBy, compactGroupBy, assumeSorted, orderBy, - having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); + having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); } class TUnionAll: public IRealSource { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index ab5a864788..6d71ed45c6 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -77,8 +77,8 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { return GetIdentifier(ctx, node.GetAlt_keyword1().GetRule_keyword_compat1()); case TRule_keyword::kAltKeyword2: return GetIdentifier(ctx, node.GetAlt_keyword2().GetRule_keyword_expr_uncompat1()); - case TRule_keyword::kAltKeyword3: - return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); + case TRule_keyword::kAltKeyword3: + return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); case TRule_keyword::kAltKeyword4: return GetIdentifier(ctx, node.GetAlt_keyword4().GetRule_keyword_select_uncompat1()); case TRule_keyword::kAltKeyword5: @@ -183,7 +183,7 @@ static std::pair<bool, TString> Id(const TRule_id_or_at& node, TTranslation& ctx return std::make_pair(hasAt, Id(node.GetRule_an_id_or_type2(), ctx) ); } -static TString Id(const TRule_id_table& node, TTranslation& ctx) { +static TString Id(const TRule_id_table& node, TTranslation& ctx) { //id_table: // identifier // | keyword_compat @@ -196,7 +196,7 @@ static TString Id(const TRule_id_table& node, TTranslation& ctx) { // | keyword_hint_uncompat // | keyword_schema_uncompat //; - switch (node.Alt_case()) { + switch (node.Alt_case()) { case TRule_id_table::kAltIdTable1: return Id(node.GetAlt_id_table1().GetRule_identifier1(), ctx); case TRule_id_table::kAltIdTable2: @@ -215,9 +215,9 @@ static TString Id(const TRule_id_table& node, TTranslation& ctx) { return GetKeyword(ctx, node.GetAlt_id_table8().GetRule_keyword_schema_uncompat1()); default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - + } +} + static TString Id(const TRule_an_id_table& node, TTranslation& ctx) { // an_id_table: id_table | STRING_VALUE; switch (node.Alt_case()) { @@ -782,18 +782,18 @@ static bool IsColumnsOnly(const TVector<TSortSpecificationPtr>& container) { return true; } -static bool PackageVersionFromString(const TString& s, ui32& version) { - if (s == "release") { - version = 0; - return true; - } - if (s == "draft") { - version = 1; - return true; - } - return TryFromString(s, version); -} - +static bool PackageVersionFromString(const TString& s, ui32& version) { + if (s == "release") { + version = 0; + return true; + } + if (s == "draft") { + version = 1; + return true; + } + return TryFromString(s, version); +} + class TSqlQuery; struct TSymbolNameWithPos { @@ -1412,20 +1412,20 @@ bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWild } if (value.GetLiteral()) { - TString clusterName = *value.GetLiteral(); + TString clusterName = *value.GetLiteral(); if (allowBinding && to_lower(clusterName) == "bindings") { isBinding = true; return true; } - TString normalizedClusterName; - auto foundProvider = Ctx.GetClusterProvider(clusterName, normalizedClusterName); + TString normalizedClusterName; + auto foundProvider = Ctx.GetClusterProvider(clusterName, normalizedClusterName); if (!foundProvider) { - Ctx.Error() << "Unknown cluster: " << clusterName; + Ctx.Error() << "Unknown cluster: " << clusterName; return false; } if (service && *foundProvider != service) { - Ctx.Error() << "Mismatch of cluster " << clusterName << " service, expected: " + Ctx.Error() << "Mismatch of cluster " << clusterName << " service, expected: " << *foundProvider << ", got: " << service; return false; } @@ -1434,7 +1434,7 @@ bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWild service = *foundProvider; } - value = TDeferredAtom(Ctx.Pos(), normalizedClusterName); + value = TDeferredAtom(Ctx.Pos(), normalizedClusterName); } else { if (!service) { Ctx.Error() << "Cluster service is not set"; @@ -1478,7 +1478,7 @@ bool TSqlTranslation::ApplyTableBinding(const TString& binding, TTableRef& tr, T const auto& bindSettings = (pit != settings.PrivateBindings.end()) ? pit->second : sit->second; - if (!IsIn({S3ProviderName, PqProviderName}, bindSettings.ClusterType)) { + if (!IsIn({S3ProviderName, PqProviderName}, bindSettings.ClusterType)) { Ctx.Error() << "Cluster type " << bindSettings.ClusterType << " is not supported for table bindings"; return false; } @@ -1522,13 +1522,13 @@ bool TSqlTranslation::ApplyTableBinding(const TString& binding, TTableRef& tr, T TVector<TTableArg> args; for (auto& arg : { path, format }) { args.emplace_back(); - args.back().Expr = BuildLiteralRawString(Ctx.Pos(), arg); + args.back().Expr = BuildLiteralRawString(Ctx.Pos(), arg); } for (auto& [key, value] : kvs) { YQL_ENSURE(!key.empty()); args.emplace_back(); - args.back().Expr = BuildLiteralRawString(Ctx.Pos(), value); + args.back().Expr = BuildLiteralRawString(Ctx.Pos(), value); args.back().Expr->SetLabel(key); } @@ -6000,7 +6000,7 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVect TPosition pos(Ctx.Pos()); Ctx.IncrementMonCounter("sql_select_clusters", table.Cluster.GetLiteral() ? *table.Cluster.GetLiteral() : "unknown"); - return BuildTableSource(pos, table); + return BuildTableSource(pos, table); } } case TRule_single_source::kAltSingleSource2: { @@ -6323,10 +6323,10 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)? // (WITH external_call_settings)? // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)? - + Token(node.GetToken1()); TPosition startPos(Ctx.Pos()); - + if (!selectPos) { selectPos = startPos; } @@ -6349,8 +6349,8 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS source = BuildMuxSource(pos, std::move(sources)); } - const bool processStream = node.HasBlock2(); - + const bool processStream = node.HasBlock2(); + if (!hasUsing) { return BuildProcess(startPos, std::move(source), nullptr, false, {}, false, processStream, settings, {}); } @@ -6365,7 +6365,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS } Ctx.IncrementMonCounter("sql_features", "ProcessWhere"); } else { - Ctx.IncrementMonCounter("sql_features", processStream ? "ProcessStream" : "Process"); + Ctx.IncrementMonCounter("sql_features", processStream ? "ProcessStream" : "Process"); } if (block5.HasBlock6()) { @@ -6583,7 +6583,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet return nullptr; } - const bool selectStream = node.HasBlock3(); + const bool selectStream = node.HasBlock3(); TVector<TNodePtr> without; if (node.HasBlock8()) { if (!ColumnList(without, node.GetBlock8().GetRule_without_column_list2())) { @@ -6617,7 +6617,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet bool compactGroupBy = false; if (node.HasBlock11()) { TGroupByClause clause(Ctx, Mode); - if (!clause.Build(node.GetBlock11().GetRule_group_by_clause1(), source->IsStream())) { + if (!clause.Build(node.GetBlock11().GetRule_group_by_clause1(), source->IsStream())) { return nullptr; } for (const auto& exprAlias: clause.Aliases()) { @@ -6643,7 +6643,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet TWinSpecs windowSpec; if (node.HasBlock13()) { - if (source->IsStream()) { + if (source->IsStream()) { Ctx.Error() << "WINDOW is not allowed in streaming queries"; return nullptr; } @@ -6661,7 +6661,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet Token(orderBlock.GetRule_order_by_clause2().GetToken1()); - if (source->IsStream()) { + if (source->IsStream()) { Ctx.Error() << "ORDER BY is not allowed in streaming queries"; return nullptr; } @@ -6716,7 +6716,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet return nullptr; } return BuildSelectCore(Ctx, startPos, std::move(source), groupByExpr, groupBy, compactGroupBy, assumeSorted, orderBy, having, - std::move(windowSpec), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); + std::move(windowSpec), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); } TString TSqlTranslation::FrameSettingsToString(EFrameSettings settings, bool isUnbounded) { @@ -9222,14 +9222,14 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.IncrementMonCounter("sql_pragma", "udf"); success = true; return BuildPragma(Ctx.Pos(), TString(ConfigProviderName), "ImportUdfs", values, false); - } else if (normalizedPragma == "packageversion") { - if (values.size() != 2 || pragmaValueDefault) { - Error() << "Expected package name and version"; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - ui32 version = 0; + } else if (normalizedPragma == "packageversion") { + if (values.size() != 2 || pragmaValueDefault) { + Error() << "Expected package name and version"; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + ui32 version = 0; TString versionString; TString packageName; if (!values[0].GetLiteral(packageName, Ctx) || !values[1].GetLiteral(versionString, Ctx)) { @@ -9238,15 +9238,15 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } if (!PackageVersionFromString(versionString, version)) { - Error() << "Unable to parse package version, possible values 0, 1, draft, release"; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - + Error() << "Unable to parse package version, possible values 0, 1, draft, release"; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + Ctx.SetPackageVersion(packageName, version); - Ctx.IncrementMonCounter("sql_pragma", "PackageVersion"); - success = true; - return BuildPragma(Ctx.Pos(), TString(ConfigProviderName), "SetPackageVersion", TVector<TDeferredAtom>{ values[0], TDeferredAtom(values[1].Build()->GetPos(), ToString(version)) }, false); + Ctx.IncrementMonCounter("sql_pragma", "PackageVersion"); + success = true; + return BuildPragma(Ctx.Pos(), TString(ConfigProviderName), "SetPackageVersion", TVector<TDeferredAtom>{ values[0], TDeferredAtom(values[1].Build()->GetPos(), ToString(version)) }, false); } else if (normalizedPragma == "file") { if (values.size() != 2U || pragmaValueDefault) { Error() << "Expected file alias and url as pragma values"; @@ -9540,28 +9540,28 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success // PositionalUnionAll implies OrderedColumns Ctx.OrderedColumns = true; Ctx.IncrementMonCounter("sql_pragma", "PositionalUnionAll"); - } else if (normalizedPragma == "pqreadby") { + } else if (normalizedPragma == "pqreadby") { if (values.size() != 1 || !values[0].GetLiteral()) { Error() << "Expected string literal as a single argument for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - // special guard to raise error on situation: - // use cluster1; - // pragma PqReadPqBy="cluster2"; - const TString* currentClusterLiteral = Ctx.Scoped->CurrCluster.GetLiteral(); - if (currentClusterLiteral && *values[0].GetLiteral() != "dq" && *currentClusterLiteral != *values[0].GetLiteral()) { - Error() << "Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: " << *values[0].GetLiteral() << " != " << *currentClusterLiteral; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - Ctx.PqReadByRtmrCluster = *values[0].GetLiteral(); - Ctx.IncrementMonCounter("sql_pragma", "PqReadBy"); + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + // special guard to raise error on situation: + // use cluster1; + // pragma PqReadPqBy="cluster2"; + const TString* currentClusterLiteral = Ctx.Scoped->CurrCluster.GetLiteral(); + if (currentClusterLiteral && *values[0].GetLiteral() != "dq" && *currentClusterLiteral != *values[0].GetLiteral()) { + Error() << "Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: " << *values[0].GetLiteral() << " != " << *currentClusterLiteral; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + Ctx.PqReadByRtmrCluster = *values[0].GetLiteral(); + Ctx.IncrementMonCounter("sql_pragma", "PqReadBy"); } else if (normalizedPragma == "bogousstaringroupbyoverjoin") { Ctx.BogousStarInGroupByOverJoin = true; - Ctx.IncrementMonCounter("sql_pragma", "BogousStarInGroupByOverJoin"); + Ctx.IncrementMonCounter("sql_pragma", "BogousStarInGroupByOverJoin"); } else if (normalizedPragma == "strictjoinkeytypes") { Ctx.Scoped->StrictJoinKeyTypes = true; Ctx.IncrementMonCounter("sql_pragma", "StrictJoinKeyTypes"); @@ -9650,7 +9650,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } } else if (std::find(Providers.cbegin(), Providers.cend(), lowerPrefix) == Providers.cend()) { - if (!Ctx.HasCluster(prefix)) { + if (!Ctx.HasCluster(prefix)) { Error() << "Unknown pragma prefix: " << prefix << ", please use cluster name or one of provider " << JoinRange(", ", Providers.cbegin(), Providers.cend()); Ctx.IncrementMonCounter("sql_errors", "UnknownPragma"); @@ -9658,7 +9658,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } } - if (normalizedPragma != "flags" && normalizedPragma != "packageversion") { + if (normalizedPragma != "flags" && normalizedPragma != "packageversion") { if (values.size() > 1) { Error() << "Expected at most one value in the pragma"; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); @@ -9692,7 +9692,7 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { return nullptr; } - TSourcePtr source = BuildTableSource(Ctx.Pos(), table); + TSourcePtr source = BuildTableSource(Ctx.Pos(), table); if (stmt.HasBlock4()) { switch (stmt.GetBlock4().Alt_case()) { @@ -9745,7 +9745,7 @@ TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) { case TRule_update_stmt_TBlock3::kAlt1: { const auto& alt = stmt.GetBlock3().GetAlt1(); TSourcePtr values = Build(alt.GetRule_set_clause_choice2()); - auto source = BuildTableSource(Ctx.Pos(), table); + auto source = BuildTableSource(Ctx.Pos(), table); if (alt.HasBlock3()) { TColumnRefScope scope(Ctx, EColumnRefState::Allow); @@ -10079,15 +10079,15 @@ TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) } TSqlQuery query(ctx, ctx.Settings.Mode, true); TNodePtr node(query.Build(static_cast<const TSQLv1ParserAST&>(protoAst))); - try { - if (node && node->Init(ctx, nullptr)) { - return node->Translate(ctx); - } - } catch (const NProtoAST::TTooManyErrors&) { - // do not add error issue, no room for it + try { + if (node && node->Init(ctx, nullptr)) { + return node->Translate(ctx); + } + } catch (const NProtoAST::TTooManyErrors&) { + // do not add error issue, no room for it } - - return nullptr; + + return nullptr; } void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message& protoAst, @@ -10100,7 +10100,7 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message ctx.IncrementMonCounter("sql_errors", "AstToYqlError"); } else { ctx.IncrementMonCounter("sql_errors", "AstToYqlSilentError"); - ctx.Error() << "Error occurred on parse SQL query, but no error is collected" << + ctx.Error() << "Error occurred on parse SQL query, but no error is collected" << ", please send this request over bug report into YQL interface or write on yql@ maillist"; } } diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 85b6e0dfcd..0aabcf5857 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -31,7 +31,7 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES const auto service = provider ? provider : TString(NYql::YtProviderName); const TString cluster = "plato"; settings.ClusterMapping[cluster] = service; - settings.ClusterMapping["hahn"] = NYql::YtProviderName; + settings.ClusterMapping["hahn"] = NYql::YtProviderName; settings.MaxErrors = maxErrors; settings.Mode = mode; settings.Arena = &arena; @@ -883,66 +883,66 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", 10, TString(NYql::RtmrProviderName)); UNIT_ASSERT(res.Root); - - res = SqlToYql( - "USE plato; INSERT INTO Output SELECT key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamNonRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: SELECT STREAM is unsupported for non-streaming sources\n"); - } - + + res = SqlToYql( + "USE plato; INSERT INTO Output SELECT key FROM Input;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectStreamNonRtmr) { + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", + 10); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: SELECT STREAM is unsupported for non-streaming sources\n"); + } + Y_UNIT_TEST(GroupByHopRtmr) { NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input + USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); )", 10, TString(NYql::RtmrProviderName)); UNIT_ASSERT(res.Root); } - Y_UNIT_TEST(GroupByHopRtmrSubquery) { - // 'use plato' intentially avoided - NYql::TAstParseResult res = SqlToYql(R"( - SELECT COUNT(*) AS value FROM (SELECT * FROM plato.Input) - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByHopRtmrSubqueryBinding) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - $q = SELECT * FROM Input; - INSERT INTO Output SELECT STREAM * FROM ( - SELECT COUNT(*) AS value FROM $q - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - ); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByNoHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input - GROUP BY key; - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:22: Error: Streaming group by query must have a hopping window specification.\n"); - } - + Y_UNIT_TEST(GroupByHopRtmrSubquery) { + // 'use plato' intentially avoided + NYql::TAstParseResult res = SqlToYql(R"( + SELECT COUNT(*) AS value FROM (SELECT * FROM plato.Input) + GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByHopRtmrSubqueryBinding) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + $q = SELECT * FROM Input; + INSERT INTO Output SELECT STREAM * FROM ( + SELECT COUNT(*) AS value FROM $q + GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") + ); + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByNoHopRtmr) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input + GROUP BY key; + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:22: Error: Streaming group by query must have a hopping window specification.\n"); + } + Y_UNIT_TEST(KikimrInserts) { NYql::TAstParseResult res = SqlToYql(R"( USE plato; @@ -1457,7 +1457,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { Y_UNIT_TEST(PqReadByAfterUse) { ExpectFailWithError("use plato; pragma PqReadBy='plato2';", - "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); + "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); @@ -1498,8 +1498,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { bindSettings.Settings["format"] = "format"; bindSettings.Settings["compression"] = "ccompression"; bindSettings.Settings["bar"] = "1"; - // schema is not validated in this test but should be valid YSON text - bindSettings.Settings["schema"] = R"__("[ + // schema is not validated in this test but should be valid YSON text + bindSettings.Settings["schema"] = R"__("[ "StructType"; [ [ @@ -1523,7 +1523,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { "String" ] ] - ]])__"; + ]])__"; settings.PrivateBindings["foo"] = bindSettings; @@ -1536,7 +1536,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "MrObject") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrObject '"path" '"format" '('('"bar" (String '"1")) '('"compression" (String '"ccompression")))))__")); + line.find(R"__((MrObject '"path" '"format" '('('"bar" (String '"1")) '('"compression" (String '"ccompression")))))__")); } else if (word == "userschema") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__('('('"userschema" (SqlTypeFromYson)__")); @@ -1667,17 +1667,17 @@ Y_UNIT_TEST_SUITE(ExternalFunction) { Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(StrayUTF8) { /// 'c' in plato is russian here - NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); + NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); UNIT_ASSERT(!res.Root); - - TString a1 = Err2Str(res); + + TString a1 = Err2Str(res); TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input... - + <main>:1:15: Error: Unexpected character : cannot match to any predicted input... - -)foo"); - - UNIT_ASSERT_NO_DIFF(a1, a2); + +)foo"); + + UNIT_ASSERT_NO_DIFF(a1, a2); } Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { @@ -2089,13 +2089,13 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(UpsertValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: UPSERT INTO ... VALUES requires specification of table columns\n"); } Y_UNIT_TEST(ReplaceValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:20: Error: REPLACE INTO ... VALUES requires specification of table columns\n"); } @@ -2356,7 +2356,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(InsertIntoWithTruncateKikimr) { - NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT INTO WITH TRUNCATE is not supported for kikimr tables\n"); } @@ -2368,7 +2368,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { } Y_UNIT_TEST(UpsertWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:39: Error: VALUES have 2 columns, UPSERT INTO expects: 3\n"); } @@ -2524,16 +2524,16 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: IF EXISTS in DROP TABLE is not supported.\n"); } - + Y_UNIT_TEST(TooManyErrors) { - const char* q = R"( - USE plato; + const char* q = R"( + USE plato; select A, B, C, D, E, F, G, H, I, J, K, L, M, N from (select b from `abc`); -)"; - - NYql::TAstParseResult res = SqlToYql(q, 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), +)"; + + NYql::TAstParseResult res = SqlToYql(q, 10); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), R"(<main>:3:16: Error: Column A is not in source column set. Did you mean b? <main>:3:19: Error: Column B is not in source column set. Did you mean b? <main>:3:22: Error: Column C is not in source column set. Did you mean b? @@ -2544,7 +2544,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { <main>:3:37: Error: Column H is not in source column set. Did you mean b? <main>:3:40: Error: Column I is not in source column set. Did you mean b? <main>: Error: Too many issues, code: 1 -)"); +)"); }; Y_UNIT_TEST(ShouldCloneBindingForNamedParameter) { @@ -2689,24 +2689,24 @@ select FormatType($f()); UNIT_ASSERT(res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); } - - Y_UNIT_TEST(YtCaseInsensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); - UNIT_ASSERT(res.Root); - - res = SqlToYql("use PlatO; select * from foo;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrCaseSensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Unknown cluster: PlatO\n"); - - res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); - } + + Y_UNIT_TEST(YtCaseInsensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); + UNIT_ASSERT(res.Root); + + res = SqlToYql("use PlatO; select * from foo;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(KikimrCaseSensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Unknown cluster: PlatO\n"); + + res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); + } Y_UNIT_TEST(DiscoveryModeForbidden) { NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY); @@ -3940,7 +3940,7 @@ Y_UNIT_TEST_SUITE(SessionWindowNegative) { ";", "<main>:7:5: Error: Duplicate session window specification:\n" - "<main>:5:5: Error: Previous session window is declared here\n"); + "<main>:5:5: Error: Previous session window is declared here\n"); ExpectFailWithError( "SELECT\n" @@ -3996,27 +3996,27 @@ Y_UNIT_TEST_SUITE(SessionWindowNegative) { ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); } - + Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); } - Y_UNIT_TEST(SessionWindowInRtmr) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:54: Error: Streaming group by query must have a hopping window specification.\n"); - - res = SqlToYql(R"( - SELECT key, SUM(value) AS value FROM plato.Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"), SessionWindow(ts, 10); - )", 10, TString(NYql::RtmrProviderName)); - - UNIT_ASSERT(!res.Root); + Y_UNIT_TEST(SessionWindowInRtmr) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:54: Error: Streaming group by query must have a hopping window specification.\n"); + + res = SqlToYql(R"( + SELECT key, SUM(value) AS value FROM plato.Input + GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"), SessionWindow(ts, 10); + )", 10, TString(NYql::RtmrProviderName)); + + UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); - } + } } Y_UNIT_TEST_SUITE(LibraSqlSugar) { diff --git a/ydb/library/yql/sql/ya.make b/ydb/library/yql/sql/ya.make index e10a314ebc..574c50dc7b 100644 --- a/ydb/library/yql/sql/ya.make +++ b/ydb/library/yql/sql/ya.make @@ -17,7 +17,7 @@ PEERDIR( ) SRCS( - cluster_mapping.cpp + cluster_mapping.cpp sql.cpp ) diff --git a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp index 763fea8024..4f39df0c1d 100644 --- a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp +++ b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp @@ -54,8 +54,8 @@ namespace { ythrow yexception() << "Error in FullSplitDate"; } - TimezoneId = timezoneId; - + TimezoneId = timezoneId; + Year = year; Month = month; Day = day; @@ -70,14 +70,14 @@ namespace { if (!IsUniversal(TimezoneId)) { ui32 datetime; if (!builder.MakeDatetime(Year, Month, Day, local ? 0 : Hour, local ? 0 : Minute, local ? 0 : Second, datetime, TimezoneId)) { - ythrow yexception() << "Error in MakeDatetime"; - } + ythrow yexception() << "Error in MakeDatetime"; + } return datetime / 86400u; } else { ui16 date; - if (!builder.MakeDate(Year, Month, Day, date)) { - ythrow yexception() << "Error in MakeDate"; - } + if (!builder.MakeDate(Year, Month, Day, date)) { + ythrow yexception() << "Error in MakeDate"; + } return date; } } @@ -87,9 +87,9 @@ namespace { if (!builder.FullSplitDatetime2(value, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, timezoneId)) { ythrow yexception() << "Error in FullSplitDatetime"; - } + } - TimezoneId = timezoneId; + TimezoneId = timezoneId; Year = year; Month = month; Day = day; @@ -104,10 +104,10 @@ namespace { } inline ui32 ToDatetime(const IDateBuilder& builder) const { - ui32 datetime = 0; - if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) { - ythrow yexception() << "Error in MakeDatetime"; - } + ui32 datetime = 0; + if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) { + ythrow yexception() << "Error in MakeDatetime"; + } return datetime; } @@ -130,7 +130,7 @@ namespace { ui32 year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek; if (!builder.FullSplitDatetime2(datetime, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, TimezoneId)) { ythrow yexception() << "Error in FullSplitDatetime."; - } + } DayOfYear = dayOfYear; WeekOfYear = weekOfYear; @@ -294,13 +294,13 @@ namespace { template <typename TUserDataType> class TSplit : public TBoxedValue { - const TSourcePosition Pos_; - + const TSourcePosition Pos_; + public: - explicit TSplit(TSourcePosition pos) - : Pos_(pos) - {} - + explicit TSplit(TSourcePosition pos) + : Pos_(pos) + {} + TUnboxedValue Run( const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override; @@ -315,7 +315,7 @@ namespace { builder.Returns(builder.Resource(TMResourceName)); if (!typesOnly) { - builder.Implementation(new TSplit<TUserDataType>(builder.GetSourcePosition())); + builder.Implementation(new TSplit<TUserDataType>(builder.GetSourcePosition())); } } }; @@ -325,17 +325,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromDate(builder, args[0].Get<ui16>()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromDate(builder, args[0].Get<ui16>()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } template <> @@ -343,17 +343,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromDatetime(builder, args[0].Get<ui32>()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromDatetime(builder, args[0].Get<ui32>()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } template <> @@ -361,17 +361,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromTimestamp(builder, args[0].Get<ui64>()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromTimestamp(builder, args[0].Get<ui64>()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } template <> @@ -379,17 +379,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromDate(builder, args[0].Get<ui16>(), args[0].GetTimezoneId()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromDate(builder, args[0].Get<ui16>(), args[0].GetTimezoneId()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } template <> @@ -397,17 +397,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromDatetime(builder, args[0].Get<ui32>(), args[0].GetTimezoneId()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromDatetime(builder, args[0].Get<ui32>(), args[0].GetTimezoneId()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } template <> @@ -415,17 +415,17 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - auto& builder = valueBuilder->GetDateBuilder(); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.FromTimestamp(builder, args[0].Get<ui64>(), args[0].GetTimezoneId()); - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + + auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.FromTimestamp(builder, args[0].Get<ui64>(), args[0].GetTimezoneId()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } } // Make* @@ -554,85 +554,85 @@ namespace { // Update class TUpdate : public TBoxedValue { - const TSourcePosition Pos_; - public: - explicit TUpdate(TSourcePosition pos) - : Pos_(pos) - {} - + const TSourcePosition Pos_; + public: + explicit TUpdate(TSourcePosition pos) + : Pos_(pos) + {} + TUnboxedValue Run( const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - auto result = args[0]; - - if (args[1]) { - auto year = args[1].Get<ui16>(); - if (!ValidateYear(year)) { - return TUnboxedValuePod(); - } - SetYear(result, year); + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + auto result = args[0]; + + if (args[1]) { + auto year = args[1].Get<ui16>(); + if (!ValidateYear(year)) { + return TUnboxedValuePod(); + } + SetYear(result, year); } - if (args[2]) { - auto month = args[2].Get<ui8>(); - if (!ValidateMonth(month)) { - return TUnboxedValuePod(); - } - SetMonth(result, month); + if (args[2]) { + auto month = args[2].Get<ui8>(); + if (!ValidateMonth(month)) { + return TUnboxedValuePod(); + } + SetMonth(result, month); } - if (args[3]) { - auto day = args[3].Get<ui8>(); - if (!ValidateDay(day)) { - return TUnboxedValuePod(); - } - SetDay(result, day); + if (args[3]) { + auto day = args[3].Get<ui8>(); + if (!ValidateDay(day)) { + return TUnboxedValuePod(); + } + SetDay(result, day); } - if (args[4]) { - auto hour = args[4].Get<ui8>(); - if (!ValidateHour(hour)) { - return TUnboxedValuePod(); - } - SetHour(result, hour); + if (args[4]) { + auto hour = args[4].Get<ui8>(); + if (!ValidateHour(hour)) { + return TUnboxedValuePod(); + } + SetHour(result, hour); } - if (args[5]) { - auto minute = args[5].Get<ui8>(); - if (!ValidateMinute(minute)) { - return TUnboxedValuePod(); - } - SetMinute(result, minute); + if (args[5]) { + auto minute = args[5].Get<ui8>(); + if (!ValidateMinute(minute)) { + return TUnboxedValuePod(); + } + SetMinute(result, minute); } - if (args[6]) { - auto second = args[6].Get<ui8>(); - if (!ValidateSecond(second)) { - return TUnboxedValuePod(); - } - SetSecond(result, second); + if (args[6]) { + auto second = args[6].Get<ui8>(); + if (!ValidateSecond(second)) { + return TUnboxedValuePod(); + } + SetSecond(result, second); } - if (args[7]) { - auto microsecond = args[7].Get<ui32>(); - if (!ValidateMicrosecond(microsecond)) { - return TUnboxedValuePod(); - } - SetMicrosecond(result, microsecond); + if (args[7]) { + auto microsecond = args[7].Get<ui32>(); + if (!ValidateMicrosecond(microsecond)) { + return TUnboxedValuePod(); + } + SetMicrosecond(result, microsecond); } - if (args[8]) { - auto timezoneId = args[8].Get<ui16>(); - if (!ValidateTimezoneId(timezoneId)) { - return TUnboxedValuePod(); - } - SetTimezoneId(result, timezoneId); - } - - auto& builder = valueBuilder->GetDateBuilder(); - auto& storage = TTMStorage::Reference(result); - if (!storage.Validate(builder)) { + if (args[8]) { + auto timezoneId = args[8].Get<ui16>(); + if (!ValidateTimezoneId(timezoneId)) { + return TUnboxedValuePod(); + } + SetTimezoneId(result, timezoneId); + } + + auto& builder = valueBuilder->GetDateBuilder(); + auto& storage = TTMStorage::Reference(result); + if (!storage.Validate(builder)) { return TUnboxedValuePod(); } - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } } @@ -667,7 +667,7 @@ namespace { builder.Returns(optionalResourceType); if (!typesOnly) { - builder.Implementation(new TUpdate(builder.GetSourcePosition())); + builder.Implementation(new TUpdate(builder.GetSourcePosition())); } return true; @@ -1028,11 +1028,11 @@ namespace { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try { - return TUnboxedValuePod(new TFormat(args[0], Pos_)); + return TUnboxedValuePod(new TFormat(args[0], Pos_)); } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - const TSourcePosition Pos_; + const TSourcePosition Pos_; }; static const TStringRef& Name() { @@ -1063,7 +1063,7 @@ namespace { } private: - const TSourcePosition Pos_; + const TSourcePosition Pos_; const TUnboxedValue Format_; std::vector<std::function<size_t(char*, const TUnboxedValuePod&, const IDateBuilder&)>> Printers_; @@ -1083,36 +1083,36 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - const auto value = args[0]; + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); + const auto value = args[0]; - auto& builder = valueBuilder->GetDateBuilder(); + auto& builder = valueBuilder->GetDateBuilder(); - auto result = valueBuilder->NewStringNotFilled(ReservedSize_); - auto pos = result.AsStringRef().Data(); - ui32 size = 0U; + auto result = valueBuilder->NewStringNotFilled(ReservedSize_); + auto pos = result.AsStringRef().Data(); + ui32 size = 0U; - for (const auto& printer : Printers_) { - if (const auto plus = printer(pos, value, builder)) { - size += plus; - pos += plus; - } + for (const auto& printer : Printers_) { + if (const auto plus = printer(pos, value, builder)) { + size += plus; + pos += plus; + } } - if (size < ReservedSize_) { - result = valueBuilder->SubString(result.Release(), 0U, size); - } - - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + if (size < ReservedSize_) { + result = valueBuilder->SubString(result.Release(), 0U, size); + } + + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } } - TFormat(const TUnboxedValuePod& runConfig, TSourcePosition pos) - : Pos_(pos) - , Format_(runConfig) + TFormat(const TUnboxedValuePod& runConfig, TSourcePosition pos) + : Pos_(pos) + , Format_(runConfig) { const std::string_view formatView(Format_.AsStringRef()); auto dataStart = formatView.begin(); @@ -1334,12 +1334,12 @@ namespace { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try { - return TUnboxedValuePod(new TParse(args[0], Pos_)); + return TUnboxedValuePod(new TParse(args[0], Pos_)); } catch (const std::exception& e) { UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } - const TSourcePosition Pos_; + const TSourcePosition Pos_; }; static const TStringRef& Name() { @@ -1374,7 +1374,7 @@ namespace { } private: - const TSourcePosition Pos_; + const TSourcePosition Pos_; const TUnboxedValue Format_; std::vector<std::function<bool(std::string_view::const_iterator& it, size_t, TUnboxedValuePod&, const IDateBuilder&)>> Scanners_; @@ -1395,36 +1395,36 @@ namespace { const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override { - try { - EMPTY_RESULT_ON_EMPTY_ARG(0); - - const std::string_view buffer = args[0].AsStringRef(); + try { + EMPTY_RESULT_ON_EMPTY_ARG(0); - TUnboxedValuePod result(0); - auto& storage = TTMStorage::Reference(result); - storage.MakeDefault(); + const std::string_view buffer = args[0].AsStringRef(); - auto& builder = valueBuilder->GetDateBuilder(); + TUnboxedValuePod result(0); + auto& storage = TTMStorage::Reference(result); + storage.MakeDefault(); - auto it = buffer.begin(); - for (const auto& scanner : Scanners_) { - if (!scanner(it, std::distance(it, buffer.end()), result, builder)) { - return TUnboxedValuePod(); - } - } + auto& builder = valueBuilder->GetDateBuilder(); - if (buffer.end() != it || !storage.Validate(builder)) { + auto it = buffer.begin(); + for (const auto& scanner : Scanners_) { + if (!scanner(it, std::distance(it, buffer.end()), result, builder)) { + return TUnboxedValuePod(); + } + } + + if (buffer.end() != it || !storage.Validate(builder)) { return TUnboxedValuePod(); } - return result; - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + return result; + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); } } - TParse(const TUnboxedValuePod& runConfig, TSourcePosition pos) - : Pos_(pos) - , Format_(runConfig) + TParse(const TUnboxedValuePod& runConfig, TSourcePosition pos) + : Pos_(pos) + , Format_(runConfig) { const std::string_view formatView(Format_.AsStringRef()); auto dataStart = formatView.begin(); diff --git a/ydb/library/yql/udfs/common/json/json_udf.cpp b/ydb/library/yql/udfs/common/json/json_udf.cpp index e994e589bd..cf9addee9a 100644 --- a/ydb/library/yql/udfs/common/json/json_udf.cpp +++ b/ydb/library/yql/udfs/common/json/json_udf.cpp @@ -8,9 +8,9 @@ using namespace NUdf; namespace { class TGetField: public TBoxedValue { public: - typedef bool TTypeAwareMarker; - - public: + typedef bool TTypeAwareMarker; + + public: static TStringRef Name() { return TStringRef::Of("GetField"); } @@ -58,19 +58,19 @@ namespace { auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) { builder.SetError("Missing or invalid user type."); - return true; + return true; } auto argsTypeTuple = userTypeInspector.GetElementType(0); auto argsTypeInspector = TTupleTypeInspector(*typeHelper, argsTypeTuple); if (!argsTypeInspector) { builder.SetError("Invalid user type - expected tuple."); - return true; + return true; } if (argsTypeInspector.GetElementsCount() != 2) { builder.SetError("Invalid user type - expected two arguments."); - return true; + return true; } auto inputType = argsTypeInspector.GetElementType(0); diff --git a/ydb/library/yql/udfs/common/math/lib/round.h b/ydb/library/yql/udfs/common/math/lib/round.h index 2b33a6e8a8..4adda50415 100644 --- a/ydb/library/yql/udfs/common/math/lib/round.h +++ b/ydb/library/yql/udfs/common/math/lib/round.h @@ -1,43 +1,43 @@ #pragma once -#include <util/system/types.h> +#include <util/system/types.h> #include <cmath> -#include <optional> - -namespace NMathUdf { +#include <optional> +namespace NMathUdf { + template <class T> inline T RoundToDecimal(T v, int decShift) { T div = std::pow(T(10), decShift); return std::floor(v / div + T(0.5)) * div; } - -inline std::optional<i64> Mod(i64 value, i64 m) { - if (!m) { - return {}; - } - - const i64 result = value % m; - if ((result < 0 && m > 0) || (result > 0 && m < 0)) { - return result + m; - } - return result; -} - -inline std::optional<i64> Rem(i64 value, i64 m) { - if (!m) { - return {}; - } - - const i64 result = value % m; - if (result < 0 && value > 0) { - return result + m; - } - - if (result > 0 && value < 0) { - return result - m; - } - return result; -} - -} + +inline std::optional<i64> Mod(i64 value, i64 m) { + if (!m) { + return {}; + } + + const i64 result = value % m; + if ((result < 0 && m > 0) || (result > 0 && m < 0)) { + return result + m; + } + return result; +} + +inline std::optional<i64> Rem(i64 value, i64 m) { + if (!m) { + return {}; + } + + const i64 result = value % m; + if (result < 0 && value > 0) { + return result + m; + } + + if (result > 0 && value < 0) { + return result - m; + } + return result; +} + +} diff --git a/ydb/library/yql/udfs/common/math/lib/round_ut.cpp b/ydb/library/yql/udfs/common/math/lib/round_ut.cpp index 81ded7d28c..8325b294cd 100644 --- a/ydb/library/yql/udfs/common/math/lib/round_ut.cpp +++ b/ydb/library/yql/udfs/common/math/lib/round_ut.cpp @@ -4,58 +4,58 @@ #include <util/string/hex.h> -using namespace NMathUdf; - -Y_UNIT_TEST_SUITE(TRound) { - Y_UNIT_TEST(Basic) { +using namespace NMathUdf; + +Y_UNIT_TEST_SUITE(TRound) { + Y_UNIT_TEST(Basic) { double value = 1930.0 / 3361.0; - double result = RoundToDecimal<long double>(value, -3); - double answer = 0.574; - UNIT_ASSERT_VALUES_EQUAL( - HexEncode(&result, sizeof(double)), - HexEncode(&answer, sizeof(double))); - } - - Y_UNIT_TEST(Mod) { - UNIT_ASSERT_VALUES_EQUAL(*Mod(-1, 7), 6); - UNIT_ASSERT_VALUES_EQUAL(*Mod(1, 7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Mod(0, 7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Mod(-1, -7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Mod(1, -7), -6); - UNIT_ASSERT_VALUES_EQUAL(*Mod(0, -7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Mod(-15, 7), 6); - UNIT_ASSERT_VALUES_EQUAL(*Mod(15, 7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Mod(14, 7), 0); - UNIT_ASSERT_VALUES_EQUAL(*Mod(-14, 7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Mod(-15, -7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Mod(15, -7), -6); - UNIT_ASSERT_VALUES_EQUAL(*Mod(14, -7), 0); - UNIT_ASSERT_VALUES_EQUAL(*Mod(-14, -7), 0); - - UNIT_ASSERT(!Mod(-14, 0)); - } - - Y_UNIT_TEST(Rem) { - UNIT_ASSERT_VALUES_EQUAL(*Rem(-1, 7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(1, 7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(0, 7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Rem(-1, -7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(1, -7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(0, -7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Rem(-15, 7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(15, 7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(14, 7), 0); - UNIT_ASSERT_VALUES_EQUAL(*Rem(-14, 7), 0); - - UNIT_ASSERT_VALUES_EQUAL(*Rem(-15, -7), -1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(15, -7), 1); - UNIT_ASSERT_VALUES_EQUAL(*Rem(14, -7), 0); - UNIT_ASSERT_VALUES_EQUAL(*Rem(-14, -7), 0); - UNIT_ASSERT(!Rem(-14, 0)); - } + double result = RoundToDecimal<long double>(value, -3); + double answer = 0.574; + UNIT_ASSERT_VALUES_EQUAL( + HexEncode(&result, sizeof(double)), + HexEncode(&answer, sizeof(double))); + } + + Y_UNIT_TEST(Mod) { + UNIT_ASSERT_VALUES_EQUAL(*Mod(-1, 7), 6); + UNIT_ASSERT_VALUES_EQUAL(*Mod(1, 7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Mod(0, 7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Mod(-1, -7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Mod(1, -7), -6); + UNIT_ASSERT_VALUES_EQUAL(*Mod(0, -7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Mod(-15, 7), 6); + UNIT_ASSERT_VALUES_EQUAL(*Mod(15, 7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Mod(14, 7), 0); + UNIT_ASSERT_VALUES_EQUAL(*Mod(-14, 7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Mod(-15, -7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Mod(15, -7), -6); + UNIT_ASSERT_VALUES_EQUAL(*Mod(14, -7), 0); + UNIT_ASSERT_VALUES_EQUAL(*Mod(-14, -7), 0); + + UNIT_ASSERT(!Mod(-14, 0)); + } + + Y_UNIT_TEST(Rem) { + UNIT_ASSERT_VALUES_EQUAL(*Rem(-1, 7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(1, 7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(0, 7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Rem(-1, -7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(1, -7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(0, -7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Rem(-15, 7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(15, 7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(14, 7), 0); + UNIT_ASSERT_VALUES_EQUAL(*Rem(-14, 7), 0); + + UNIT_ASSERT_VALUES_EQUAL(*Rem(-15, -7), -1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(15, -7), 1); + UNIT_ASSERT_VALUES_EQUAL(*Rem(14, -7), 0); + UNIT_ASSERT_VALUES_EQUAL(*Rem(-14, -7), 0); + UNIT_ASSERT(!Rem(-14, 0)); + } } diff --git a/ydb/library/yql/udfs/common/math/math_ir.h b/ydb/library/yql/udfs/common/math/math_ir.h index 2386be96b9..6a3f6a1970 100644 --- a/ydb/library/yql/udfs/common/math/math_ir.h +++ b/ydb/library/yql/udfs/common/math/math_ir.h @@ -109,21 +109,21 @@ void FuzzyEqualsIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const extern "C" UDF_ALWAYS_INLINE void RoundIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { - const double val = NMathUdf::RoundToDecimal<long double>(args[0].Get<double>(), args[1].GetOrDefault<int>(0)); + const double val = NMathUdf::RoundToDecimal<long double>(args[0].Get<double>(), args[1].GetOrDefault<int>(0)); *result = TUnboxedValuePod(val); } -extern "C" UDF_ALWAYS_INLINE -void ModIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { - const auto val = NMathUdf::Mod(args[0].Get<i64>(), args[1].Get<i64>()); - *result = val ? TUnboxedValuePod(*val) : TUnboxedValuePod(); -} - -extern "C" UDF_ALWAYS_INLINE -void RemIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { - const auto val = NMathUdf::Rem(args[0].Get<i64>(), args[1].Get<i64>()); - *result = val ? TUnboxedValuePod(*val) : TUnboxedValuePod(); -} - +extern "C" UDF_ALWAYS_INLINE +void ModIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { + const auto val = NMathUdf::Mod(args[0].Get<i64>(), args[1].Get<i64>()); + *result = val ? TUnboxedValuePod(*val) : TUnboxedValuePod(); +} + +extern "C" UDF_ALWAYS_INLINE +void RemIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { + const auto val = NMathUdf::Rem(args[0].Get<i64>(), args[1].Get<i64>()); + *result = val ? TUnboxedValuePod(*val) : TUnboxedValuePod(); +} + } // NUdf } // NYql diff --git a/ydb/library/yql/udfs/common/math/math_udf.cpp b/ydb/library/yql/udfs/common/math/math_udf.cpp index 04aaa29fea..13375e42b8 100644 --- a/ydb/library/yql/udfs/common/math/math_udf.cpp +++ b/ydb/library/yql/udfs/common/math/math_udf.cpp @@ -43,8 +43,8 @@ XX(IsNaN, bool(TAutoMap<double>), ;) \ XX(Sigmoid, double(TAutoMap<double>), ;) \ XX(FuzzyEquals, bool(TAutoMap<double>, TAutoMap<double>, TEpsilon), builder.OptionalArgs(1)) \ - XX(Mod, TOptional<i64>(TAutoMap<i64>, i64), ;) \ - XX(Rem, TOptional<i64>(TAutoMap<i64>, i64), ;) \ + XX(Mod, TOptional<i64>(TAutoMap<i64>, i64), ;) \ + XX(Rem, TOptional<i64>(TAutoMap<i64>, i64), ;) \ XXL(Round, double(TAutoMap<double>, TPrecision), builder.OptionalArgs(1)) #define MATH_UDF_IMPL(name, signature, options) \ diff --git a/ydb/library/yql/udfs/common/math/ya.make b/ydb/library/yql/udfs/common/math/ya.make index 7aaedf9619..0b0107a977 100644 --- a/ydb/library/yql/udfs/common/math/ya.make +++ b/ydb/library/yql/udfs/common/math/ya.make @@ -56,8 +56,8 @@ LLVM_BC( SigmoidIR FuzzyEqualsIR RoundIR - ModIR - RemIR + ModIR + RemIR ) PEERDIR( diff --git a/ydb/library/yql/udfs/common/re2/re2_udf.cpp b/ydb/library/yql/udfs/common/re2/re2_udf.cpp index b478675fb6..e800fb3180 100644 --- a/ydb/library/yql/udfs/common/re2/re2_udf.cpp +++ b/ydb/library/yql/udfs/common/re2/re2_udf.cpp @@ -412,10 +412,10 @@ namespace { void GetAllFunctions(IFunctionsSink& sink) const final { sink.Add(TRe2Udf::Name(TRe2Udf::EMode::MATCH)); sink.Add(TRe2Udf::Name(TRe2Udf::EMode::GREP)); - sink.Add(TRe2Udf::Name(TRe2Udf::EMode::CAPTURE))->SetTypeAwareness(); + sink.Add(TRe2Udf::Name(TRe2Udf::EMode::CAPTURE))->SetTypeAwareness(); sink.Add(TRe2Udf::Name(TRe2Udf::EMode::REPLACE)); sink.Add(TRe2Udf::Name(TRe2Udf::EMode::COUNT)); - sink.Add(TRe2Udf::Name(TRe2Udf::EMode::FIND_AND_CONSUME)); + sink.Add(TRe2Udf::Name(TRe2Udf::EMode::FIND_AND_CONSUME)); sink.Add(TEscape::Name()); sink.Add(TPatternFromLike::Name()); sink.Add(TOptions::Name()); diff --git a/ydb/library/yql/udfs/common/string/string_udf.cpp b/ydb/library/yql/udfs/common/string/string_udf.cpp index c468215090..bd5f990cc4 100644 --- a/ydb/library/yql/udfs/common/string/string_udf.cpp +++ b/ydb/library/yql/udfs/common/string/string_udf.cpp @@ -482,30 +482,30 @@ namespace { return result; } - SIMPLE_UDF(TFromByteList, char*(TListType<ui8>)) { - auto input = args[0]; - - if (auto elems = input.GetElements()) { - const auto elemCount = input.GetListLength(); - TUnboxedValue result = valueBuilder->NewStringNotFilled(input.GetListLength()); - auto bufferPtr = result.AsStringRef().Data(); - for (ui64 i = 0; i != elemCount; ++i) { - *(bufferPtr++) = elems[i].Get<ui8>(); - } - return result; - } - - std::vector<char, NKikimr::NUdf::TStdAllocatorForUdf<char>> buffer; - buffer.reserve(TUnboxedValuePod::InternalBufferSize); - - const auto& iter = input.GetListIterator(); - for (NUdf::TUnboxedValue item; iter.Next(item); ) { - buffer.push_back(item.Get<ui8>()); - } - - return valueBuilder->NewString(TStringRef(buffer.data(), buffer.size())); - } - + SIMPLE_UDF(TFromByteList, char*(TListType<ui8>)) { + auto input = args[0]; + + if (auto elems = input.GetElements()) { + const auto elemCount = input.GetListLength(); + TUnboxedValue result = valueBuilder->NewStringNotFilled(input.GetListLength()); + auto bufferPtr = result.AsStringRef().Data(); + for (ui64 i = 0; i != elemCount; ++i) { + *(bufferPtr++) = elems[i].Get<ui8>(); + } + return result; + } + + std::vector<char, NKikimr::NUdf::TStdAllocatorForUdf<char>> buffer; + buffer.reserve(TUnboxedValuePod::InternalBufferSize); + + const auto& iter = input.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item); ) { + buffer.push_back(item.Get<ui8>()); + } + + return valueBuilder->NewString(TStringRef(buffer.data(), buffer.size())); + } + #define STRING_REGISTER_UDF(udfName, ...) T##udfName, STRING_UDF_MAP(STRING_UDF) @@ -552,8 +552,8 @@ namespace { THumanReadableQuantity, THumanReadableBytes, TPrec, - TToByteList, - TFromByteList) + TToByteList, + TFromByteList) } REGISTER_MODULES(TStringModule) diff --git a/ydb/library/yql/udfs/logs/dsv/dsv_udf.cpp b/ydb/library/yql/udfs/logs/dsv/dsv_udf.cpp index b74bf80b18..16fe2256bd 100644 --- a/ydb/library/yql/udfs/logs/dsv/dsv_udf.cpp +++ b/ydb/library/yql/udfs/logs/dsv/dsv_udf.cpp @@ -206,7 +206,7 @@ public: void GetAllFunctions(IFunctionsSink& sink) const final { sink.Add(TStringRef::Of("ReadRecord")); sink.Add(TStringRef::Of("Parse")); - sink.Add(TStringRef::Of("Serialize"))->SetTypeAwareness(); + sink.Add(TStringRef::Of("Serialize"))->SetTypeAwareness(); } void BuildFunctionTypeInfo( diff --git a/ydb/library/yql/utils/backtrace/backtrace.cpp b/ydb/library/yql/utils/backtrace/backtrace.cpp index f9487f6671..dee2a6862c 100644 --- a/ydb/library/yql/utils/backtrace/backtrace.cpp +++ b/ydb/library/yql/utils/backtrace/backtrace.cpp @@ -96,12 +96,12 @@ void KikimrBacktraceFormatImpl(IOutputStream* out) { void* array[300]; const size_t s = BackTrace(array, Y_ARRAY_SIZE(array)); KikimrBacktraceFormatImpl(out, array, s); -} - +} + void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t stackSize) { - using namespace llvm; - using namespace symbolize; - + using namespace llvm; + using namespace symbolize; + TRawOStreamProxy outStream(*out); THashMap<TString, TDllInfo> dlls; #ifdef _linux_ @@ -111,8 +111,8 @@ void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t st DIPrinter printer(outStream, true, true, false); LLVMSymbolizer::Options opts; LLVMSymbolizer symbolyzer(opts); - for (const auto i : xrange(stackSize)) { - ui64 address = (ui64)stack[i] - 1; // last byte of the call instruction + for (const auto i : xrange(stackSize)) { + ui64 address = (ui64)stack[i] - 1; // last byte of the call instruction ui64 offset = 0; TString modulePath = binaryPath; #ifdef _linux_ diff --git a/ydb/library/yql/utils/bind_in_range.cpp b/ydb/library/yql/utils/bind_in_range.cpp index 75942f9038..9bcbfc329a 100644 --- a/ydb/library/yql/utils/bind_in_range.cpp +++ b/ydb/library/yql/utils/bind_in_range.cpp @@ -1,27 +1,27 @@ -#include "bind_in_range.h" - +#include "bind_in_range.h" + #include <ydb/library/yql/utils/log/log.h> -#include <util/datetime/base.h> - -namespace NYql { - -TVector<NBus::TBindResult> BindInRange(TRangeWalker<int>& portWalker) { - const int cyclesLimit = 3; - const int rangeSize = portWalker.GetRangeSize(); - const auto cycleDelay = TDuration::Seconds(3); - - for (int cycle = 0; cycle < cyclesLimit; ++cycle) { - for (int i = 0; i < rangeSize; ++i) { - try { +#include <util/datetime/base.h> + +namespace NYql { + +TVector<NBus::TBindResult> BindInRange(TRangeWalker<int>& portWalker) { + const int cyclesLimit = 3; + const int rangeSize = portWalker.GetRangeSize(); + const auto cycleDelay = TDuration::Seconds(3); + + for (int cycle = 0; cycle < cyclesLimit; ++cycle) { + for (int i = 0; i < rangeSize; ++i) { + try { return NBus::BindOnPort(portWalker.MoveToNext(), false).second; - } catch (const TSystemError&) { - YQL_LOG(DEBUG) << CurrentExceptionMessage(); - } - } - - Sleep(cycleDelay); - } - - ythrow yexception() << "Unable to bind within port range [" << portWalker.GetStart() << ", " << portWalker.GetFinish() << "]"; -} -} + } catch (const TSystemError&) { + YQL_LOG(DEBUG) << CurrentExceptionMessage(); + } + } + + Sleep(cycleDelay); + } + + ythrow yexception() << "Unable to bind within port range [" << portWalker.GetStart() << ", " << portWalker.GetFinish() << "]"; +} +} diff --git a/ydb/library/yql/utils/bind_in_range.h b/ydb/library/yql/utils/bind_in_range.h index 28c6069f81..67bf1ff25c 100644 --- a/ydb/library/yql/utils/bind_in_range.h +++ b/ydb/library/yql/utils/bind_in_range.h @@ -1,8 +1,8 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/utils/range_walker.h> #include <library/cpp/messagebus/network.h> - -namespace NYql { -TVector<NBus::TBindResult> BindInRange(TRangeWalker<int>& portWalker); -} + +namespace NYql { +TVector<NBus::TBindResult> BindInRange(TRangeWalker<int>& portWalker); +} diff --git a/ydb/library/yql/utils/fetch/fetch.cpp b/ydb/library/yql/utils/fetch/fetch.cpp index 100c88c385..5e822fafad 100644 --- a/ydb/library/yql/utils/fetch/fetch.cpp +++ b/ydb/library/yql/utils/fetch/fetch.cpp @@ -67,9 +67,9 @@ public: Socket.Reset(new TSocket(TNetworkAddress(host, port), timeout)); SocketInput.Reset(new TSocketInput(*Socket)); SocketOutput.Reset(new TSocketOutput(*Socket)); - - Socket->SetSocketTimeout(timeout.Seconds(), timeout.MilliSeconds() % 1000); - + + Socket->SetSocketTimeout(timeout.Seconds(), timeout.MilliSeconds() % 1000); + if (https) { Ssl.Reset(new TOpenSslClientIO(SocketInput.Get(), SocketOutput.Get())); } @@ -156,18 +156,18 @@ TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders if (responseCode >= 200 && responseCode < 300) { return fr; - } - - if (responseCode == HTTP_NOT_MODIFIED) { - return fr; - } - - if (IsRedirectCode(responseCode)) { + } + + if (responseCode == HTTP_NOT_MODIFIED) { + return fr; + } + + if (IsRedirectCode(responseCode)) { currentUrl = fr->GetRedirectURL(currentUrl); YQL_LOG(INFO) << "Got redirect to " << currentUrl.PrintS(); - continue; + continue; } - + TString errorBody; try { errorBody = fr->GetStream().ReadAll(); diff --git a/ydb/library/yql/utils/future_action.cpp b/ydb/library/yql/utils/future_action.cpp index 26deb8de7e..64c0942042 100644 --- a/ydb/library/yql/utils/future_action.cpp +++ b/ydb/library/yql/utils/future_action.cpp @@ -1 +1 @@ -#include "future_action.h" +#include "future_action.h" diff --git a/ydb/library/yql/utils/future_action.h b/ydb/library/yql/utils/future_action.h index d4a8dd69aa..cdf6d377c4 100644 --- a/ydb/library/yql/utils/future_action.h +++ b/ydb/library/yql/utils/future_action.h @@ -1,59 +1,59 @@ -#pragma once - +#pragma once + #include <library/cpp/threading/future/future.h> - -namespace NYql { -/* -Let's call 'action' attached functor to future which should be called in the main thread. -This trick allows us to execute attached actions not in the future's execution thread but rather in the main thread -and preserve single-threaded logic. -*/ - -/* -Make ready future with constant action inside. -*/ -template <typename T> -NThreading::TFuture<std::function<T()>> MakeFutureWithConstantAction(const T& value) { - return NThreading::MakeFuture<std::function<T()>>([value]() { - return value; - }); -} - -template <typename T, typename A> -auto AddActionToFuture(NThreading::TFuture<T> f, const A& action) { - using V = decltype(action(std::declval<T>())); - - return f.Apply([action](NThreading::TFuture<T> f) { - std::function<V()> r = [f, action]() { - return action(f.GetValue()); - }; - - return r; - }); -} - -/* -Apply continuation with constant action -*/ -template <typename T, typename V> -NThreading::TFuture<std::function<V()>> AddConstantActionToFuture(NThreading::TFuture<T> f, const V& value) { - return AddActionToFuture(f, [value](const T&) { return value; }); -} - -/* -Transform action result by applying mapper -*/ -template <typename R, typename TMapper, typename ...Args> -auto MapFutureAction(NThreading::TFuture<std::function<R(Args&&...)>> f, const TMapper& mapper) { - using V = decltype(mapper(std::declval<R>())); - - return f.Apply([mapper](NThreading::TFuture<std::function<R(Args&&...)>> f) { - std::function<V(Args&&...)> r = [f, mapper](Args&& ...args) { - return mapper(f.GetValue()(std::forward<Args>(args)...)); - }; - - return r; - }); -} - -} + +namespace NYql { +/* +Let's call 'action' attached functor to future which should be called in the main thread. +This trick allows us to execute attached actions not in the future's execution thread but rather in the main thread +and preserve single-threaded logic. +*/ + +/* +Make ready future with constant action inside. +*/ +template <typename T> +NThreading::TFuture<std::function<T()>> MakeFutureWithConstantAction(const T& value) { + return NThreading::MakeFuture<std::function<T()>>([value]() { + return value; + }); +} + +template <typename T, typename A> +auto AddActionToFuture(NThreading::TFuture<T> f, const A& action) { + using V = decltype(action(std::declval<T>())); + + return f.Apply([action](NThreading::TFuture<T> f) { + std::function<V()> r = [f, action]() { + return action(f.GetValue()); + }; + + return r; + }); +} + +/* +Apply continuation with constant action +*/ +template <typename T, typename V> +NThreading::TFuture<std::function<V()>> AddConstantActionToFuture(NThreading::TFuture<T> f, const V& value) { + return AddActionToFuture(f, [value](const T&) { return value; }); +} + +/* +Transform action result by applying mapper +*/ +template <typename R, typename TMapper, typename ...Args> +auto MapFutureAction(NThreading::TFuture<std::function<R(Args&&...)>> f, const TMapper& mapper) { + using V = decltype(mapper(std::declval<R>())); + + return f.Apply([mapper](NThreading::TFuture<std::function<R(Args&&...)>> f) { + std::function<V(Args&&...)> r = [f, mapper](Args&& ...args) { + return mapper(f.GetValue()(std::forward<Args>(args)...)); + }; + + return r; + }); +} + +} diff --git a/ydb/library/yql/utils/md5_stream.cpp b/ydb/library/yql/utils/md5_stream.cpp index 8a1c219164..1c7f887a60 100644 --- a/ydb/library/yql/utils/md5_stream.cpp +++ b/ydb/library/yql/utils/md5_stream.cpp @@ -1,20 +1,20 @@ -#include "md5_stream.h" - -namespace NYql { - -TMd5OutputStream::TMd5OutputStream(IOutputStream& delegatee) - : Delegatee_(delegatee) -{ -} - -TString TMd5OutputStream::Finalize() { - char buf[33] = { 0 }; - return TString(Accumulator_.End(buf)); -} - -void TMd5OutputStream::DoWrite(const void* buf, size_t len) { - Delegatee_.Write(buf, len); - Accumulator_.Update(buf, len); -} - -} +#include "md5_stream.h" + +namespace NYql { + +TMd5OutputStream::TMd5OutputStream(IOutputStream& delegatee) + : Delegatee_(delegatee) +{ +} + +TString TMd5OutputStream::Finalize() { + char buf[33] = { 0 }; + return TString(Accumulator_.End(buf)); +} + +void TMd5OutputStream::DoWrite(const void* buf, size_t len) { + Delegatee_.Write(buf, len); + Accumulator_.Update(buf, len); +} + +} diff --git a/ydb/library/yql/utils/md5_stream.h b/ydb/library/yql/utils/md5_stream.h index ca8b1c4bb9..c8649f6ad4 100644 --- a/ydb/library/yql/utils/md5_stream.h +++ b/ydb/library/yql/utils/md5_stream.h @@ -1,19 +1,19 @@ -#pragma once - -#include <util/stream/output.h> +#pragma once + +#include <util/stream/output.h> #include <library/cpp/digest/md5/md5.h> - -namespace NYql { -class TMd5OutputStream : public IOutputStream { -public: - explicit TMd5OutputStream(IOutputStream& delegatee); - TString Finalize(); - -private: - void DoWrite(const void* buf, size_t len) override; - -private: - IOutputStream& Delegatee_; - MD5 Accumulator_; -}; -} + +namespace NYql { +class TMd5OutputStream : public IOutputStream { +public: + explicit TMd5OutputStream(IOutputStream& delegatee); + TString Finalize(); + +private: + void DoWrite(const void* buf, size_t len) override; + +private: + IOutputStream& Delegatee_; + MD5 Accumulator_; +}; +} diff --git a/ydb/library/yql/utils/md5_stream_ut.cpp b/ydb/library/yql/utils/md5_stream_ut.cpp index 1d04c632d4..d44da69dc0 100644 --- a/ydb/library/yql/utils/md5_stream_ut.cpp +++ b/ydb/library/yql/utils/md5_stream_ut.cpp @@ -1,47 +1,47 @@ -#include "md5_stream.h" -#include <util/stream/input.h> -#include <util/stream/str.h> +#include "md5_stream.h" +#include <util/stream/input.h> +#include <util/stream/str.h> #include <library/cpp/testing/unittest/registar.h> - -using namespace NYql; - -namespace { -TString Consume(const TString& input) { - TStringInput s(input); - - TString output; - TStringOutput outputStream(output); - TMd5OutputStream md5Stream(outputStream); - - UNIT_ASSERT_VALUES_EQUAL(input.size(), TransferData(&s, &md5Stream)); - UNIT_ASSERT_VALUES_EQUAL(input, output); - return md5Stream.Finalize(); -} -} - + +using namespace NYql; + +namespace { +TString Consume(const TString& input) { + TStringInput s(input); + + TString output; + TStringOutput outputStream(output); + TMd5OutputStream md5Stream(outputStream); + + UNIT_ASSERT_VALUES_EQUAL(input.size(), TransferData(&s, &md5Stream)); + UNIT_ASSERT_VALUES_EQUAL(input, output); + return md5Stream.Finalize(); +} +} + Y_UNIT_TEST_SUITE(TStreamMd5Tests) { Y_UNIT_TEST(Empty) { - const auto md5 = Consume(""); - const TString emptyStringMd5 = "d41d8cd98f00b204e9800998ecf8427e"; - UNIT_ASSERT_VALUES_EQUAL(md5, emptyStringMd5); - } - + const auto md5 = Consume(""); + const TString emptyStringMd5 = "d41d8cd98f00b204e9800998ecf8427e"; + UNIT_ASSERT_VALUES_EQUAL(md5, emptyStringMd5); + } + Y_UNIT_TEST(ShortText) { - const auto md5 = Consume("hello from Y!"); - const TString expectedMd5 = "abf59ed7b0daa71085e76e461a737cc2"; - UNIT_ASSERT_VALUES_EQUAL(md5, expectedMd5); - } - + const auto md5 = Consume("hello from Y!"); + const TString expectedMd5 = "abf59ed7b0daa71085e76e461a737cc2"; + UNIT_ASSERT_VALUES_EQUAL(md5, expectedMd5); + } + Y_UNIT_TEST(BigText) { - // TransferData uses TempBuf of 64K - const TString s(1000000, 'A'); - const auto md5 = Consume(s.c_str()); - /* - $ for i in {1..1000000};do echo -n A >> 1M.txt;done - $ md5sum 1M.txt - 48fcdb8b87ce8ef779774199a856091d 1M.txt - */ - const TString expectedMd5 = "48fcdb8b87ce8ef779774199a856091d"; - UNIT_ASSERT_VALUES_EQUAL(md5, expectedMd5); - } -} + // TransferData uses TempBuf of 64K + const TString s(1000000, 'A'); + const auto md5 = Consume(s.c_str()); + /* + $ for i in {1..1000000};do echo -n A >> 1M.txt;done + $ md5sum 1M.txt + 48fcdb8b87ce8ef779774199a856091d 1M.txt + */ + const TString expectedMd5 = "48fcdb8b87ce8ef779774199a856091d"; + UNIT_ASSERT_VALUES_EQUAL(md5, expectedMd5); + } +} diff --git a/ydb/library/yql/utils/multi_resource_lock.cpp b/ydb/library/yql/utils/multi_resource_lock.cpp index b771308236..8eda74c3e0 100644 --- a/ydb/library/yql/utils/multi_resource_lock.cpp +++ b/ydb/library/yql/utils/multi_resource_lock.cpp @@ -1,42 +1,42 @@ -#include "multi_resource_lock.h" - -using namespace NYql; - -TMultiResourceLock::TResourceLock TMultiResourceLock::Acquire(TString resourceId) { - TLock::TPtr lock = ProvideResourceLock(resourceId); - - // resource-specific mutex should be locked outside of Guard_ lock - return { *this, std::move(lock), std::move(resourceId) }; -} - -TMultiResourceLock::~TMultiResourceLock() { - with_lock(Guard_) { +#include "multi_resource_lock.h" + +using namespace NYql; + +TMultiResourceLock::TResourceLock TMultiResourceLock::Acquire(TString resourceId) { + TLock::TPtr lock = ProvideResourceLock(resourceId); + + // resource-specific mutex should be locked outside of Guard_ lock + return { *this, std::move(lock), std::move(resourceId) }; +} + +TMultiResourceLock::~TMultiResourceLock() { + with_lock(Guard_) { Y_VERIFY(Locks_.empty(), "~TMultiResourceLock: we still have %lu unreleased locks", Locks_.size()); - } -} - -TMultiResourceLock::TLock::TPtr TMultiResourceLock::ProvideResourceLock(const TString& resourceId) { - with_lock(Guard_) { - auto it = Locks_.find(resourceId); - if (it == Locks_.end()) { - it = Locks_.emplace(resourceId, MakeIntrusive<TLock>()).first; - } - - // important: ref count will be incremented under lock - // in this case we have guarantee TryCleanup will not erase this resource just after exit from this method and before entering lock->Mutex_.Acquire() - return it->second; - } -} - -void TMultiResourceLock::TryCleanup(const TString& resourceId) { - with_lock(Guard_) { - auto it = Locks_.find(resourceId); - if (it == Locks_.end()) { - return; - } - - if (it->second->IsUnique()) { - Locks_.erase(it); - } - } -} + } +} + +TMultiResourceLock::TLock::TPtr TMultiResourceLock::ProvideResourceLock(const TString& resourceId) { + with_lock(Guard_) { + auto it = Locks_.find(resourceId); + if (it == Locks_.end()) { + it = Locks_.emplace(resourceId, MakeIntrusive<TLock>()).first; + } + + // important: ref count will be incremented under lock + // in this case we have guarantee TryCleanup will not erase this resource just after exit from this method and before entering lock->Mutex_.Acquire() + return it->second; + } +} + +void TMultiResourceLock::TryCleanup(const TString& resourceId) { + with_lock(Guard_) { + auto it = Locks_.find(resourceId); + if (it == Locks_.end()) { + return; + } + + if (it->second->IsUnique()) { + Locks_.erase(it); + } + } +} diff --git a/ydb/library/yql/utils/multi_resource_lock.h b/ydb/library/yql/utils/multi_resource_lock.h index 61804fa9a4..4e75965787 100644 --- a/ydb/library/yql/utils/multi_resource_lock.h +++ b/ydb/library/yql/utils/multi_resource_lock.h @@ -1,81 +1,81 @@ -#pragma once - -#include "yql_panic.h" - -#include <util/generic/map.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/system/mutex.h> - -namespace NYql { - -class TMultiResourceLock : private TNonCopyable { -private: - struct TLock : public TThrRefBase { - typedef TIntrusivePtr<TLock> TPtr; - - bool IsUnique() const { - return RefCount() == 1; - } - - TMutex Mutex_; - }; - -public: - struct TResourceLock : private TNonCopyable { - TResourceLock(TMultiResourceLock& owner, TLock::TPtr lock, TString resourceId) - : Owner_(owner) - , Lock_(std::move(lock)) - , ResourceId_(std::move(resourceId)) - { - Y_ENSURE(Lock_); - Lock_->Mutex_.Acquire(); - } - - TResourceLock(TResourceLock&& other) - : Owner_(other.Owner_) - , Lock_(std::move(other.Lock_)) - , ResourceId_(std::move(other.ResourceId_)) - { - - } - - TResourceLock& operator=(TResourceLock&&) = delete; - - ~TResourceLock() { - if (!Lock_) { - return; - } - - Lock_->Mutex_.Release(); - // decrement ref count before TryCleanup - Lock_ = nullptr; - Owner_.TryCleanup(ResourceId_); - } - - private: - TMultiResourceLock& Owner_; - TLock::TPtr Lock_; - TString ResourceId_; - }; - - TResourceLock Acquire(TString resourceId); - - template <typename F> - auto RunWithLock(TString resourceId, const F& f) -> decltype(f()) { - auto lock = Acquire(std::move(resourceId)); - return f(); - } - - ~TMultiResourceLock(); - -private: - TLock::TPtr ProvideResourceLock(const TString& resourceId); - void TryCleanup(const TString& resourceId); - -private: - TMutex Guard_; - TMap<TString, TLock::TPtr> Locks_; -}; - -} +#pragma once + +#include "yql_panic.h" + +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/system/mutex.h> + +namespace NYql { + +class TMultiResourceLock : private TNonCopyable { +private: + struct TLock : public TThrRefBase { + typedef TIntrusivePtr<TLock> TPtr; + + bool IsUnique() const { + return RefCount() == 1; + } + + TMutex Mutex_; + }; + +public: + struct TResourceLock : private TNonCopyable { + TResourceLock(TMultiResourceLock& owner, TLock::TPtr lock, TString resourceId) + : Owner_(owner) + , Lock_(std::move(lock)) + , ResourceId_(std::move(resourceId)) + { + Y_ENSURE(Lock_); + Lock_->Mutex_.Acquire(); + } + + TResourceLock(TResourceLock&& other) + : Owner_(other.Owner_) + , Lock_(std::move(other.Lock_)) + , ResourceId_(std::move(other.ResourceId_)) + { + + } + + TResourceLock& operator=(TResourceLock&&) = delete; + + ~TResourceLock() { + if (!Lock_) { + return; + } + + Lock_->Mutex_.Release(); + // decrement ref count before TryCleanup + Lock_ = nullptr; + Owner_.TryCleanup(ResourceId_); + } + + private: + TMultiResourceLock& Owner_; + TLock::TPtr Lock_; + TString ResourceId_; + }; + + TResourceLock Acquire(TString resourceId); + + template <typename F> + auto RunWithLock(TString resourceId, const F& f) -> decltype(f()) { + auto lock = Acquire(std::move(resourceId)); + return f(); + } + + ~TMultiResourceLock(); + +private: + TLock::TPtr ProvideResourceLock(const TString& resourceId); + void TryCleanup(const TString& resourceId); + +private: + TMutex Guard_; + TMap<TString, TLock::TPtr> Locks_; +}; + +} diff --git a/ydb/library/yql/utils/multi_resource_lock_ut.cpp b/ydb/library/yql/utils/multi_resource_lock_ut.cpp index 0af9cea3ff..2752da8ec8 100644 --- a/ydb/library/yql/utils/multi_resource_lock_ut.cpp +++ b/ydb/library/yql/utils/multi_resource_lock_ut.cpp @@ -1,54 +1,54 @@ -#include "multi_resource_lock.h" -#include <util/generic/xrange.h> +#include "multi_resource_lock.h" +#include <util/generic/xrange.h> #include <library/cpp/threading/future/async.h> #include <library/cpp/testing/unittest/registar.h> - -namespace NYql { -using namespace NThreading; - + +namespace NYql { +using namespace NThreading; + Y_UNIT_TEST_SUITE(TMultiResourceLock) { Y_UNIT_TEST(ManyResources) { - TMultiResourceLock multiLock; - const int workersCount = 3; - TVector<TVector<int>> workersData; - workersData.resize(workersCount); - + TMultiResourceLock multiLock; + const int workersCount = 3; + TVector<TVector<int>> workersData; + workersData.resize(workersCount); + TAdaptiveThreadPool queue; - queue.Start(0); - - TVector<NThreading::TFuture<void>> workers; - workers.reserve(workersCount); - TManualEvent startEvent; - - for (int i = 0; i < workersCount; ++i) { - TString resourceId = ToString(i); - TVector<int>& data = workersData.at(i); - NThreading::TFuture<void> f = NThreading::Async([&, resourceId]() { - startEvent.Wait(); - - for (int j = 0; j < 1000; ++j) { - const auto& l = multiLock.Acquire(resourceId); - Y_UNUSED(l); - data.push_back(j); - } - }, queue); - - workers.push_back(std::move(f)); - } - - startEvent.Signal(); - + queue.Start(0); + + TVector<NThreading::TFuture<void>> workers; + workers.reserve(workersCount); + TManualEvent startEvent; + + for (int i = 0; i < workersCount; ++i) { + TString resourceId = ToString(i); + TVector<int>& data = workersData.at(i); + NThreading::TFuture<void> f = NThreading::Async([&, resourceId]() { + startEvent.Wait(); + + for (int j = 0; j < 1000; ++j) { + const auto& l = multiLock.Acquire(resourceId); + Y_UNUSED(l); + data.push_back(j); + } + }, queue); + + workers.push_back(std::move(f)); + } + + startEvent.Signal(); + NThreading::TFuture<void> all = NThreading::WaitExceptionOrAll(workers); - all.GetValueSync(); - queue.Stop(); - - // analyze workersData: - auto range0_999 = xrange(0, 1000); - for (auto& w : workersData) { - UNIT_ASSERT_VALUES_EQUAL(w.size(), 1000); - UNIT_ASSERT(std::equal(range0_999.begin(), range0_999.end(), w.begin())); - } - } -} - -} + all.GetValueSync(); + queue.Stop(); + + // analyze workersData: + auto range0_999 = xrange(0, 1000); + for (auto& w : workersData) { + UNIT_ASSERT_VALUES_EQUAL(w.size(), 1000); + UNIT_ASSERT(std::equal(range0_999.begin(), range0_999.end(), w.begin())); + } + } +} + +} diff --git a/ydb/library/yql/utils/parse_double.cpp b/ydb/library/yql/utils/parse_double.cpp index 0c06ad9544..2b49b503ff 100644 --- a/ydb/library/yql/utils/parse_double.cpp +++ b/ydb/library/yql/utils/parse_double.cpp @@ -1,70 +1,70 @@ -#include "parse_double.h" - -#include <util/string/ascii.h> -#include <util/string/cast.h> - -namespace NYql { - -namespace { -template <typename T> -bool GenericTryFloatFromString(TStringBuf buf, T& value) { - value = 0; +#include "parse_double.h" + +#include <util/string/ascii.h> +#include <util/string/cast.h> + +namespace NYql { + +namespace { +template <typename T> +bool GenericTryFloatFromString(TStringBuf buf, T& value) { + value = 0; if (!buf.size() || !TryFromString(buf.data(), buf.size(), value)) { const char* ptr = buf.data(); ui32 size = buf.size(); - char sign = '+'; - if (*ptr == '+' || *ptr == '-') { - sign = *ptr; - ++ptr; - --size; - } - - if (size != 3) { - return false; - } - - // NaN or Inf (ignoring case) - if (AsciiToUpper(ptr[0]) == 'N' && AsciiToUpper(ptr[1]) == 'A' && AsciiToUpper(ptr[2]) == 'N') { - value = std::numeric_limits<T>::quiet_NaN(); - } else if (AsciiToUpper(ptr[0]) == 'I' && AsciiToUpper(ptr[1]) == 'N' && AsciiToUpper(ptr[2]) == 'F') { - value = std::numeric_limits<T>::infinity(); - } else { - return false; - } - - if (sign == '-') { - value = -value; - } - } - - return true; -} -} - -float FloatFromString(TStringBuf buf) { - float result = 0; - if (!TryFloatFromString(buf, result)) { - throw yexception() << "unable to parse float from '" << buf << "'"; - } - - return result; -} - -double DoubleFromString(TStringBuf buf) { - double result = 0; - if (!TryDoubleFromString(buf, result)) { - throw yexception() << "unable to parse double from '" << buf << "'"; - } - - return result; -} - -bool TryFloatFromString(TStringBuf buf, float& value) { - return GenericTryFloatFromString(buf, value); -} - -bool TryDoubleFromString(TStringBuf buf, double& value) { - return GenericTryFloatFromString(buf, value); -} - -} + char sign = '+'; + if (*ptr == '+' || *ptr == '-') { + sign = *ptr; + ++ptr; + --size; + } + + if (size != 3) { + return false; + } + + // NaN or Inf (ignoring case) + if (AsciiToUpper(ptr[0]) == 'N' && AsciiToUpper(ptr[1]) == 'A' && AsciiToUpper(ptr[2]) == 'N') { + value = std::numeric_limits<T>::quiet_NaN(); + } else if (AsciiToUpper(ptr[0]) == 'I' && AsciiToUpper(ptr[1]) == 'N' && AsciiToUpper(ptr[2]) == 'F') { + value = std::numeric_limits<T>::infinity(); + } else { + return false; + } + + if (sign == '-') { + value = -value; + } + } + + return true; +} +} + +float FloatFromString(TStringBuf buf) { + float result = 0; + if (!TryFloatFromString(buf, result)) { + throw yexception() << "unable to parse float from '" << buf << "'"; + } + + return result; +} + +double DoubleFromString(TStringBuf buf) { + double result = 0; + if (!TryDoubleFromString(buf, result)) { + throw yexception() << "unable to parse double from '" << buf << "'"; + } + + return result; +} + +bool TryFloatFromString(TStringBuf buf, float& value) { + return GenericTryFloatFromString(buf, value); +} + +bool TryDoubleFromString(TStringBuf buf, double& value) { + return GenericTryFloatFromString(buf, value); +} + +} diff --git a/ydb/library/yql/utils/parse_double.h b/ydb/library/yql/utils/parse_double.h index 61d1d940c9..400dd48330 100644 --- a/ydb/library/yql/utils/parse_double.h +++ b/ydb/library/yql/utils/parse_double.h @@ -1,18 +1,18 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NYql { - -/* -These parse functions can understand nan, inf, -inf case-insensitively -They do not parse empty string to zero -*/ - -float FloatFromString(TStringBuf buf); -double DoubleFromString(TStringBuf buf); - -bool TryFloatFromString(TStringBuf buf, float& value); -bool TryDoubleFromString(TStringBuf buf, double& value); - -} +#pragma once + +#include <util/generic/strbuf.h> + +namespace NYql { + +/* +These parse functions can understand nan, inf, -inf case-insensitively +They do not parse empty string to zero +*/ + +float FloatFromString(TStringBuf buf); +double DoubleFromString(TStringBuf buf); + +bool TryFloatFromString(TStringBuf buf, float& value); +bool TryDoubleFromString(TStringBuf buf, double& value); + +} diff --git a/ydb/library/yql/utils/parse_double_ut.cpp b/ydb/library/yql/utils/parse_double_ut.cpp index 6f9cdea2f1..fc63d5c297 100644 --- a/ydb/library/yql/utils/parse_double_ut.cpp +++ b/ydb/library/yql/utils/parse_double_ut.cpp @@ -1,52 +1,52 @@ -#include "parse_double.h" +#include "parse_double.h" #include <library/cpp/testing/unittest/registar.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(TParseDouble) { - - template <typename T, typename F> - void ParseAndCheck(TStringBuf buf, F f, T expected) { - T result = 0; - UNIT_ASSERT(f(buf, result)); - UNIT_ASSERT_DOUBLES_EQUAL(expected, result, 1e-6); - } - - Y_UNIT_TEST(ExactValues) { + +namespace NYql { + +Y_UNIT_TEST_SUITE(TParseDouble) { + + template <typename T, typename F> + void ParseAndCheck(TStringBuf buf, F f, T expected) { + T result = 0; + UNIT_ASSERT(f(buf, result)); + UNIT_ASSERT_DOUBLES_EQUAL(expected, result, 1e-6); + } + + Y_UNIT_TEST(ExactValues) { ParseAndCheck(TStringBuf("nan"), TryFloatFromString, std::numeric_limits<float>::quiet_NaN()); ParseAndCheck(TStringBuf("nAn"), TryDoubleFromString, std::numeric_limits<double>::quiet_NaN()); - + ParseAndCheck(TStringBuf("+nan"), TryFloatFromString, std::numeric_limits<float>::quiet_NaN()); ParseAndCheck(TStringBuf("+NAN"), TryDoubleFromString, std::numeric_limits<double>::quiet_NaN()); - + ParseAndCheck(TStringBuf("-nan"), TryFloatFromString, std::numeric_limits<float>::quiet_NaN()); ParseAndCheck(TStringBuf("-NaN"), TryDoubleFromString, std::numeric_limits<double>::quiet_NaN()); - + ParseAndCheck(TStringBuf("inf"), TryFloatFromString, std::numeric_limits<float>::infinity()); ParseAndCheck(TStringBuf("iNf"), TryDoubleFromString, std::numeric_limits<double>::infinity()); - + ParseAndCheck(TStringBuf("+inf"), TryFloatFromString, std::numeric_limits<float>::infinity()); ParseAndCheck(TStringBuf("+INF"), TryDoubleFromString, std::numeric_limits<double>::infinity()); - + ParseAndCheck(TStringBuf("-inf"), TryFloatFromString, -std::numeric_limits<float>::infinity()); ParseAndCheck(TStringBuf("-InF"), TryDoubleFromString, -std::numeric_limits<double>::infinity()); - + ParseAndCheck<float>(TStringBuf("-12.3456"), TryFloatFromString, -12.3456); ParseAndCheck(TStringBuf("-12.3456"), TryDoubleFromString, -12.3456); - + ParseAndCheck<float>(TStringBuf("1.23e-2"), TryFloatFromString, 0.0123); ParseAndCheck(TStringBuf("1.23e-2"), TryDoubleFromString, 0.0123); - + UNIT_ASSERT_EQUAL(FloatFromString(TStringBuf("iNf")), std::numeric_limits<float>::infinity()); UNIT_ASSERT_EQUAL(DoubleFromString(TStringBuf("iNf")), std::numeric_limits<float>::infinity()); - } - - Y_UNIT_TEST(Errors) { + } + + Y_UNIT_TEST(Errors) { UNIT_ASSERT_EXCEPTION_CONTAINS(FloatFromString(TStringBuf("")), std::exception, "unable to parse float from ''"); UNIT_ASSERT_EXCEPTION_CONTAINS(DoubleFromString(TStringBuf("")), std::exception, "unable to parse double from ''"); - + UNIT_ASSERT_EXCEPTION_CONTAINS(FloatFromString(TStringBuf("info")), std::exception, "unable to parse float from 'info'"); UNIT_ASSERT_EXCEPTION_CONTAINS(DoubleFromString(TStringBuf("-nana")), std::exception, "unable to parse double from '-nana'"); - } -} -} + } +} +} diff --git a/ydb/library/yql/utils/rand_guid.cpp b/ydb/library/yql/utils/rand_guid.cpp index 383d250c0d..982e65108b 100644 --- a/ydb/library/yql/utils/rand_guid.cpp +++ b/ydb/library/yql/utils/rand_guid.cpp @@ -1,32 +1,32 @@ -#include "rand_guid.h" - -#include <util/system/datetime.h> -#include <util/system/getpid.h> -#include <util/system/unaligned_mem.h> -#include <util/generic/guid.h> - -namespace NYql { - -TAtomic TRandGuid::Counter = 0; - -TRandGuid::TRandGuid() { - ResetSeed(); -} - -void TRandGuid::ResetSeed() { - Rnd.Reset(new TMersenne<ui64>(GetCycleCount() + MicroSeconds() + GetPID())); -} - -TString TRandGuid::GenGuid() { - TGUID ret = {}; +#include "rand_guid.h" + +#include <util/system/datetime.h> +#include <util/system/getpid.h> +#include <util/system/unaligned_mem.h> +#include <util/generic/guid.h> + +namespace NYql { + +TAtomic TRandGuid::Counter = 0; + +TRandGuid::TRandGuid() { + ResetSeed(); +} + +void TRandGuid::ResetSeed() { + Rnd.Reset(new TMersenne<ui64>(GetCycleCount() + MicroSeconds() + GetPID())); +} + +TString TRandGuid::GenGuid() { + TGUID ret = {}; WriteUnaligned<ui64>(ret.dw, Rnd->GenRand()); - ret.dw[2] = (ui32)Rnd->GenRand(); - ret.dw[3] = AtomicIncrement(Counter); - - return GetGuidAsString(ret); -} - -ui64 TRandGuid::GenNumber() { - return Rnd->GenRand(); -} -} + ret.dw[2] = (ui32)Rnd->GenRand(); + ret.dw[3] = AtomicIncrement(Counter); + + return GetGuidAsString(ret); +} + +ui64 TRandGuid::GenNumber() { + return Rnd->GenRand(); +} +} diff --git a/ydb/library/yql/utils/rand_guid.h b/ydb/library/yql/utils/rand_guid.h index 97d50cfa65..6594b05b29 100644 --- a/ydb/library/yql/utils/rand_guid.h +++ b/ydb/library/yql/utils/rand_guid.h @@ -1,22 +1,22 @@ -#pragma once - -#include <util/random/mersenne.h> -#include <util/generic/ptr.h> - -namespace NYql { -class TRandGuid { -public: - TRandGuid(); - TRandGuid(TRandGuid&&) = default; - TRandGuid& operator=(TRandGuid&&) = default; - - void ResetSeed(); - - TString GenGuid(); - ui64 GenNumber(); - -private: - THolder<TMersenne<ui64>> Rnd; - static TAtomic Counter; -}; -} +#pragma once + +#include <util/random/mersenne.h> +#include <util/generic/ptr.h> + +namespace NYql { +class TRandGuid { +public: + TRandGuid(); + TRandGuid(TRandGuid&&) = default; + TRandGuid& operator=(TRandGuid&&) = default; + + void ResetSeed(); + + TString GenGuid(); + ui64 GenNumber(); + +private: + THolder<TMersenne<ui64>> Rnd; + static TAtomic Counter; +}; +} diff --git a/ydb/library/yql/utils/range_walker.h b/ydb/library/yql/utils/range_walker.h index 268fe36472..4ecb765556 100644 --- a/ydb/library/yql/utils/range_walker.h +++ b/ydb/library/yql/utils/range_walker.h @@ -1,47 +1,47 @@ -#pragma once - -#include <util/generic/yexception.h> - -namespace NYql { - -template <typename T> -class TRangeWalker { -private: - const T Start_; - const T Finish_; - T Current_; - -public: - TRangeWalker(T start, T finish) - : Start_(start) - , Finish_(finish) - , Current_(start) - { - if (Start_ > Finish_) { - ythrow yexception() << "Invalid range for walker"; - } - } - - T GetStart() const { - return Start_; - } - - T GetFinish() const { - return Finish_; - } - - T GetRangeSize() const { - return Finish_ - Start_ + 1; - } - - T MoveToNext() { - T result = Current_++; - - if (Current_ > Finish_) { - Current_ = Start_; - } - - return result; - } -}; -} +#pragma once + +#include <util/generic/yexception.h> + +namespace NYql { + +template <typename T> +class TRangeWalker { +private: + const T Start_; + const T Finish_; + T Current_; + +public: + TRangeWalker(T start, T finish) + : Start_(start) + , Finish_(finish) + , Current_(start) + { + if (Start_ > Finish_) { + ythrow yexception() << "Invalid range for walker"; + } + } + + T GetStart() const { + return Start_; + } + + T GetFinish() const { + return Finish_; + } + + T GetRangeSize() const { + return Finish_ - Start_ + 1; + } + + T MoveToNext() { + T result = Current_++; + + if (Current_ > Finish_) { + Current_ = Start_; + } + + return result; + } +}; +} diff --git a/ydb/library/yql/utils/range_walker_ut.cpp b/ydb/library/yql/utils/range_walker_ut.cpp index d6a86cc804..215998571d 100644 --- a/ydb/library/yql/utils/range_walker_ut.cpp +++ b/ydb/library/yql/utils/range_walker_ut.cpp @@ -1,35 +1,35 @@ -#include "range_walker.h" - +#include "range_walker.h" + #include <library/cpp/testing/unittest/registar.h> - -using namespace NYql; - + +using namespace NYql; + Y_UNIT_TEST_SUITE(TRangeWalkerTests) { Y_UNIT_TEST(InvalidRange) { - UNIT_ASSERT_EXCEPTION_CONTAINS(TRangeWalker<int>(2, 1), yexception, "Invalid range for walker"); - } - + UNIT_ASSERT_EXCEPTION_CONTAINS(TRangeWalker<int>(2, 1), yexception, "Invalid range for walker"); + } + Y_UNIT_TEST(SingleValueRange) { - TRangeWalker<int> w(5, 5); - UNIT_ASSERT_EQUAL(5, w.GetStart()); - UNIT_ASSERT_EQUAL(5, w.GetFinish()); - UNIT_ASSERT_EQUAL(1, w.GetRangeSize()); - - for (int i = 0; i < 10; ++i) { - UNIT_ASSERT_EQUAL(5, w.MoveToNext()); - } - } - + TRangeWalker<int> w(5, 5); + UNIT_ASSERT_EQUAL(5, w.GetStart()); + UNIT_ASSERT_EQUAL(5, w.GetFinish()); + UNIT_ASSERT_EQUAL(1, w.GetRangeSize()); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_EQUAL(5, w.MoveToNext()); + } + } + Y_UNIT_TEST(ManyValuesRange) { - TRangeWalker<int> w(5, 7); - UNIT_ASSERT_EQUAL(5, w.GetStart()); - UNIT_ASSERT_EQUAL(7, w.GetFinish()); - UNIT_ASSERT_EQUAL(3, w.GetRangeSize()); - - for (int i = 0; i < 10; ++i) { - UNIT_ASSERT_EQUAL(5, w.MoveToNext()); - UNIT_ASSERT_EQUAL(6, w.MoveToNext()); - UNIT_ASSERT_EQUAL(7, w.MoveToNext()); - } - } -} + TRangeWalker<int> w(5, 7); + UNIT_ASSERT_EQUAL(5, w.GetStart()); + UNIT_ASSERT_EQUAL(7, w.GetFinish()); + UNIT_ASSERT_EQUAL(3, w.GetRangeSize()); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_EQUAL(5, w.MoveToNext()); + UNIT_ASSERT_EQUAL(6, w.MoveToNext()); + UNIT_ASSERT_EQUAL(7, w.MoveToNext()); + } + } +} diff --git a/ydb/library/yql/utils/retry.cpp b/ydb/library/yql/utils/retry.cpp index ac40d79b12..6720eeb13a 100644 --- a/ydb/library/yql/utils/retry.cpp +++ b/ydb/library/yql/utils/retry.cpp @@ -1 +1 @@ -#include "retry.h" +#include "retry.h" diff --git a/ydb/library/yql/utils/retry.h b/ydb/library/yql/utils/retry.h index 7735aa1a8c..afd0859e4c 100644 --- a/ydb/library/yql/utils/retry.h +++ b/ydb/library/yql/utils/retry.h @@ -1,17 +1,17 @@ -#pragma once - -namespace NYql { - -template <typename TRetriableException, typename TAction, typename TExceptionHander> -auto WithRetry(int attempts, TAction&& a, TExceptionHander&& exceptionHandler) { - for (int i = 1; i < attempts; ++i) { - try { - return a(); - } catch (const TRetriableException& e) { - exceptionHandler(e, i, attempts); - } - } - - return a(); -} -} +#pragma once + +namespace NYql { + +template <typename TRetriableException, typename TAction, typename TExceptionHander> +auto WithRetry(int attempts, TAction&& a, TExceptionHander&& exceptionHandler) { + for (int i = 1; i < attempts; ++i) { + try { + return a(); + } catch (const TRetriableException& e) { + exceptionHandler(e, i, attempts); + } + } + + return a(); +} +} diff --git a/ydb/library/yql/utils/retry_ut.cpp b/ydb/library/yql/utils/retry_ut.cpp index 47cb35fd73..5c9a64a02d 100644 --- a/ydb/library/yql/utils/retry_ut.cpp +++ b/ydb/library/yql/utils/retry_ut.cpp @@ -1,73 +1,73 @@ -#include "retry.h" - +#include "retry.h" + #include <library/cpp/testing/unittest/registar.h> - -using namespace NYql; - -namespace { - -class TMyError : public yexception { -}; - -} - + +using namespace NYql; + +namespace { + +class TMyError : public yexception { +}; + +} + Y_UNIT_TEST_SUITE(TRetryTests) { Y_UNIT_TEST(ZeroAttempts) { - auto r = WithRetry<TMyError>(0, - []() { return TString("abc"); }, - [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }); - - UNIT_ASSERT_VALUES_EQUAL("abc", r); - } - + auto r = WithRetry<TMyError>(0, + []() { return TString("abc"); }, + [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }); + + UNIT_ASSERT_VALUES_EQUAL("abc", r); + } + Y_UNIT_TEST(NoRetries) { - auto r = WithRetry<TMyError>(5, - []() { return TString("abc"); }, - [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }); - - UNIT_ASSERT_VALUES_EQUAL("abc", r); - } - + auto r = WithRetry<TMyError>(5, + []() { return TString("abc"); }, + [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }); + + UNIT_ASSERT_VALUES_EQUAL("abc", r); + } + Y_UNIT_TEST(NoRetriesButException) { - UNIT_ASSERT_EXCEPTION_CONTAINS(WithRetry<TMyError>(5, - []() { throw yexception() << "xxxx"; }, - [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }), yexception, "xxxx"); - } - + UNIT_ASSERT_EXCEPTION_CONTAINS(WithRetry<TMyError>(5, + []() { throw yexception() << "xxxx"; }, + [](auto, auto, auto) { UNIT_FAIL("Exception handler invoked"); }), yexception, "xxxx"); + } + Y_UNIT_TEST(FewRetries) { - int counter = 0; - int exceptions = 0; - auto r = WithRetry<TMyError>(3, [&]() { - if (counter++ < 2) { - throw TMyError() << "yyyy"; - } - - return counter; - }, [&](const auto& e, int attempt, int attemptCount) { - ++exceptions; - UNIT_ASSERT_VALUES_EQUAL(e.what(), "yyyy"); - UNIT_ASSERT_VALUES_EQUAL(attempt, counter); - UNIT_ASSERT_VALUES_EQUAL(attemptCount, 3); - }); - - UNIT_ASSERT_VALUES_EQUAL(2, exceptions); - UNIT_ASSERT_VALUES_EQUAL(3, r); - UNIT_ASSERT_VALUES_EQUAL(3, counter); - } - + int counter = 0; + int exceptions = 0; + auto r = WithRetry<TMyError>(3, [&]() { + if (counter++ < 2) { + throw TMyError() << "yyyy"; + } + + return counter; + }, [&](const auto& e, int attempt, int attemptCount) { + ++exceptions; + UNIT_ASSERT_VALUES_EQUAL(e.what(), "yyyy"); + UNIT_ASSERT_VALUES_EQUAL(attempt, counter); + UNIT_ASSERT_VALUES_EQUAL(attemptCount, 3); + }); + + UNIT_ASSERT_VALUES_EQUAL(2, exceptions); + UNIT_ASSERT_VALUES_EQUAL(3, r); + UNIT_ASSERT_VALUES_EQUAL(3, counter); + } + Y_UNIT_TEST(ManyRetries) { - int counter = 0; - int exceptions = 0; - UNIT_ASSERT_EXCEPTION_CONTAINS(WithRetry<TMyError>(3, [&]() { - throw TMyError() << "yyyy" << counter++; - }, [&](const auto& e, int attempt, int attemptCount) { - ++exceptions; - UNIT_ASSERT_STRING_CONTAINS(e.what(), "yyyy"); - UNIT_ASSERT_VALUES_EQUAL(attempt, counter); - UNIT_ASSERT_VALUES_EQUAL(attemptCount, 3); - }), TMyError, "yyyy2"); - - UNIT_ASSERT_VALUES_EQUAL(2, exceptions); - UNIT_ASSERT_VALUES_EQUAL(3, counter); - } -} + int counter = 0; + int exceptions = 0; + UNIT_ASSERT_EXCEPTION_CONTAINS(WithRetry<TMyError>(3, [&]() { + throw TMyError() << "yyyy" << counter++; + }, [&](const auto& e, int attempt, int attemptCount) { + ++exceptions; + UNIT_ASSERT_STRING_CONTAINS(e.what(), "yyyy"); + UNIT_ASSERT_VALUES_EQUAL(attempt, counter); + UNIT_ASSERT_VALUES_EQUAL(attemptCount, 3); + }), TMyError, "yyyy2"); + + UNIT_ASSERT_VALUES_EQUAL(2, exceptions); + UNIT_ASSERT_VALUES_EQUAL(3, counter); + } +} diff --git a/ydb/library/yql/utils/ut/ya.make b/ydb/library/yql/utils/ut/ya.make index 0d26eb2c5f..9841fc454f 100644 --- a/ydb/library/yql/utils/ut/ya.make +++ b/ydb/library/yql/utils/ut/ya.make @@ -1,15 +1,15 @@ UNITTEST_FOR(ydb/library/yql/utils) - -OWNER(g:yql) - -SRCS( + +OWNER(g:yql) + +SRCS( fp_bits_ut.cpp - md5_stream_ut.cpp - multi_resource_lock_ut.cpp - parse_double_ut.cpp - range_walker_ut.cpp - retry_ut.cpp + md5_stream_ut.cpp + multi_resource_lock_ut.cpp + parse_double_ut.cpp + range_walker_ut.cpp + retry_ut.cpp utf8_ut.cpp -) - -END() +) + +END() diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make index afba70be53..b9b5efab23 100644 --- a/ydb/library/yql/utils/ya.make +++ b/ydb/library/yql/utils/ya.make @@ -11,38 +11,38 @@ SRCS( cast.h debug_info.cpp debug_info.h - future_action.cpp - future_action.h + future_action.cpp + future_action.h hash.h hash.cpp - md5_stream.cpp - md5_stream.h - multi_resource_lock.cpp - multi_resource_lock.h - parse_double.cpp - parse_double.h + md5_stream.cpp + md5_stream.h + multi_resource_lock.cpp + multi_resource_lock.h + parse_double.cpp + parse_double.h proc_alive.cpp proc_alive.h - rand_guid.cpp - rand_guid.h + rand_guid.cpp + rand_guid.h resetable_setting.h - retry.cpp - retry.h + retry.cpp + retry.h swap_bytes.cpp swap_bytes.h - yql_panic.cpp - yql_panic.h + yql_panic.cpp + yql_panic.h yql_paths.cpp yql_paths.h utf8.cpp ) -PEERDIR( +PEERDIR( library/cpp/digest/md5 library/cpp/threading/future library/cpp/messagebus -) - +) + END() RECURSE( diff --git a/ydb/public/api/protos/yq.proto b/ydb/public/api/protos/yq.proto index 6aa7169a89..874a8afd67 100644 --- a/ydb/public/api/protos/yq.proto +++ b/ydb/public/api/protos/yq.proto @@ -607,7 +607,7 @@ message DataStreamsBinding { string format = 2 [(Ydb.length).le = 1024]; string compression = 3 [(Ydb.length).le = 1024]; Schema schema = 4; - map<string, string> format_setting = 5 [(Ydb.size).le = 100]; + map<string, string> format_setting = 5 [(Ydb.size).le = 100]; } message ObjectStorageBinding { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp index f9f2a16e85..d431d87cf2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp @@ -25,17 +25,17 @@ std::shared_ptr<IProfileManager> CreateYdbProfileManager(const TString& ydbDir) TCommandConfig::TCommandConfig() : TClientCommandTree("config", {}, "Manage YDB CLI configuration") { - AddCommand(std::make_unique<TCommandProfile>()); + AddCommand(std::make_unique<TCommandProfile>()); } TCommandProfile::TCommandProfile() : TClientCommandTree("profile", {}, "Manage configuration profiles") { - AddCommand(std::make_unique<TCommandCreateProfile>()); - AddCommand(std::make_unique<TCommandDeleteProfile>()); - AddCommand(std::make_unique<TCommandActivateProfile>()); - AddCommand(std::make_unique<TCommandListProfiles>()); - AddCommand(std::make_unique<TCommandGetProfile>()); + AddCommand(std::make_unique<TCommandCreateProfile>()); + AddCommand(std::make_unique<TCommandDeleteProfile>()); + AddCommand(std::make_unique<TCommandActivateProfile>()); + AddCommand(std::make_unique<TCommandListProfiles>()); + AddCommand(std::make_unique<TCommandGetProfile>()); } namespace { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 2cccf42a41..d1f58668c9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -26,17 +26,17 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TClientSettings& settin , Settings (settings) { ValidateSettings(); - AddCommand(std::make_unique<TCommandDiscovery>()); - AddCommand(std::make_unique<TCommandScheme>()); - AddCommand(std::make_unique<TCommandScripting>()); - AddCommand(std::make_unique<TCommandTable>()); - AddCommand(std::make_unique<TCommandTools>()); - AddCommand(std::make_unique<TCommandExport>(Settings.UseExportToYt.GetRef())); - AddCommand(std::make_unique<TCommandImport>()); - AddCommand(std::make_unique<TCommandOperation>()); - AddCommand(std::make_unique<TCommandConfig>()); - AddCommand(std::make_unique<TCommandInit>()); - AddCommand(std::make_unique<TCommandYql>()); + AddCommand(std::make_unique<TCommandDiscovery>()); + AddCommand(std::make_unique<TCommandScheme>()); + AddCommand(std::make_unique<TCommandScripting>()); + AddCommand(std::make_unique<TCommandTable>()); + AddCommand(std::make_unique<TCommandTools>()); + AddCommand(std::make_unique<TCommandExport>(Settings.UseExportToYt.GetRef())); + AddCommand(std::make_unique<TCommandImport>()); + AddCommand(std::make_unique<TCommandOperation>()); + AddCommand(std::make_unique<TCommandConfig>()); + AddCommand(std::make_unique<TCommandInit>()); + AddCommand(std::make_unique<TCommandYql>()); AddCommand(std::make_unique<TCommandStream>()); AddCommand(std::make_unique<TCommandWorkload>()); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp index 18aa3a65e5..800d5f963b 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp @@ -6,8 +6,8 @@ namespace NConsoleClient { TCommandDiscovery::TCommandDiscovery() : TClientCommandTree("discovery", {}, "Discovery service operations") { - AddCommand(std::make_unique<TCommandListEndpoints>()); - AddCommand(std::make_unique<TCommandWhoAmI>()); + AddCommand(std::make_unique<TCommandListEndpoints>()); + AddCommand(std::make_unique<TCommandWhoAmI>()); } TCommandListEndpoints::TCommandListEndpoints() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp index f997343dc8..0c4430d5dc 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp @@ -72,9 +72,9 @@ TCommandExport::TCommandExport(bool useExportToYt) : TClientCommandTree("export", {}, "Export service operations") { if (useExportToYt) { - AddCommand(std::make_unique<TCommandExportToYt>()); + AddCommand(std::make_unique<TCommandExportToYt>()); } - AddCommand(std::make_unique<TCommandExportToS3>()); + AddCommand(std::make_unique<TCommandExportToS3>()); } /// YT diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp index b28e881428..4025a3d872 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp @@ -15,8 +15,8 @@ namespace NConsoleClient { TCommandImport::TCommandImport() : TClientCommandTree("import", {}, "Import service operations") { - AddCommand(std::make_unique<TCommandImportFromS3>()); - AddCommand(std::make_unique<TCommandImportFromFile>()); + AddCommand(std::make_unique<TCommandImportFromS3>()); + AddCommand(std::make_unique<TCommandImportFromFile>()); } /// S3 @@ -128,8 +128,8 @@ int TCommandImportFromS3::Run(TConfig& config) { TCommandImportFromFile::TCommandImportFromFile() : TClientCommandTree("file", {}, "Import data from file") { - AddCommand(std::make_unique<TCommandImportFromCsv>()); - AddCommand(std::make_unique<TCommandImportFromTsv>()); + AddCommand(std::make_unique<TCommandImportFromCsv>()); + AddCommand(std::make_unique<TCommandImportFromTsv>()); } /// CSV diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp index 45b694cf94..0ee76deff2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp @@ -42,10 +42,10 @@ namespace { TCommandOperation::TCommandOperation() : TClientCommandTree("operation", {}, "Operation service operations") { - AddCommand(std::make_unique<TCommandGetOperation>()); - AddCommand(std::make_unique<TCommandCancelOperation>()); - AddCommand(std::make_unique<TCommandForgetOperation>()); - AddCommand(std::make_unique<TCommandListOperations>()); + AddCommand(std::make_unique<TCommandGetOperation>()); + AddCommand(std::make_unique<TCommandCancelOperation>()); + AddCommand(std::make_unique<TCommandForgetOperation>()); + AddCommand(std::make_unique<TCommandListOperations>()); } void TCommandWithOperationId::Config(TConfig& config) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index a6f81fe432..e62f61e2e4 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -13,11 +13,11 @@ namespace NConsoleClient { TCommandScheme::TCommandScheme() : TClientCommandTree("scheme", {}, "Scheme service operations") { - AddCommand(std::make_unique<TCommandMakeDirectory>()); - AddCommand(std::make_unique<TCommandRemoveDirectory>()); - AddCommand(std::make_unique<TCommandDescribe>()); - AddCommand(std::make_unique<TCommandList>()); - AddCommand(std::make_unique<TCommandPermissions>()); + AddCommand(std::make_unique<TCommandMakeDirectory>()); + AddCommand(std::make_unique<TCommandRemoveDirectory>()); + AddCommand(std::make_unique<TCommandDescribe>()); + AddCommand(std::make_unique<TCommandList>()); + AddCommand(std::make_unique<TCommandPermissions>()); } TCommandMakeDirectory::TCommandMakeDirectory() @@ -892,11 +892,11 @@ void TCommandList::AddEntriesRecursive( TCommandPermissions::TCommandPermissions() : TClientCommandTree("permissions", {}, "Modify permissions") { - AddCommand(std::make_unique<TCommandPermissionGrant>()); - AddCommand(std::make_unique<TCommandPermissionRevoke>()); - AddCommand(std::make_unique<TCommandPermissionSet>()); - AddCommand(std::make_unique<TCommandChangeOwner>()); - AddCommand(std::make_unique<TCommandPermissionClear>()); + AddCommand(std::make_unique<TCommandPermissionGrant>()); + AddCommand(std::make_unique<TCommandPermissionRevoke>()); + AddCommand(std::make_unique<TCommandPermissionSet>()); + AddCommand(std::make_unique<TCommandChangeOwner>()); + AddCommand(std::make_unique<TCommandPermissionClear>()); } TCommandPermissionGrant::TCommandPermissionGrant() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index 4a25c372ce..c3f726a538 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -12,7 +12,7 @@ namespace NConsoleClient { TCommandScripting::TCommandScripting() : TClientCommandTree("scripting", {}, "Scripting service operations") { - AddCommand(std::make_unique<TCommandExecuteYqlScript>()); + AddCommand(std::make_unique<TCommandExecuteYqlScript>()); } TCommandExecuteYqlScript::TCommandExecuteYqlScript() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_stream.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_stream.cpp index 24b478f881..51754b4aee 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_stream.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_stream.cpp @@ -68,10 +68,10 @@ namespace NYdb::NConsoleClient { TCommandStream::TCommandStream() : TClientCommandTree("stream", {}, "DataStreams service operations") { - AddCommand(std::make_unique<TCommandStreamCreate>()); - AddCommand(std::make_unique<TCommandStreamAlter>()); - AddCommand(std::make_unique<TCommandStreamDrop>()); - AddCommand(std::make_unique<TCommandStreamConsumer>()); + AddCommand(std::make_unique<TCommandStreamCreate>()); + AddCommand(std::make_unique<TCommandStreamAlter>()); + AddCommand(std::make_unique<TCommandStreamDrop>()); + AddCommand(std::make_unique<TCommandStreamConsumer>()); } TCommandStreamCreate::TCommandStreamCreate() @@ -198,8 +198,8 @@ namespace NYdb::NConsoleClient { } TCommandStreamConsumer::TCommandStreamConsumer() : TClientCommandTree("consumer", {}, "Consumer operations") { - AddCommand(std::make_unique<TCommandStreamConsumerAdd>()); - AddCommand(std::make_unique<TCommandStreamConsumerDrop>()); + AddCommand(std::make_unique<TCommandStreamConsumerAdd>()); + AddCommand(std::make_unique<TCommandStreamConsumerDrop>()); } TCommandStreamConsumerAdd::TCommandStreamConsumerAdd() : TYdbCommand("add", {}, "Consumer add operation") {} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 2f94b2fc03..e000e55a72 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -21,48 +21,48 @@ namespace NConsoleClient { TCommandTable::TCommandTable() : TClientCommandTree("table", {}, "Table service operations") { - //AddCommand(std::make_unique<TCommandCreateTable>()); - AddCommand(std::make_unique<TCommandDropTable>()); - AddCommand(std::make_unique<TCommandQuery>()); - AddCommand(std::make_unique<TCommandReadTable>()); - AddCommand(std::make_unique<TCommandIndex>()); - AddCommand(std::make_unique<TCommandAttribute>()); - AddCommand(std::make_unique<TCommandTtl>()); + //AddCommand(std::make_unique<TCommandCreateTable>()); + AddCommand(std::make_unique<TCommandDropTable>()); + AddCommand(std::make_unique<TCommandQuery>()); + AddCommand(std::make_unique<TCommandReadTable>()); + AddCommand(std::make_unique<TCommandIndex>()); + AddCommand(std::make_unique<TCommandAttribute>()); + AddCommand(std::make_unique<TCommandTtl>()); } TCommandQuery::TCommandQuery() : TClientCommandTree("query", {}, "Query operations") { - AddCommand(std::make_unique<TCommandExecuteQuery>()); - AddCommand(std::make_unique<TCommandExplain>()); + AddCommand(std::make_unique<TCommandExecuteQuery>()); + AddCommand(std::make_unique<TCommandExplain>()); } TCommandIndex::TCommandIndex() : TClientCommandTree("index", {}, "Index operations") { - AddCommand(std::make_unique<TCommandIndexAdd>()); - AddCommand(std::make_unique<TCommandIndexDrop>()); + AddCommand(std::make_unique<TCommandIndexAdd>()); + AddCommand(std::make_unique<TCommandIndexDrop>()); } TCommandAttribute::TCommandAttribute() : TClientCommandTree("attribute", {"attr"}, "Attribute operations") { - AddCommand(std::make_unique<TCommandAttributeAdd>()); - AddCommand(std::make_unique<TCommandAttributeDrop>()); + AddCommand(std::make_unique<TCommandAttributeAdd>()); + AddCommand(std::make_unique<TCommandAttributeDrop>()); } TCommandTtl::TCommandTtl() : TClientCommandTree("ttl", {}, "Ttl operations") { - AddCommand(std::make_unique<TCommandTtlSet>()); - AddCommand(std::make_unique<TCommandTtlDrop>()); + AddCommand(std::make_unique<TCommandTtlSet>()); + AddCommand(std::make_unique<TCommandTtlDrop>()); } TCommandIndexAdd::TCommandIndexAdd() : TClientCommandTree("add", {}, "Add index in to the specified table") { - AddCommand(std::make_unique<TCommandIndexAddGlobalSync>()); - AddCommand(std::make_unique<TCommandIndexAddGlobalAsync>()); + AddCommand(std::make_unique<TCommandIndexAddGlobalSync>()); + AddCommand(std::make_unique<TCommandIndexAddGlobalAsync>()); } TTableCommand::TTableCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index db4ec14d30..496e67ec16 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -13,10 +13,10 @@ namespace NYdb::NConsoleClient { TCommandTools::TCommandTools() : TClientCommandTree("tools", {}, "YDB tools service") { - AddCommand(std::make_unique<TCommandDump>()); - AddCommand(std::make_unique<TCommandRestore>()); - AddCommand(std::make_unique<TCommandCopy>()); - AddCommand(std::make_unique<TCommandRename>()); + AddCommand(std::make_unique<TCommandDump>()); + AddCommand(std::make_unique<TCommandRestore>()); + AddCommand(std::make_unique<TCommandCopy>()); + AddCommand(std::make_unique<TCommandRename>()); } TToolsCommand::TToolsCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index ce7334d738..ef11117b09 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -253,12 +253,12 @@ TClientCommandTree::TClientCommandTree(const TString& name, const std::initializ Args[0] = "<subcommand>"; } -void TClientCommandTree::AddCommand(std::unique_ptr<TClientCommand> command) { +void TClientCommandTree::AddCommand(std::unique_ptr<TClientCommand> command) { for (const TString& a : command->Aliases) { Aliases[a] = command->Name; } command->Parent = this; - SubCommands[command->Name] = std::move(command); + SubCommands[command->Name] = std::move(command); } void TClientCommandTree::Config(TConfig& config) { @@ -302,7 +302,7 @@ void TClientCommandTree::Parse(TConfig& config) { it = SubCommands.find("*"); } if (it != SubCommands.end()) { - SelectedCommand = it->second.get(); + SelectedCommand = it->second.get(); } else { throw yexception() << "Invalid command '" << cmd << "'"; } diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index 441206cdf7..ef2eb77be9 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -325,12 +325,12 @@ private: class TClientCommandTree : public TClientCommand { public: - TMap<TString, std::unique_ptr<TClientCommand>> SubCommands; + TMap<TString, std::unique_ptr<TClientCommand>> SubCommands; TMap<TString, TString> Aliases; TClientCommand* SelectedCommand; TClientCommandTree(const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString()); - void AddCommand(std::unique_ptr<TClientCommand> command); + void AddCommand(std::unique_ptr<TClientCommand> command); virtual void Config(TConfig& config) override; virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h index ebd16c5a83..032f4514bc 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h @@ -99,7 +99,7 @@ private: } protected: - gpr_timespec Deadline_ = {}; + gpr_timespec Deadline_ = {}; private: std::mutex Mutex_; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h index 7287b69894..a3715dc51e 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h @@ -282,7 +282,7 @@ protected: // Template for visitor implementation. struct TBaseHandlersVisitor { - TBaseHandlersVisitor(const TSettings& settings, TEventInfo& eventInfo) + TBaseHandlersVisitor(const TSettings& settings, TEventInfo& eventInfo) : Settings(settings) , EventInfo(eventInfo) {} diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp index e7dd0a87e5..c06f7ea5da 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp @@ -1788,8 +1788,8 @@ void TReadSessionEventsQueue::SignalEventImpl(TIntrusivePtr<TPartitionStreamImpl if (Closed) { return; } - auto session = partitionStream->GetSession(); - Events.emplace(std::move(partitionStream), std::move(session)); + auto session = partitionStream->GetSession(); + Events.emplace(std::move(partitionStream), std::move(session)); SignalWaiterImpl(deferred); } @@ -1855,7 +1855,7 @@ bool TReadSessionEventsQueue::ApplyCallbacksToReadyEventsImpl(TDeferredActions& } const TIntrusivePtr<TPartitionStreamImpl> partitionStreamForSignalling = eventInfo->IsDataEvent() ? eventInfo->PartitionStream : nullptr; applied = true; - if (!ApplyHandler(*eventInfo, deferred)) { // Close session event. + if (!ApplyHandler(*eventInfo, deferred)) { // Close session event. break; } if (partitionStreamForSignalling) { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp index af4e08b979..6d5e21e033 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp @@ -754,7 +754,7 @@ bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) { auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); result = memoryUsage.NowOk && !memoryUsage.WasOk; //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - const auto& front = SentPackedMessage.front(); + const auto& front = SentPackedMessage.front(); if (front.Compressed) { compressedSize = front.Data.size(); } else { @@ -763,7 +763,7 @@ bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) { (*Counters->MessagesWritten) += front.MessageCount; (*Counters->MessagesInflight) -= front.MessageCount; - (*Counters->BytesWritten) += front.OriginalSize; + (*Counters->BytesWritten) += front.OriginalSize; SentPackedMessage.pop(); } else { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h index 324550bb02..0391b5aae0 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h @@ -29,7 +29,7 @@ public: {} void PushEvent(TEventInfo eventInfo) { - if (Closed || ApplyHandler(eventInfo)) { + if (Closed || ApplyHandler(eventInfo)) { return; } @@ -92,8 +92,8 @@ public: waiter = TWaiter(Waiter.ExtractPromise(), this); } - TEventInfo info(event); - ApplyHandler(info); + TEventInfo info(event); + ApplyHandler(info); waiter.Signal(); } @@ -121,8 +121,8 @@ private: }; - bool ApplyHandler(TEventInfo& eventInfo) { - THandlersVisitor visitor(Settings, eventInfo); + bool ApplyHandler(TEventInfo& eventInfo) { + THandlersVisitor visitor(Settings, eventInfo); return visitor.Visit(); } diff --git a/ydb/services/yq/ut_integration/ya.make b/ydb/services/yq/ut_integration/ya.make index bc62df4563..0b8453a366 100644 --- a/ydb/services/yq/ut_integration/ya.make +++ b/ydb/services/yq/ut_integration/ya.make @@ -21,7 +21,7 @@ PEERDIR( library/cpp/regex/pcre library/cpp/svnversion ydb/core/testlib - ydb/core/yq/libs/control_plane_storage + ydb/core/yq/libs/control_plane_storage ydb/core/yq/libs/db_resolver ydb/core/yq/libs/db_schema ydb/core/yq/libs/private_client diff --git a/ydb/services/yq/ut_integration/yq_ut.cpp b/ydb/services/yq/ut_integration/yq_ut.cpp index d4a35f195e..ba9171b0c7 100644 --- a/ydb/services/yq/ut_integration/yq_ut.cpp +++ b/ydb/services/yq/ut_integration/yq_ut.cpp @@ -7,7 +7,7 @@ #include <ydb/core/yq/libs/private_client/utils.h> #include <ydb/core/yq/libs/private_client/private_client.h> -#include <ydb/core/yq/libs/control_plane_storage/message_builders.h> +#include <ydb/core/yq/libs/control_plane_storage/message_builders.h> #include <ydb/core/yq/libs/actors/database_resolver.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> |