diff options
author | zhenyok <zhenyok@yandex-team.ru> | 2022-02-10 16:50:32 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:32 +0300 |
commit | ae11a371d50c25e7fff450f692a12cba03ceaf4f (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | b4b9936f4dce18fb9da09a1e46342c289d19663b (diff) | |
download | ydb-ae11a371d50c25e7fff450f692a12cba03ceaf4f.tar.gz |
Restoring authorship annotation for <zhenyok@yandex-team.ru>. Commit 2 of 2.
130 files changed, 6338 insertions, 6338 deletions
diff --git a/library/cpp/messagebus/message_status_counter.cpp b/library/cpp/messagebus/message_status_counter.cpp index dd417081e0..891c8f5bb2 100644 --- a/library/cpp/messagebus/message_status_counter.cpp +++ b/library/cpp/messagebus/message_status_counter.cpp @@ -5,7 +5,7 @@ #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> -#include <util/stream/str.h> +#include <util/stream/str.h> using namespace NBus; using namespace NBus::NPrivate; @@ -55,17 +55,17 @@ TString TMessageStatusCounter::PrintToString() const { return ss.Str(); } -void TMessageStatusCounter::FillErrorsProtobuf(TConnectionStatusMonRecord* status) const { - status->clear_errorcountbystatus(); - for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { - if (i == MESSAGE_OK) { +void TMessageStatusCounter::FillErrorsProtobuf(TConnectionStatusMonRecord* status) const { + status->clear_errorcountbystatus(); + for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { + if (i == MESSAGE_OK) { Y_VERIFY(Counts[i] == 0); - continue; - } - if (Counts[i] != 0) { - TMessageStatusRecord* description = status->add_errorcountbystatus(); - description->SetStatus(TMessageStatusCounter::MessageStatusToProtobuf((EMessageStatus)i)); - description->SetCount(Counts[i]); - } - } -} + continue; + } + if (Counts[i] != 0) { + TMessageStatusRecord* description = status->add_errorcountbystatus(); + description->SetStatus(TMessageStatusCounter::MessageStatusToProtobuf((EMessageStatus)i)); + description->SetCount(Counts[i]); + } + } +} diff --git a/library/cpp/messagebus/message_status_counter.h b/library/cpp/messagebus/message_status_counter.h index 5fbfca2478..e8ba2fdd31 100644 --- a/library/cpp/messagebus/message_status_counter.h +++ b/library/cpp/messagebus/message_status_counter.h @@ -3,7 +3,7 @@ #include "message_status.h" #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> - + #include <util/generic/string.h> #include <array> @@ -25,9 +25,9 @@ namespace NBus { } TMessageStatusCounter(); - + TMessageStatusCounter& operator+=(const TMessageStatusCounter&); - + TString PrintToString() const; void FillErrorsProtobuf(TConnectionStatusMonRecord*) const; }; diff --git a/library/cpp/messagebus/message_status_counter_ut.cpp b/library/cpp/messagebus/message_status_counter_ut.cpp index 10f636844d..9598651329 100644 --- a/library/cpp/messagebus/message_status_counter_ut.cpp +++ b/library/cpp/messagebus/message_status_counter_ut.cpp @@ -1,23 +1,23 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "message_status_counter.h" #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> - -using namespace NBus; -using namespace NBus::NPrivate; - + +using namespace NBus; +using namespace NBus::NPrivate; + Y_UNIT_TEST_SUITE(MessageStatusCounter) { Y_UNIT_TEST(MessageStatusConversion) { - const ::google::protobuf::EnumDescriptor* descriptor = - TMessageStatusRecord_EMessageStatus_descriptor(); - - for (int i = 0; i < MESSAGE_STATUS_COUNT; i++) { - const ::google::protobuf::EnumValueDescriptor* valueDescriptor = - descriptor->FindValueByName(ToString((EMessageStatus)i)); - UNIT_ASSERT_UNEQUAL(valueDescriptor, nullptr); - UNIT_ASSERT_EQUAL(valueDescriptor->number(), i); - } - UNIT_ASSERT_EQUAL(MESSAGE_STATUS_COUNT, descriptor->value_count()); - } -} + const ::google::protobuf::EnumDescriptor* descriptor = + TMessageStatusRecord_EMessageStatus_descriptor(); + + for (int i = 0; i < MESSAGE_STATUS_COUNT; i++) { + const ::google::protobuf::EnumValueDescriptor* valueDescriptor = + descriptor->FindValueByName(ToString((EMessageStatus)i)); + UNIT_ASSERT_UNEQUAL(valueDescriptor, nullptr); + UNIT_ASSERT_EQUAL(valueDescriptor->number(), i); + } + UNIT_ASSERT_EQUAL(MESSAGE_STATUS_COUNT, descriptor->value_count()); + } +} diff --git a/library/cpp/messagebus/monitoring/mon_proto.proto b/library/cpp/messagebus/monitoring/mon_proto.proto index 3b465a54ad..73b6614481 100644 --- a/library/cpp/messagebus/monitoring/mon_proto.proto +++ b/library/cpp/messagebus/monitoring/mon_proto.proto @@ -4,30 +4,30 @@ package NBus; option java_package = "ru.yandex.messagebus.monitoring.proto"; -message TMessageStatusRecord { - enum EMessageStatus { - MESSAGE_OK = 0; - MESSAGE_CONNECT_FAILED = 1; - MESSAGE_TIMEOUT = 2; - MESSAGE_SERVICE_UNKNOWN = 3; - MESSAGE_BUSY = 4; - MESSAGE_UNKNOWN = 5; - MESSAGE_DESERIALIZE_ERROR = 6; - MESSAGE_HEADER_CORRUPTED = 7; - MESSAGE_DECOMPRESS_ERROR = 8; - MESSAGE_MESSAGE_TOO_LARGE = 9; - MESSAGE_REPLY_FAILED = 10; - MESSAGE_DELIVERY_FAILED = 11; - MESSAGE_INVALID_VERSION = 12; - MESSAGE_SERVICE_TOOMANY = 13; - MESSAGE_SHUTDOWN = 14; - MESSAGE_DONT_ASK = 15; - } - - optional EMessageStatus Status = 1; - optional uint32 Count = 2; -} - +message TMessageStatusRecord { + enum EMessageStatus { + MESSAGE_OK = 0; + MESSAGE_CONNECT_FAILED = 1; + MESSAGE_TIMEOUT = 2; + MESSAGE_SERVICE_UNKNOWN = 3; + MESSAGE_BUSY = 4; + MESSAGE_UNKNOWN = 5; + MESSAGE_DESERIALIZE_ERROR = 6; + MESSAGE_HEADER_CORRUPTED = 7; + MESSAGE_DECOMPRESS_ERROR = 8; + MESSAGE_MESSAGE_TOO_LARGE = 9; + MESSAGE_REPLY_FAILED = 10; + MESSAGE_DELIVERY_FAILED = 11; + MESSAGE_INVALID_VERSION = 12; + MESSAGE_SERVICE_TOOMANY = 13; + MESSAGE_SHUTDOWN = 14; + MESSAGE_DONT_ASK = 15; + } + + optional EMessageStatus Status = 1; + optional uint32 Count = 2; +} + message TConnectionStatusMonRecord { optional uint32 SendQueueSize = 1 [ (NMonProto.Metric).Type = GAUGE ]; // client only @@ -44,8 +44,8 @@ message TConnectionStatusMonRecord { optional uint64 ReadMessages = 22 [ (NMonProto.Metric).Type = RATE ]; optional uint64 ReadSyscalls = 23; optional uint64 ReadActs = 24; - - repeated TMessageStatusRecord ErrorCountByStatus = 25; + + repeated TMessageStatusRecord ErrorCountByStatus = 25; } message TSessionStatusMonRecord { diff --git a/library/cpp/messagebus/remote_connection_status.cpp b/library/cpp/messagebus/remote_connection_status.cpp index 4e4e01dcf5..2c48b2a287 100644 --- a/library/cpp/messagebus/remote_connection_status.cpp +++ b/library/cpp/messagebus/remote_connection_status.cpp @@ -128,32 +128,32 @@ size_t TRemoteConnectionWriterStatus::GetInFlight() const { return SendQueueSize + AckMessagesSize; } -TConnectionStatusMonRecord TRemoteConnectionStatus::GetStatusProtobuf() const { - TConnectionStatusMonRecord status; - - // TODO: fill unfilled fields - status.SetSendQueueSize(WriterStatus.SendQueueSize); - status.SetAckMessagesSize(WriterStatus.AckMessagesSize); - // status.SetErrorCount(); - // status.SetWriteBytes(); - // status.SetWriteBytesCompressed(); - // status.SetWriteMessages(); - status.SetWriteSyscalls(WriterStatus.Incremental.NetworkOps); - status.SetWriteActs(WriterStatus.Acts); - // status.SetReadBytes(); - // status.SetReadBytesCompressed(); - // status.SetReadMessages(); - status.SetReadSyscalls(ReaderStatus.Incremental.NetworkOps); - status.SetReadActs(ReaderStatus.Acts); - - TMessageStatusCounter sumStatusCounter; - sumStatusCounter += WriterStatus.Incremental.StatusCounter; - sumStatusCounter += ReaderStatus.Incremental.StatusCounter; - sumStatusCounter.FillErrorsProtobuf(&status); - - return status; -} - +TConnectionStatusMonRecord TRemoteConnectionStatus::GetStatusProtobuf() const { + TConnectionStatusMonRecord status; + + // TODO: fill unfilled fields + status.SetSendQueueSize(WriterStatus.SendQueueSize); + status.SetAckMessagesSize(WriterStatus.AckMessagesSize); + // status.SetErrorCount(); + // status.SetWriteBytes(); + // status.SetWriteBytesCompressed(); + // status.SetWriteMessages(); + status.SetWriteSyscalls(WriterStatus.Incremental.NetworkOps); + status.SetWriteActs(WriterStatus.Acts); + // status.SetReadBytes(); + // status.SetReadBytesCompressed(); + // status.SetReadMessages(); + status.SetReadSyscalls(ReaderStatus.Incremental.NetworkOps); + status.SetReadActs(ReaderStatus.Acts); + + TMessageStatusCounter sumStatusCounter; + sumStatusCounter += WriterStatus.Incremental.StatusCounter; + sumStatusCounter += ReaderStatus.Incremental.StatusCounter; + sumStatusCounter.FillErrorsProtobuf(&status); + + return status; +} + TString TRemoteConnectionStatus::PrintToString() const { TStringStream ss; diff --git a/library/cpp/messagebus/remote_connection_status.h b/library/cpp/messagebus/remote_connection_status.h index 46d83ba4cd..5db10e51ea 100644 --- a/library/cpp/messagebus/remote_connection_status.h +++ b/library/cpp/messagebus/remote_connection_status.h @@ -11,10 +11,10 @@ #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -namespace NBus { - class TConnectionStatusMonRecord; -} - +namespace NBus { + class TConnectionStatusMonRecord; +} + namespace NBus { namespace NPrivate { #define WRITER_STATE_MAP(XX) \ diff --git a/library/cpp/messagebus/session_impl.cpp b/library/cpp/messagebus/session_impl.cpp index 7d70b9171f..ddf9f360c4 100644 --- a/library/cpp/messagebus/session_impl.cpp +++ b/library/cpp/messagebus/session_impl.cpp @@ -250,12 +250,12 @@ TString TBusSessionImpl::GetStatus(ui16 flags) { TConnectionStatusMonRecord TBusSessionImpl::GetStatusProtobuf() { Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), "GetStatus must not be called from executor thread"); - - TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); - - return StatusData.StatusDumpCached.ConnectionStatusSummary.GetStatusProtobuf(); -} - + + TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); + + return StatusData.StatusDumpCached.ConnectionStatusSummary.GetStatusProtobuf(); +} + TString TBusSessionImpl::GetStatusSingleLine() { TSessionDumpStatus status = GetStatusRecordInternal(); diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 95da4576bb..f5fedfe19b 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -128,15 +128,15 @@ struct TKikimrEvents : TEvents { ES_SCHEME_BOARD_MON, ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY, ES_BLOB_CACHE, - ES_LONG_TX_SERVICE, - ES_TEST_SHARD, - ES_DATASTREAMS_PROXY, - ES_IAM_TOKEN_SERVICE, + ES_LONG_TX_SERVICE, + ES_TEST_SHARD, + ES_DATASTREAMS_PROXY, + ES_IAM_TOKEN_SERVICE, ES_HEALTH_CHECK, ES_DQ = NYql::NDq::TDqEvents::ES_DQ_COMPUTE, // 4212 ES_YQ, // 4213 ES_CHANGE_EXCHANGE, - ES_DATABASE_SERVICE, //4215 + ES_DATABASE_SERVICE, //4215 ES_SEQUENCESHARD, // 4216 ES_SEQUENCEPROXY, // 4217 ES_CLOUD_STORAGE, diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index e9e5ab38da..237bba147b 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -166,7 +166,7 @@ protected: request->Record.MutableTabletConfig()->SetCacheSize(10*1024*1024); request->Record.SetTxId(12345); auto tabletConfig = request->Record.MutableTabletConfig(); - tabletConfig->SetTopicName(topic); + tabletConfig->SetTopicName(topic); tabletConfig->SetVersion(version); auto config = tabletConfig->MutablePartitionConfig(); config->SetMaxCountInPartition(20000000); diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 970ae9bd5a..450d4135c6 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -7,7 +7,7 @@ #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> @@ -805,7 +805,7 @@ public: class TClientCommandSchemaTableOptions : public TClientCommand { public: - NGrpc::TGRpcClientConfig ClientConfig; + NGrpc::TGRpcClientConfig ClientConfig; TClientCommandSchemaTableOptions() : TClientCommand("options", {}, "Describe table options") @@ -923,7 +923,7 @@ public: class TClientCommandSchemaTableCopy : public TClientCommand { public: - NGrpc::TGRpcClientConfig ClientConfig; + NGrpc::TGRpcClientConfig ClientConfig; TString DatabaseName; TVector<TString> SrcValues; TVector<TString> DstValues; diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp index c5b3928b71..197eccfcfd 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp @@ -119,7 +119,7 @@ public: if (!hostname) { NMsgBusProxy::TMsgBusClientConfig::CrackAddress(Address, hostname, port); } - CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(hostname + ':' + ToString(port)); + CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(hostname + ':' + ToString(port)); if (config.EnableSsl) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); p->EnableSsl = config.EnableSsl; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp index 3ee8c33130..fcc9305996 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp @@ -27,7 +27,7 @@ namespace NDriverClient { TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(Address); switch (endpoint.ServerType) { case TCommandConfig::EServerType::GRpc: - ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); + ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); if (endpoint.EnableSsl.Defined()) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&ClientConfig.GetRef()); p->EnableSsl = endpoint.EnableSsl.GetRef(); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp index 36b4176952..ec049fab11 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp @@ -56,7 +56,7 @@ public: switch (endpoint.ServerType) { case TCommandConfig::EServerType::GRpc: - CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); + CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); if (config.EnableSsl) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); p->EnableSsl = config.EnableSsl; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 917da9cc37..6766dd3171 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -1098,7 +1098,7 @@ protected: private: NClient::TKikimr GetKikimr(const TString& addr) { TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(addr); - NGrpc::TGRpcClientConfig grpcConfig(endpoint.Address, TDuration::Seconds(5)); + NGrpc::TGRpcClientConfig grpcConfig(endpoint.Address, TDuration::Seconds(5)); grpcConfig.LoadBalancingPolicy = "round_robin"; if (endpoint.EnableSsl.Defined()) { grpcConfig.EnableSsl = endpoint.EnableSsl.GetRef(); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp index b921e0ddc7..db44a3a516 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp @@ -1,7 +1,7 @@ #include "cli.h" #include "cli_cmds.h" - + #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> @@ -121,7 +121,7 @@ template <typename TService, typename TRequest, typename TResponse, class TTenantClientGRpcCommand : public TTenantClientCommand { public: TRequest GRpcRequest; - NGrpc::TGRpcClientConfig ClientConfig; + NGrpc::TGRpcClientConfig ClientConfig; TTenantClientGRpcCommand(const TString &name, const std::initializer_list<TString> &aliases, diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 175521920b..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -531,7 +531,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["import"] = &hasImport; bool hasAnalytics = false; names["analytics"] = &hasAnalytics; - bool hasDataStreams = false; + bool hasDataStreams = false; names["datastreams"] = &hasDataStreams; bool hasYandexQuery = false; names["yq"] = &hasYandexQuery; @@ -709,9 +709,9 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { server.AddService(new NGRpcService::TGRpcAuthService(ActorSystem.Get(), Counters, grpcRequestProxyId)); } - if (hasDataStreams) { - server.AddService(new NGRpcService::TGRpcDataStreamsService(ActorSystem.Get(), Counters, grpcRequestProxyId)); - } + if (hasDataStreams) { + server.AddService(new NGRpcService::TGRpcDataStreamsService(ActorSystem.Get(), Counters, grpcRequestProxyId)); + } if (hasYandexQuery) { server.AddService(new NGRpcService::TGRpcYandexQueryService(ActorSystem.Get(), Counters, grpcRequestProxyId)); diff --git a/ydb/core/grpc_caching/cache_item.h b/ydb/core/grpc_caching/cache_item.h index 2edae85fb2..4e22d34f7f 100644 --- a/ydb/core/grpc_caching/cache_item.h +++ b/ydb/core/grpc_caching/cache_item.h @@ -1,75 +1,75 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/grpc/client/grpc_client_low.h> - -namespace NKikimr { - - template <typename TRequestEvent, typename TResponseEvent, typename TPrivateResponseEvent> - struct TCacheItem : TThrRefBase { - static constexpr TDuration SuccessLifeTime = TDuration::Hours(1); - static constexpr TDuration ErrorLifeTime = TDuration::Minutes(1); - - using TRequestRecord = decltype(TRequestEvent().Request); - using TResponseRecord = decltype(TResponseEvent().Response); - TString CacheKey; - TInstant UpdateTimestamp; - TResponseRecord Response; - NGrpc::TGrpcStatus Status; - TVector<THolder<NActors::IEventHandle>> Waiters; - TRequestRecord* Request = nullptr; - TString RequestId; - - TCacheItem(const TString& key) - : CacheKey(key) - {} - - const TRequestRecord& GetRequest() const { - Y_VERIFY(Request != nullptr); - return *Request; - } - - bool IsTemporaryErrorStatus() const { - return Status.InternalError - || Status.GRpcStatusCode == grpc::StatusCode::UNKNOWN - || Status.GRpcStatusCode == grpc::StatusCode::DEADLINE_EXCEEDED - || Status.GRpcStatusCode == grpc::StatusCode::INTERNAL - || Status.GRpcStatusCode == grpc::StatusCode::UNAVAILABLE; - } - - bool IsValid(TInstant now) const { - if (UpdateTimestamp != TInstant()) { - if (!Status.Ok()) { - return !IsTemporaryErrorStatus() && (now - UpdateTimestamp) < ErrorLifeTime; - } else { - return (now - UpdateTimestamp) < SuccessLifeTime; - } - } else { - return false; - } - } - - void Respond(THolder<NActors::IEventHandle>&& request, const NActors::TActorContext& ctx) const { - TResponseEvent* result = new TResponseEvent(); - result->Response = Response; - result->Status = Status; - result->Request = std::move(request); - ctx.Send(result->Request->Sender, result, 0, result->Request->Cookie); - } - - void Update(TPrivateResponseEvent&& result, const NActors::TActorContext& ctx) { - Response = std::move(result.Response); - Status = std::move(result.Status); - UpdateTimestamp = ctx.Now(); - for (THolder<NActors::IEventHandle>& request : Waiters) { - Respond(std::move(request), ctx); - } - Waiters.clear(); - } - }; - -} - - +#pragma once + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/grpc/client/grpc_client_low.h> + +namespace NKikimr { + + template <typename TRequestEvent, typename TResponseEvent, typename TPrivateResponseEvent> + struct TCacheItem : TThrRefBase { + static constexpr TDuration SuccessLifeTime = TDuration::Hours(1); + static constexpr TDuration ErrorLifeTime = TDuration::Minutes(1); + + using TRequestRecord = decltype(TRequestEvent().Request); + using TResponseRecord = decltype(TResponseEvent().Response); + TString CacheKey; + TInstant UpdateTimestamp; + TResponseRecord Response; + NGrpc::TGrpcStatus Status; + TVector<THolder<NActors::IEventHandle>> Waiters; + TRequestRecord* Request = nullptr; + TString RequestId; + + TCacheItem(const TString& key) + : CacheKey(key) + {} + + const TRequestRecord& GetRequest() const { + Y_VERIFY(Request != nullptr); + return *Request; + } + + bool IsTemporaryErrorStatus() const { + return Status.InternalError + || Status.GRpcStatusCode == grpc::StatusCode::UNKNOWN + || Status.GRpcStatusCode == grpc::StatusCode::DEADLINE_EXCEEDED + || Status.GRpcStatusCode == grpc::StatusCode::INTERNAL + || Status.GRpcStatusCode == grpc::StatusCode::UNAVAILABLE; + } + + bool IsValid(TInstant now) const { + if (UpdateTimestamp != TInstant()) { + if (!Status.Ok()) { + return !IsTemporaryErrorStatus() && (now - UpdateTimestamp) < ErrorLifeTime; + } else { + return (now - UpdateTimestamp) < SuccessLifeTime; + } + } else { + return false; + } + } + + void Respond(THolder<NActors::IEventHandle>&& request, const NActors::TActorContext& ctx) const { + TResponseEvent* result = new TResponseEvent(); + result->Response = Response; + result->Status = Status; + result->Request = std::move(request); + ctx.Send(result->Request->Sender, result, 0, result->Request->Cookie); + } + + void Update(TPrivateResponseEvent&& result, const NActors::TActorContext& ctx) { + Response = std::move(result.Response); + Status = std::move(result.Status); + UpdateTimestamp = ctx.Now(); + for (THolder<NActors::IEventHandle>& request : Waiters) { + Respond(std::move(request), ctx); + } + Waiters.clear(); + } + }; + +} + + diff --git a/ydb/core/grpc_caching/cached_grpc_request_actor.cpp b/ydb/core/grpc_caching/cached_grpc_request_actor.cpp index b56bedf673..38556b085d 100644 --- a/ydb/core/grpc_caching/cached_grpc_request_actor.cpp +++ b/ydb/core/grpc_caching/cached_grpc_request_actor.cpp @@ -1,3 +1,3 @@ -#include "cached_grpc_request_actor.h" - - +#include "cached_grpc_request_actor.h" + + diff --git a/ydb/core/grpc_caching/cached_grpc_request_actor.h b/ydb/core/grpc_caching/cached_grpc_request_actor.h index 15a8cffea5..d9e569ede2 100644 --- a/ydb/core/grpc_caching/cached_grpc_request_actor.h +++ b/ydb/core/grpc_caching/cached_grpc_request_actor.h @@ -1,232 +1,232 @@ -#pragma once - -#include "cache_item.h" -#include "request_data.h" - +#pragma once + +#include "cache_item.h" +#include "request_data.h" + #include <ydb/core/base/defs.h> #include <ydb/core/base/events.h> #include <ydb/core/util/simple_cache.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/log.h> -#include <library/cpp/grpc/client/grpc_client_low.h> - -#include <util/datetime/base.h> - -namespace NKikimr { - - namespace NPrivate { - - template<ui32 RequestTag, ui32 ResponseTag, ui32 ResultTag, class TProtoService, class TProtoRequest, class TProtoResponse, class TRequestData> - struct TGrpcRequestActorTypes { - - struct TEvRequest; - struct TEvResponse; - struct TEvCachedResult; - - using TCachedItemPtr = TIntrusivePtr<typename TEvCachedResult::TCachedItem>; - using TEvRequestType = TEvRequest; - using TEvResponseType = TEvResponse; - using TEvResultType = TEvCachedResult; - using TServiceConnection = NGrpc::TServiceConnection<TProtoService>; - using TGrpcRequestType = typename NGrpc::TSimpleRequestProcessor<typename TProtoService::Stub, TProtoRequest, TProtoResponse>::TAsyncRequest; - - struct TEvRequest : TEventLocal<TEvRequest, RequestTag> { - TProtoRequest Request; - TRequestData Data; - - TEvRequest() = default; - TEvRequest(TRequestData data, TProtoRequest request) - : Request(std::move(request)), Data(std::move(data)) { - } - TEvRequest(TProtoRequest request) - : Request(std::move(request)) { - } - - TString GetCacheKey() const { return Request.ShortDebugString(); } - }; - - struct TEvResponse : TEventLocal<TEvResponse, ResponseTag> { - THolder <IEventHandle> Request; - TProtoResponse Response; - NGrpc::TGrpcStatus Status; - }; - - struct TEvCachedResult : TEventLocal<TEvCachedResult, ResultTag> { - struct TCachedItem : public TCacheItem<TEvRequest, TEvResponse, TEvCachedResult> { - using TBase = TCacheItem<TEvRequest, TEvResponse, TEvCachedResult>; - TRequestData Data; - - TCachedItem(const TString& key) - : TBase(key) - {} - }; - - TIntrusivePtr<TCachedItem> CacheItem; - TProtoResponse Response; - NGrpc::TGrpcStatus Status; - - TEvCachedResult(TIntrusivePtr<TCachedItem> cacheItem) - : CacheItem(cacheItem) {} - }; - - }; - - } - - template<ui32 RequestTag, ui32 ResponseTag, ui32 ResultTag, class TProtoService, class TProtoRequest, class TProtoResponse, class TRequestData> - struct TCachedRequestProcessor { - - class TGrpcRequestActor; - - using TTypes = NPrivate::TGrpcRequestActorTypes<RequestTag, ResponseTag, ResultTag, TProtoService, TProtoRequest, TProtoResponse, TRequestData>; - - using TService = TProtoService; - using TRequest = TProtoRequest; - using TResponse = TProtoResponse; - using TCachedItem = typename TTypes::TEvCachedResult::TCachedItem; - using TCachedItemPtr = typename TTypes::TCachedItemPtr; - using TEvRequestType = typename TTypes::TEvRequest; - using TEvResponseType = typename TTypes::TEvResponse; - using TEvResultType = typename TTypes::TEvCachedResult; - using TServiceConnection = NGrpc::TServiceConnection<TService>; - using TGrpcRequestType = typename NGrpc::TSimpleRequestProcessor<typename TService::Stub, TRequest, TResponse>::TAsyncRequest; - - struct TEvPrivate { - enum EEv { - EvResponse = EventSpaceBegin(TEvents::ES_PRIVATE), - EvError, - EvEnd - }; - - struct TEvResponse : TEventLocal<TEvResponse, EvResponse> { - TResponse Response; - - TEvResponse(TResponse&& response) - : Response(std::move(response)) - {} - }; - - struct TEvError : TEventLocal<TEvError, EvError> { - NGrpc::TGrpcStatus Status; - - TEvError(NGrpc::TGrpcStatus&& status) - : Status(std::move(status)) - {} - }; - - static_assert(EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE)"); - }; - - TSimpleCache<TString, typename TTypes::TCachedItemPtr> Cache; - std::shared_ptr<TServiceConnection> GrpcConnection; - NGrpc::IQueueClientContextProvider& GrpcContextProvider; - TGrpcRequestType GrpcProcessor; - - TCachedRequestProcessor(std::shared_ptr<TServiceConnection> connection, NGrpc::IQueueClientContextProvider& contextProvider, TGrpcRequestType processor) - : GrpcConnection(connection) - , GrpcContextProvider(contextProvider) - , GrpcProcessor(processor) - {} - - class TGrpcRequestActor : public TActorBootstrapped<TGrpcRequestActor> { - private: - using TThis = TGrpcRequestActor; - using TBase = TActorBootstrapped<TGrpcRequestActor>; - - TActorId Sender; - TCachedItemPtr CacheItem; - std::shared_ptr<TServiceConnection> Connection; - std::shared_ptr<NGrpc::IQueueClientContext> Context; - TGrpcRequestType GrpcProcessor; - - public: - TGrpcRequestActor( - std::shared_ptr<TServiceConnection> connection, - std::shared_ptr<NGrpc::IQueueClientContext> context, - const TActorId& sender, - TCachedItemPtr item, - TGrpcRequestType processor - ) - : Sender(sender) - , CacheItem(item) - , Connection(connection) - , Context(context) - , GrpcProcessor(processor) - {} - - void Bootstrap(const TActorContext& ctx) { - TActorId actorId = ctx.SelfID; - TActorSystem* actorSystem = ctx.ExecutorThread.ActorSystem; - - NGrpc::TResponseCallback<TResponse> responseCb = - [actorId, actorSystem](NGrpc::TGrpcStatus&& status, TResponse&& response) -> void { - if (status.Ok()) { - actorSystem->Send(actorId, new typename TEvPrivate::TEvResponse(std::move(response))); - } else { - actorSystem->Send(actorId, new typename TEvPrivate::TEvError(std::move(status))); - } - }; - - auto callMeta = CacheItem->Data.FillCallMeta(); - Connection->DoRequest(CacheItem->GetRequest(), std::move(responseCb), GrpcProcessor, callMeta); - TBase::Become(&TThis::StateWork, DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT * 2, new TEvents::TEvWakeup()); - } - - void Handle(typename TEvPrivate::TEvResponse::TPtr& ev, const TActorContext& ctx) { - THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); - result->Response = std::move(ev->Get()->Response); - ctx.Send(Sender, result.Release()); - TBase::Die(ctx); - } - - void Handle(typename TEvPrivate::TEvError::TPtr& ev, const TActorContext& ctx) { - THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); - result->Status = ev->Get()->Status; - ctx.Send(Sender, result.Release()); - TBase::Die(ctx); - } - - void HandleTimeout(const TActorContext& ctx) { - THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); - result->Status = NGrpc::TGrpcStatus("Timeout", grpc::StatusCode::DEADLINE_EXCEEDED, true); - ctx.Send(Sender, result.Release()); - TBase::Die(ctx); - } - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvPrivate::TEvResponse, Handle); - HFunc(TEvPrivate::TEvError, Handle); - CFunc(TEvents::TSystem::Wakeup, HandleTimeout); - CFunc(TEvents::TSystem::PoisonPill, TBase::Die); - } - } - }; - - void HandleRequest(typename TEvRequestType::TPtr& ev, const TActorContext& ctx) { - TString key = ev->Get()->GetCacheKey(); - auto item = Cache.Update(key); - if (item != nullptr && item->IsValid(ctx.Now())) { - item->Respond(ev, ctx); - return; - } - ev->Get()->Data.FillCachedRequestData(item ? TMaybe<TRequestData>(item->Data) : Nothing()); - if (item == nullptr) { - item = Cache.Update(key, new TCachedItem(key)); - item->Data = ev->Get()->Data; - } - if (item->Waiters.empty()) { - item->Request = &ev->Get()->Request; - ctx.Register(new TGrpcRequestActor(GrpcConnection, GrpcContextProvider.CreateContext(), ctx.SelfID, item, GrpcProcessor)); - } - item->Waiters.emplace_back(std::move(ev)); - } - - void HandleResponse(typename TEvResultType::TPtr ev, const TActorContext& ctx) { - ev->Get()->CacheItem->Update(std::move(*ev->Get()), ctx); - } - }; - -} + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/log.h> +#include <library/cpp/grpc/client/grpc_client_low.h> + +#include <util/datetime/base.h> + +namespace NKikimr { + + namespace NPrivate { + + template<ui32 RequestTag, ui32 ResponseTag, ui32 ResultTag, class TProtoService, class TProtoRequest, class TProtoResponse, class TRequestData> + struct TGrpcRequestActorTypes { + + struct TEvRequest; + struct TEvResponse; + struct TEvCachedResult; + + using TCachedItemPtr = TIntrusivePtr<typename TEvCachedResult::TCachedItem>; + using TEvRequestType = TEvRequest; + using TEvResponseType = TEvResponse; + using TEvResultType = TEvCachedResult; + using TServiceConnection = NGrpc::TServiceConnection<TProtoService>; + using TGrpcRequestType = typename NGrpc::TSimpleRequestProcessor<typename TProtoService::Stub, TProtoRequest, TProtoResponse>::TAsyncRequest; + + struct TEvRequest : TEventLocal<TEvRequest, RequestTag> { + TProtoRequest Request; + TRequestData Data; + + TEvRequest() = default; + TEvRequest(TRequestData data, TProtoRequest request) + : Request(std::move(request)), Data(std::move(data)) { + } + TEvRequest(TProtoRequest request) + : Request(std::move(request)) { + } + + TString GetCacheKey() const { return Request.ShortDebugString(); } + }; + + struct TEvResponse : TEventLocal<TEvResponse, ResponseTag> { + THolder <IEventHandle> Request; + TProtoResponse Response; + NGrpc::TGrpcStatus Status; + }; + + struct TEvCachedResult : TEventLocal<TEvCachedResult, ResultTag> { + struct TCachedItem : public TCacheItem<TEvRequest, TEvResponse, TEvCachedResult> { + using TBase = TCacheItem<TEvRequest, TEvResponse, TEvCachedResult>; + TRequestData Data; + + TCachedItem(const TString& key) + : TBase(key) + {} + }; + + TIntrusivePtr<TCachedItem> CacheItem; + TProtoResponse Response; + NGrpc::TGrpcStatus Status; + + TEvCachedResult(TIntrusivePtr<TCachedItem> cacheItem) + : CacheItem(cacheItem) {} + }; + + }; + + } + + template<ui32 RequestTag, ui32 ResponseTag, ui32 ResultTag, class TProtoService, class TProtoRequest, class TProtoResponse, class TRequestData> + struct TCachedRequestProcessor { + + class TGrpcRequestActor; + + using TTypes = NPrivate::TGrpcRequestActorTypes<RequestTag, ResponseTag, ResultTag, TProtoService, TProtoRequest, TProtoResponse, TRequestData>; + + using TService = TProtoService; + using TRequest = TProtoRequest; + using TResponse = TProtoResponse; + using TCachedItem = typename TTypes::TEvCachedResult::TCachedItem; + using TCachedItemPtr = typename TTypes::TCachedItemPtr; + using TEvRequestType = typename TTypes::TEvRequest; + using TEvResponseType = typename TTypes::TEvResponse; + using TEvResultType = typename TTypes::TEvCachedResult; + using TServiceConnection = NGrpc::TServiceConnection<TService>; + using TGrpcRequestType = typename NGrpc::TSimpleRequestProcessor<typename TService::Stub, TRequest, TResponse>::TAsyncRequest; + + struct TEvPrivate { + enum EEv { + EvResponse = EventSpaceBegin(TEvents::ES_PRIVATE), + EvError, + EvEnd + }; + + struct TEvResponse : TEventLocal<TEvResponse, EvResponse> { + TResponse Response; + + TEvResponse(TResponse&& response) + : Response(std::move(response)) + {} + }; + + struct TEvError : TEventLocal<TEvError, EvError> { + NGrpc::TGrpcStatus Status; + + TEvError(NGrpc::TGrpcStatus&& status) + : Status(std::move(status)) + {} + }; + + static_assert(EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE)"); + }; + + TSimpleCache<TString, typename TTypes::TCachedItemPtr> Cache; + std::shared_ptr<TServiceConnection> GrpcConnection; + NGrpc::IQueueClientContextProvider& GrpcContextProvider; + TGrpcRequestType GrpcProcessor; + + TCachedRequestProcessor(std::shared_ptr<TServiceConnection> connection, NGrpc::IQueueClientContextProvider& contextProvider, TGrpcRequestType processor) + : GrpcConnection(connection) + , GrpcContextProvider(contextProvider) + , GrpcProcessor(processor) + {} + + class TGrpcRequestActor : public TActorBootstrapped<TGrpcRequestActor> { + private: + using TThis = TGrpcRequestActor; + using TBase = TActorBootstrapped<TGrpcRequestActor>; + + TActorId Sender; + TCachedItemPtr CacheItem; + std::shared_ptr<TServiceConnection> Connection; + std::shared_ptr<NGrpc::IQueueClientContext> Context; + TGrpcRequestType GrpcProcessor; + + public: + TGrpcRequestActor( + std::shared_ptr<TServiceConnection> connection, + std::shared_ptr<NGrpc::IQueueClientContext> context, + const TActorId& sender, + TCachedItemPtr item, + TGrpcRequestType processor + ) + : Sender(sender) + , CacheItem(item) + , Connection(connection) + , Context(context) + , GrpcProcessor(processor) + {} + + void Bootstrap(const TActorContext& ctx) { + TActorId actorId = ctx.SelfID; + TActorSystem* actorSystem = ctx.ExecutorThread.ActorSystem; + + NGrpc::TResponseCallback<TResponse> responseCb = + [actorId, actorSystem](NGrpc::TGrpcStatus&& status, TResponse&& response) -> void { + if (status.Ok()) { + actorSystem->Send(actorId, new typename TEvPrivate::TEvResponse(std::move(response))); + } else { + actorSystem->Send(actorId, new typename TEvPrivate::TEvError(std::move(status))); + } + }; + + auto callMeta = CacheItem->Data.FillCallMeta(); + Connection->DoRequest(CacheItem->GetRequest(), std::move(responseCb), GrpcProcessor, callMeta); + TBase::Become(&TThis::StateWork, DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT * 2, new TEvents::TEvWakeup()); + } + + void Handle(typename TEvPrivate::TEvResponse::TPtr& ev, const TActorContext& ctx) { + THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); + result->Response = std::move(ev->Get()->Response); + ctx.Send(Sender, result.Release()); + TBase::Die(ctx); + } + + void Handle(typename TEvPrivate::TEvError::TPtr& ev, const TActorContext& ctx) { + THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); + result->Status = ev->Get()->Status; + ctx.Send(Sender, result.Release()); + TBase::Die(ctx); + } + + void HandleTimeout(const TActorContext& ctx) { + THolder<TEvResultType> result = MakeHolder<TEvResultType>(CacheItem); + result->Status = NGrpc::TGrpcStatus("Timeout", grpc::StatusCode::DEADLINE_EXCEEDED, true); + ctx.Send(Sender, result.Release()); + TBase::Die(ctx); + } + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvPrivate::TEvResponse, Handle); + HFunc(TEvPrivate::TEvError, Handle); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + CFunc(TEvents::TSystem::PoisonPill, TBase::Die); + } + } + }; + + void HandleRequest(typename TEvRequestType::TPtr& ev, const TActorContext& ctx) { + TString key = ev->Get()->GetCacheKey(); + auto item = Cache.Update(key); + if (item != nullptr && item->IsValid(ctx.Now())) { + item->Respond(ev, ctx); + return; + } + ev->Get()->Data.FillCachedRequestData(item ? TMaybe<TRequestData>(item->Data) : Nothing()); + if (item == nullptr) { + item = Cache.Update(key, new TCachedItem(key)); + item->Data = ev->Get()->Data; + } + if (item->Waiters.empty()) { + item->Request = &ev->Get()->Request; + ctx.Register(new TGrpcRequestActor(GrpcConnection, GrpcContextProvider.CreateContext(), ctx.SelfID, item, GrpcProcessor)); + } + item->Waiters.emplace_back(std::move(ev)); + } + + void HandleResponse(typename TEvResultType::TPtr ev, const TActorContext& ctx) { + ev->Get()->CacheItem->Update(std::move(*ev->Get()), ctx); + } + }; + +} diff --git a/ydb/core/grpc_caching/request_data.h b/ydb/core/grpc_caching/request_data.h index 36ce2e32c4..248c55093d 100644 --- a/ydb/core/grpc_caching/request_data.h +++ b/ydb/core/grpc_caching/request_data.h @@ -1,44 +1,44 @@ -#pragma once - -#include <util/generic/guid.h> - -namespace NKikimr { - - constexpr TDuration DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT = TDuration::Seconds(10); - const TString REQUEST_ID_METADATA_NAME = "x-request-id"; - - struct TRequestIdData { - TString RequestId; - - NGrpc::TCallMeta FillCallMeta() const { - NGrpc::TCallMeta callMeta; - callMeta.Timeout = DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT; - callMeta.Aux.emplace_back(REQUEST_ID_METADATA_NAME, RequestId); - return callMeta; - } - - void FillCachedRequestData(TMaybe<TRequestIdData> cached) { - if (RequestId.empty()) { - RequestId = cached.Defined() ? cached->RequestId : CreateGuidAsString(); - } - } - - }; - - struct TTicketData { - TString Ticket; - - NGrpc::TCallMeta FillCallMeta() const { - NGrpc::TCallMeta callMeta; - callMeta.Timeout = DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT; - callMeta.Aux.emplace_back("authorization", "Bearer " + Ticket); - return callMeta; - } - - void FillCachedRequestData(TMaybe<TTicketData> cached) { - Y_UNUSED(cached); - } - - }; - -}
\ No newline at end of file +#pragma once + +#include <util/generic/guid.h> + +namespace NKikimr { + + constexpr TDuration DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT = TDuration::Seconds(10); + const TString REQUEST_ID_METADATA_NAME = "x-request-id"; + + struct TRequestIdData { + TString RequestId; + + NGrpc::TCallMeta FillCallMeta() const { + NGrpc::TCallMeta callMeta; + callMeta.Timeout = DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT; + callMeta.Aux.emplace_back(REQUEST_ID_METADATA_NAME, RequestId); + return callMeta; + } + + void FillCachedRequestData(TMaybe<TRequestIdData> cached) { + if (RequestId.empty()) { + RequestId = cached.Defined() ? cached->RequestId : CreateGuidAsString(); + } + } + + }; + + struct TTicketData { + TString Ticket; + + NGrpc::TCallMeta FillCallMeta() const { + NGrpc::TCallMeta callMeta; + callMeta.Timeout = DEFAULT_CACHED_GRPC_REQUEST_TIMEOUT; + callMeta.Aux.emplace_back("authorization", "Bearer " + Ticket); + return callMeta; + } + + void FillCachedRequestData(TMaybe<TTicketData> cached) { + Y_UNUSED(cached); + } + + }; + +}
\ No newline at end of file diff --git a/ydb/core/grpc_caching/ya.make b/ydb/core/grpc_caching/ya.make index b4c5ae9bd4..4439f33cd0 100644 --- a/ydb/core/grpc_caching/ya.make +++ b/ydb/core/grpc_caching/ya.make @@ -1,17 +1,17 @@ -OWNER( - zhenyok - g:kikimr -) - -LIBRARY() - -SRCS( - cached_grpc_request_actor.cpp -) - -PEERDIR( - library/cpp/actors/protos +OWNER( + zhenyok + g:kikimr +) + +LIBRARY() + +SRCS( + cached_grpc_request_actor.cpp +) + +PEERDIR( + library/cpp/actors/protos ydb/core/base -) - -END() +) + +END() diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index a79fc69948..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -124,8 +124,8 @@ struct TRpcServices { EvPQCreateTopic, EvPQAlterTopic, EvPQDescribeTopic, - EvPQAddReadRule, - EvPQRemoveReadRule, + EvPQAddReadRule, + EvPQRemoveReadRule, EvGetDiskSpaceUsage, EvStopServingDatabase, EvCoordinationSession, @@ -138,35 +138,35 @@ struct TRpcServices { EvExplainYqlScript, EvImportData, EvAnalyticsReserved, - EvDataStreamsCreateStream, - EvDataStreamsDeleteStream, - EvDataStreamsDescribeStream, - EvDataStreamsRegisterStreamConsumer, - EvDataStreamsDeregisterStreamConsumer, - EvDataStreamsDescribeStreamConsumer, - EvDataStreamsListStreams, - EvDataStreamsListShards, - EvDataStreamsPutRecord, - EvDataStreamsPutRecords, - EvDataStreamsGetRecords, - EvDataStreamsGetShardIterator, - EvDataStreamsSubscribeToShard, - EvDataStreamsDescribeLimits, - EvDataStreamsDescribeStreamSummary, - EvDataStreamsDecreaseStreamRetentionPeriod, - EvDataStreamsIncreaseStreamRetentionPeriod, - EvDataStreamsUpdateShardCount, - EvDataStreamsUpdateStream, - EvDataStreamsSetWriteQuota, - EvDataStreamsListStreamConsumers, - EvDataStreamsAddTagsToStream, - EvDataStreamsDisableEnhancedMonitoring, - EvDataStreamsEnableEnhancedMonitoring, - EvDataStreamsListTagsForStream, - EvDataStreamsMergeShards, - EvDataStreamsRemoveTagsFromStream, - EvDataStreamsSplitShard, - EvDataStreamsStartStreamEncryption, + EvDataStreamsCreateStream, + EvDataStreamsDeleteStream, + EvDataStreamsDescribeStream, + EvDataStreamsRegisterStreamConsumer, + EvDataStreamsDeregisterStreamConsumer, + EvDataStreamsDescribeStreamConsumer, + EvDataStreamsListStreams, + EvDataStreamsListShards, + EvDataStreamsPutRecord, + EvDataStreamsPutRecords, + EvDataStreamsGetRecords, + EvDataStreamsGetShardIterator, + EvDataStreamsSubscribeToShard, + EvDataStreamsDescribeLimits, + EvDataStreamsDescribeStreamSummary, + EvDataStreamsDecreaseStreamRetentionPeriod, + EvDataStreamsIncreaseStreamRetentionPeriod, + EvDataStreamsUpdateShardCount, + EvDataStreamsUpdateStream, + EvDataStreamsSetWriteQuota, + EvDataStreamsListStreamConsumers, + EvDataStreamsAddTagsToStream, + EvDataStreamsDisableEnhancedMonitoring, + EvDataStreamsEnableEnhancedMonitoring, + EvDataStreamsListTagsForStream, + EvDataStreamsMergeShards, + EvDataStreamsRemoveTagsFromStream, + EvDataStreamsSplitShard, + EvDataStreamsStartStreamEncryption, EvDataStreamsStopStreamEncryption, EvStreamExecuteYqlScript, EvYandexQueryCreateQuery, diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index 0eb213e709..97315f6e9f 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -93,8 +93,8 @@ protected: void Handle(TEvPQDropTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQCreateTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQAlterTopicRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQDescribeTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvExportToYtRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx); @@ -121,36 +121,36 @@ protected: void Handle(TEvLongTxRollbackRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvLongTxWriteRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvLongTxReadRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsSetWriteQuotaRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsSetWriteQuotaRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); TActorId DiscoveryCacheActorID; }; diff --git a/ydb/core/grpc_services/rpc_calls.h b/ydb/core/grpc_services/rpc_calls.h index d9332e519d..43136c2f4a 100644 --- a/ydb/core/grpc_services/rpc_calls.h +++ b/ydb/core/grpc_services/rpc_calls.h @@ -137,8 +137,8 @@ using TEvDataStreamsDescribeStreamSummaryRequest = TGRpcRequestWrapper<TRpcServi using TEvDataStreamsDecreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDecreaseStreamRetentionPeriod, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, true>; using TEvDataStreamsIncreaseStreamRetentionPeriodRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsIncreaseStreamRetentionPeriod, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, true>; using TEvDataStreamsUpdateShardCountRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateShardCount, Ydb::DataStreams::V1::UpdateShardCountRequest, Ydb::DataStreams::V1::UpdateShardCountResponse, true>; -using TEvDataStreamsUpdateStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateStream, Ydb::DataStreams::V1::UpdateStreamRequest, Ydb::DataStreams::V1::UpdateStreamResponse, true>; -using TEvDataStreamsSetWriteQuotaRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSetWriteQuota, Ydb::DataStreams::V1::SetWriteQuotaRequest, Ydb::DataStreams::V1::SetWriteQuotaResponse, true>; +using TEvDataStreamsUpdateStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsUpdateStream, Ydb::DataStreams::V1::UpdateStreamRequest, Ydb::DataStreams::V1::UpdateStreamResponse, true>; +using TEvDataStreamsSetWriteQuotaRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsSetWriteQuota, Ydb::DataStreams::V1::SetWriteQuotaRequest, Ydb::DataStreams::V1::SetWriteQuotaResponse, true>; using TEvDataStreamsListStreamConsumersRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsListStreamConsumers, Ydb::DataStreams::V1::ListStreamConsumersRequest, Ydb::DataStreams::V1::ListStreamConsumersResponse, true>; using TEvDataStreamsAddTagsToStreamRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsAddTagsToStream, Ydb::DataStreams::V1::AddTagsToStreamRequest, Ydb::DataStreams::V1::AddTagsToStreamResponse, true>; using TEvDataStreamsDisableEnhancedMonitoringRequest = TGRpcRequestWrapper<TRpcServices::EvDataStreamsDisableEnhancedMonitoring, Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, true>; diff --git a/ydb/core/grpc_services/rpc_deferrable.h b/ydb/core/grpc_services/rpc_deferrable.h index 49c1f48517..644e7b84c4 100644 --- a/ydb/core/grpc_services/rpc_deferrable.h +++ b/ydb/core/grpc_services/rpc_deferrable.h @@ -194,11 +194,11 @@ protected: Reply(status, issues, ctx); } - void Reply(Ydb::StatusIds::StatusCode status, const TActorContext& ctx) { - Request_->ReplyWithYdbStatus(status); - this->Die(ctx); - } - + void Reply(Ydb::StatusIds::StatusCode status, const TActorContext& ctx) { + Request_->ReplyWithYdbStatus(status); + this->Die(ctx); + } + void ReplyWithResult(Ydb::StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext &ctx) { diff --git a/ydb/core/grpc_services/rpc_scheme_base.h b/ydb/core/grpc_services/rpc_scheme_base.h index ef4f54adb6..0be68e802e 100644 --- a/ydb/core/grpc_services/rpc_scheme_base.h +++ b/ydb/core/grpc_services/rpc_scheme_base.h @@ -4,7 +4,7 @@ #include "rpc_deferrable.h" #include <ydb/library/persqueue/topic_parser/topic_parser.h> - + namespace NKikimr { namespace NGRpcService { @@ -66,18 +66,18 @@ protected: { return this->ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, ctx); } - break; + break; } case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: { - ui64 schemeShardTabletId = msg->Record.GetSchemeShardTabletId(); - IActor* pipeActor = NTabletPipe::CreateClient(ctx.SelfID, schemeShardTabletId); - Y_VERIFY(pipeActor); - SchemePipeActorId_ = ctx.ExecutorThread.RegisterActor(pipeActor); + ui64 schemeShardTabletId = msg->Record.GetSchemeShardTabletId(); + IActor* pipeActor = NTabletPipe::CreateClient(ctx.SelfID, schemeShardTabletId); + Y_VERIFY(pipeActor); + SchemePipeActorId_ = ctx.ExecutorThread.RegisterActor(pipeActor); auto request = MakeHolder<NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion>(); - request->Record.SetTxId(msg->Record.GetTxId()); - NTabletPipe::SendData(ctx, SchemePipeActorId_, request.Release()); - return; + request->Record.SetTxId(msg->Record.GetTxId()); + NTabletPipe::SendData(ctx, SchemePipeActorId_, request.Release()); + return; } case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest: { return this->ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, issueMessage, ctx); diff --git a/ydb/core/persqueue/blob.cpp b/ydb/core/persqueue/blob.cpp index da7742bbfb..7727f12a13 100644 --- a/ydb/core/persqueue/blob.cpp +++ b/ydb/core/persqueue/blob.cpp @@ -76,27 +76,27 @@ void TClientBlob::Serialize(TBuffer& res) const res.Append((const char*)&totalSize, sizeof(ui32)); res.Append((const char*)&SeqNo, sizeof(ui64)); ui8 outputUncompressedSize = UncompressedSize == 0 ? 0 : HAS_US; - ui8 outputKinesisData = PartitionKey.empty() ? 0 : HAS_KINESIS; + ui8 outputKinesisData = PartitionKey.empty() ? 0 : HAS_KINESIS; if (PartData) { - ui8 hasPartDataAndTS = HAS_PARTDATA + HAS_TS + HAS_TS2 + outputUncompressedSize + outputKinesisData; //mask + ui8 hasPartDataAndTS = HAS_PARTDATA + HAS_TS + HAS_TS2 + outputUncompressedSize + outputKinesisData; //mask res.Append((const char*)&hasPartDataAndTS, sizeof(char)); res.Append((const char*)&(PartData->PartNo), sizeof(ui16)); res.Append((const char*)&(PartData->TotalParts), sizeof(ui16)); res.Append((const char*)&(PartData->TotalSize), sizeof(ui32)); } else { - ui8 hasTS = HAS_TS + HAS_TS2 + outputUncompressedSize + outputKinesisData; //mask + ui8 hasTS = HAS_TS + HAS_TS2 + outputUncompressedSize + outputKinesisData; //mask res.Append((const char*)&hasTS, sizeof(char)); } - - if (outputKinesisData) { - ui8 partitionKeySize = PartitionKey.size(); - res.Append((const char*)&(partitionKeySize), sizeof(ui8)); - res.Append(PartitionKey.data(), PartitionKey.size()); - ui8 hashKeySize = ExplicitHashKey.size(); - res.Append((const char*)&(hashKeySize), sizeof(ui8)); - res.Append(ExplicitHashKey.data(), ExplicitHashKey.size()); - } - + + if (outputKinesisData) { + ui8 partitionKeySize = PartitionKey.size(); + res.Append((const char*)&(partitionKeySize), sizeof(ui8)); + res.Append(PartitionKey.data(), PartitionKey.size()); + ui8 hashKeySize = ExplicitHashKey.size(); + res.Append((const char*)&(hashKeySize), sizeof(ui8)); + res.Append(ExplicitHashKey.data(), ExplicitHashKey.size()); + } + ui64 writeTimestampMs = WriteTimestamp.MilliSeconds(); ui64 createTimestampMs = CreateTimestamp.MilliSeconds(); res.Append((const char*)&writeTimestampMs, sizeof(ui64)); @@ -108,7 +108,7 @@ void TClientBlob::Serialize(TBuffer& res) const res.Append((const char*)&sz, sizeof(ui16)); res.Append(SourceId.data(), SourceId.size()); res.Append(Data.data(), Data.size()); - + Y_VERIFY(res.Size() == psize + totalSize); } @@ -126,12 +126,12 @@ TClientBlob TClientBlob::Deserialize(const char* data, ui32 size) bool hasTS = (data[0] & HAS_TS); bool hasTS2 = (data[0] & HAS_TS2); bool hasUS = (data[0] & HAS_US); - bool hasKinesisData = (data[0] & HAS_KINESIS); - + bool hasKinesisData = (data[0] & HAS_KINESIS); + ++data; - TString partitionKey; - TString explicitHashKey; - + TString partitionKey; + TString explicitHashKey; + if (hasPartData) { ui16 partNo = ReadUnaligned<ui16>(data); data += sizeof(ui16); @@ -142,17 +142,17 @@ TClientBlob TClientBlob::Deserialize(const char* data, ui32 size) partData = TPartData{partNo, totalParts, totalSize}; } - if (hasKinesisData) { - ui8 keySize = ReadUnaligned<ui8>(data); - data += sizeof(ui8); - partitionKey = TString(data, keySize == 0 ? 256 : keySize); - data += partitionKey.size(); - keySize = ReadUnaligned<ui8>(data); - data += sizeof(ui8); - explicitHashKey = TString(data, keySize); - data += explicitHashKey.Size(); - } - + if (hasKinesisData) { + ui8 keySize = ReadUnaligned<ui8>(data); + data += sizeof(ui8); + partitionKey = TString(data, keySize == 0 ? 256 : keySize); + data += partitionKey.size(); + keySize = ReadUnaligned<ui8>(data); + data += sizeof(ui8); + explicitHashKey = TString(data, keySize); + data += explicitHashKey.Size(); + } + TInstant writeTimestamp; TInstant createTimestamp; ui32 us = 0; @@ -177,8 +177,8 @@ TClientBlob TClientBlob::Deserialize(const char* data, ui32 size) data += sz; Y_VERIFY(data < end, "size %u SeqNo %" PRIu64 " SourceId %s", size, seqNo, sourceId.c_str()); TString dt(data, end - data); - - return TClientBlob(sourceId, seqNo, dt, std::move(partData), writeTimestamp, createTimestamp, us, partitionKey, explicitHashKey); + + return TClientBlob(sourceId, seqNo, dt, std::move(partData), writeTimestamp, createTimestamp, us, partitionKey, explicitHashKey); } TString TBatch::Serialize() { @@ -212,19 +212,19 @@ void TBatch::Pack() { Packed = true; TBuffer res; - bool hasUncompressed = false; - bool hasKinesis = false; - for (ui32 i = 0; i < Blobs.size(); ++i) { - if (Blobs[i].UncompressedSize > 0) - hasUncompressed = true; - - if (!Blobs[i].PartitionKey.empty() || !Blobs[i].ExplicitHashKey.empty()) { - hasKinesis = true; - } - } - + bool hasUncompressed = false; + bool hasKinesis = false; + for (ui32 i = 0; i < Blobs.size(); ++i) { + if (Blobs[i].UncompressedSize > 0) + hasUncompressed = true; + + if (!Blobs[i].PartitionKey.empty() || !Blobs[i].ExplicitHashKey.empty()) { + hasKinesis = true; + } + } + Header.SetFormat(NKikimrPQ::TBatchHeader::ECompressed); - Header.SetHasKinesis(hasKinesis); + Header.SetHasKinesis(hasKinesis); ui32 totalCount = Blobs.size(); Y_VERIFY(totalCount == Header.GetCount() + Header.GetInternalPartsCount()); ui32 cnt = 0; @@ -323,24 +323,24 @@ void TBatch::Pack() { OutputChunk(chunk, output, res); } - if (hasKinesis) { - { - auto chunk = MakeChunk<NScheme::TVarLenCodec<false>>(output); - for (const auto &p : pos) { - chunk->AddData(Blobs[p].PartitionKey.data(), Blobs[p].PartitionKey.size()); - } - OutputChunk(chunk, output, res); - } - - { - auto chunk = MakeChunk<NScheme::TVarLenCodec<false>>(output); - for (const auto &p : pos) { - chunk->AddData(Blobs[p].ExplicitHashKey.data(), Blobs[p].ExplicitHashKey.size()); - } - OutputChunk(chunk, output, res); - } - } - + if (hasKinesis) { + { + auto chunk = MakeChunk<NScheme::TVarLenCodec<false>>(output); + for (const auto &p : pos) { + chunk->AddData(Blobs[p].PartitionKey.data(), Blobs[p].PartitionKey.size()); + } + OutputChunk(chunk, output, res); + } + + { + auto chunk = MakeChunk<NScheme::TVarLenCodec<false>>(output); + for (const auto &p : pos) { + chunk->AddData(Blobs[p].ExplicitHashKey.data(), Blobs[p].ExplicitHashKey.size()); + } + OutputChunk(chunk, output, res); + } + } + //output Ctime { auto chunk = MakeChunk<NScheme::TDeltaVarIntCodec<ui64, false>>(output); @@ -508,33 +508,33 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) { TVector<TInstant> ctime; ctime.reserve(totalBlobs); - TVector<TString> partitionKey; - TVector<TString> explicitHash; - partitionKey.reserve(totalBlobs); - explicitHash.reserve(totalBlobs); - if (Header.GetHasKinesis()) { - { - auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &stringCodecs); - auto iter = chunk->MakeIterator(); - for (ui32 i = 0; i < totalBlobs; ++i) { - auto ref = iter->Next(); - partitionKey.emplace_back(ref.Data(), ref.Size()); - } - } - - { - auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &stringCodecs); - auto iter = chunk->MakeIterator(); - for (ui32 i = 0; i < totalBlobs; ++i) { - auto ref = iter->Next(); - explicitHash.emplace_back(ref.Data(), ref.Size()); - } - } - } else { - partitionKey.resize(totalBlobs); - explicitHash.resize(totalBlobs); - } - + TVector<TString> partitionKey; + TVector<TString> explicitHash; + partitionKey.reserve(totalBlobs); + explicitHash.reserve(totalBlobs); + if (Header.GetHasKinesis()) { + { + auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &stringCodecs); + auto iter = chunk->MakeIterator(); + for (ui32 i = 0; i < totalBlobs; ++i) { + auto ref = iter->Next(); + partitionKey.emplace_back(ref.Data(), ref.Size()); + } + } + + { + auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &stringCodecs); + auto iter = chunk->MakeIterator(); + for (ui32 i = 0; i < totalBlobs; ++i) { + auto ref = iter->Next(); + explicitHash.emplace_back(ref.Data(), ref.Size()); + } + } + } else { + partitionKey.resize(totalBlobs); + explicitHash.resize(totalBlobs); + } + if (data < dataEnd) { //old versions could not have CTime { auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &ui64Codecs); @@ -567,12 +567,12 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) { blobs->resize(totalBlobs); ui32 currentSID = 0; for (ui32 i = 0; i < totalBlobs; ++i) { - TMaybe<TPartData> pd; + TMaybe<TPartData> pd; auto it = partData.find(pos[i]); if (it != partData.end()) pd = it->second; - (*blobs)[pos[i]] = TClientBlob(sourceIds[currentSID], seqNo[i], dt[i], std::move(pd), wtime[pos[i]], ctime[pos[i]], uncompressedSize[pos[i]], - partitionKey[i], explicitHash[i]); + (*blobs)[pos[i]] = TClientBlob(sourceIds[currentSID], seqNo[i], dt[i], std::move(pd), wtime[pos[i]], ctime[pos[i]], uncompressedSize[pos[i]], + partitionKey[i], explicitHash[i]); if (i + 1 == end[currentSID]) ++currentSID; } diff --git a/ydb/core/persqueue/blob.h b/ydb/core/persqueue/blob.h index 5cbaf40e83..633ec2499a 100644 --- a/ydb/core/persqueue/blob.h +++ b/ydb/core/persqueue/blob.h @@ -32,7 +32,7 @@ struct TClientBlob { static const ui8 HAS_TS = 2; static const ui8 HAS_TS2 = 4; static const ui8 HAS_US = 8; - static const ui8 HAS_KINESIS = 16; + static const ui8 HAS_KINESIS = 16; TString SourceId; ui64 SeqNo; @@ -41,8 +41,8 @@ struct TClientBlob { TInstant WriteTimestamp; TInstant CreateTimestamp; ui32 UncompressedSize; - TString PartitionKey; - TString ExplicitHashKey; + TString PartitionKey; + TString ExplicitHashKey; TClientBlob() : SeqNo(0) @@ -50,7 +50,7 @@ struct TClientBlob { {} TClientBlob(const TString& sourceId, const ui64 seqNo, const TString& data, TMaybe<TPartData> &&partData, TInstant writeTimestamp, TInstant createTimestamp, - const ui64 uncompressedSize, const TString& partitionKey, const TString& explicitHashKey) + const ui64 uncompressedSize, const TString& partitionKey, const TString& explicitHashKey) : SourceId(sourceId) , SeqNo(seqNo) , Data(data) @@ -58,11 +58,11 @@ struct TClientBlob { , WriteTimestamp(writeTimestamp) , CreateTimestamp(createTimestamp) , UncompressedSize(uncompressedSize) - , PartitionKey(partitionKey) - , ExplicitHashKey(explicitHashKey) - { - Y_VERIFY(PartitionKey.size() <= 256); - } + , PartitionKey(partitionKey) + , ExplicitHashKey(explicitHashKey) + { + Y_VERIFY(PartitionKey.size() <= 256); + } ui32 GetPartDataSize() const { if (PartData) { @@ -71,15 +71,15 @@ struct TClientBlob { return 1; } - ui32 GetKinesisSize() const { - if (PartitionKey.size() > 0) { - return 2 + PartitionKey.size() + ExplicitHashKey.size(); - } - return 0; - } - + ui32 GetKinesisSize() const { + if (PartitionKey.size() > 0) { + return 2 + PartitionKey.size() + ExplicitHashKey.size(); + } + return 0; + } + ui32 GetBlobSize() const { - return GetPartDataSize() + OVERHEAD + SourceId.size() + Data.size() + (UncompressedSize == 0 ? 0 : sizeof(ui32)) + GetKinesisSize(); + return GetPartDataSize() + OVERHEAD + SourceId.size() + Data.size() + (UncompressedSize == 0 ? 0 : sizeof(ui32)) + GetKinesisSize(); } ui16 GetPartNo() const { diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index 2f9d311319..bc892988b4 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -129,18 +129,18 @@ struct TEvPQ { ui64 WriteTimestamp; TString Data; ui32 UncompressedSize; - TString PartitionKey; - TString ExplicitHashKey; + TString PartitionKey; + TString ExplicitHashKey; bool External; }; - TEvWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe<ui64> offset, TVector<TMsg> &&msgs, bool isDirectWrite) + TEvWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe<ui64> offset, TVector<TMsg> &&msgs, bool isDirectWrite) : Cookie(cookie) , MessageNo(messageNo) , OwnerCookie(ownerCookie) , Offset(offset) , Msgs(std::move(msgs)) - , IsDirectWrite(isDirectWrite) + , IsDirectWrite(isDirectWrite) {} ui64 Cookie; @@ -148,7 +148,7 @@ struct TEvPQ { TString OwnerCookie; TMaybe<ui64> Offset; TVector<TMsg> Msgs; - bool IsDirectWrite; + bool IsDirectWrite; }; diff --git a/ydb/core/persqueue/internals_ut.cpp b/ydb/core/persqueue/internals_ut.cpp index aa701944c0..d72ffe5e11 100644 --- a/ydb/core/persqueue/internals_ut.cpp +++ b/ydb/core/persqueue/internals_ut.cpp @@ -17,7 +17,7 @@ Y_UNIT_TEST(TestPartitionedBlobSimpleTest) { THead newHead; TPartitionedBlob blob(0, 0, "sourceId", 1, 1, 10, head, newHead, false, false, 8 << 20); - TClientBlob clientBlob("sourceId", 1, "valuevalue", TMaybe<TPartData>(), TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "123", "123"); + TClientBlob clientBlob("sourceId", 1, "valuevalue", TMaybe<TPartData>(), TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "123", "123"); UNIT_ASSERT(blob.IsInited()); TString error; UNIT_ASSERT(blob.IsNextPart("sourceId", 1, 0, &error)); @@ -39,7 +39,7 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) for (ui32 i = 0; i < 50; ++i) { head.Batches.back().AddBlob(TClientBlob( "sourceId" + TString(1,'a' + rand() % 26), i + 1, value, TMaybe<TPartData>(), - TInstant::MilliSeconds(i + 1), TInstant::MilliSeconds(i + 1), 1, "", "" + TInstant::MilliSeconds(i + 1), TInstant::MilliSeconds(i + 1), 1, "", "" )); if (!headCompacted) all.push_back(head.Batches.back().Blobs.back()); @@ -61,7 +61,7 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) for (ui32 i = 0; i < 10; ++i) { newHead.Batches.back().AddBlob(TClientBlob( "sourceId2", i + 1, value, TMaybe<TPartData>(), - TInstant::MilliSeconds(i + 1000), TInstant::MilliSeconds(i + 1000), 1, "", "" + TInstant::MilliSeconds(i + 1000), TInstant::MilliSeconds(i + 1000), 1, "", "" )); all.push_back(newHead.Batches.back().Blobs.back()); //newHead always glued } @@ -79,7 +79,7 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) TMaybe<TPartData> partData = TPartData(i, parts, value2.size()); TClientBlob clientBlob( "soruceId3", 1, value2, std::move(partData), - TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 1, "", "" + TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 1, "", "" ); all.push_back(clientBlob); auto res = blob.Add(std::move(clientBlob)); @@ -173,7 +173,7 @@ Y_UNIT_TEST(TestBatchPacking) { for (ui32 i = 0; i < 100; ++i) { batch.AddBlob(TClientBlob( "sourceId1", i + 1, value, TMaybe<TPartData>(), - TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "", "" + TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "", "" )); } batch.Pack(); @@ -191,7 +191,7 @@ Y_UNIT_TEST(TestBatchPacking) { TBatch batch3; batch3.AddBlob(TClientBlob( "sourceId", 999999999999999ll, "abacaba", TPartData{33,66,4000000000u}, - TInstant::MilliSeconds(999999999999ll), TInstant::MilliSeconds(1000), 0, "", "" + TInstant::MilliSeconds(999999999999ll), TInstant::MilliSeconds(1000), 0, "", "" )); batch3.Pack(); UNIT_ASSERT(batch3.Header.GetFormat() == NKikimrPQ::TBatchHeader::EUncompressed); diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index a51c757829..cc8e2419cf 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -460,7 +460,7 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co , Partition(partition) , Config(config) , TopicName(topicName) - , TopicPath(topicPath) + , TopicPath(topicPath) , LocalDC(localDC) , DCId(std::move(dcId)) , StartOffset(0) @@ -2249,9 +2249,9 @@ static void AddResultBlob(T* read, const TClientBlob& blob, ui64 offset) cc->SetWriteTimestampMS(blob.WriteTimestamp.MilliSeconds()); cc->SetCreateTimestampMS(blob.CreateTimestamp.MilliSeconds()); cc->SetUncompressedSize(blob.UncompressedSize); - cc->SetPartitionKey(blob.PartitionKey); - cc->SetExplicitHash(blob.ExplicitHashKey); - + cc->SetPartitionKey(blob.PartitionKey); + cc->SetExplicitHash(blob.ExplicitHashKey); + if (blob.PartData) { cc->SetPartNo(blob.PartData->PartNo); cc->SetTotalParts(blob.PartData->TotalParts); @@ -3565,7 +3565,7 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c ui64 decReservedSize = 0; TStringBuf owner; - if (!mirroredPartition && !ev->Get()->IsDirectWrite) { + if (!mirroredPartition && !ev->Get()->IsDirectWrite) { owner = TOwnerInfo::GetOwnerFromOwnerCookie(ev->Get()->OwnerCookie); auto it = Owners.find(owner); @@ -3614,7 +3614,7 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c } for (const auto& msg: ev->Get()->Msgs) { //this is checked in pq_impl when forming EvWrite request - Y_VERIFY(!msg.SourceId.empty() || ev->Get()->IsDirectWrite); + Y_VERIFY(!msg.SourceId.empty() || ev->Get()->IsDirectWrite); Y_VERIFY(!msg.Data.empty()); if (msg.SeqNo > (ui64)Max<i64>()) { @@ -4144,8 +4144,8 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const WriteTimestamp = ctx.Now(); WriteTimestampEstimate = p.Msg.WriteTimestamp > 0 ? TInstant::MilliSeconds(p.Msg.WriteTimestamp) : WriteTimestamp; TClientBlob blob(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.Data, std::move(partData), WriteTimestampEstimate, - TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp), - p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime + TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp), + p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds(); WriteLagMs.Update(writeLagMs, WriteTimestamp); @@ -4689,7 +4689,7 @@ void TPartition::WriteBlobWithQuota(THolder<TEvKeyValue::TEvRequest>&& request) { // Request quota and write blob. // Mirrored topics are not quoted in local dc. - const bool skip = !IsQuotingEnabled() || TopicWriteQuotaResourcePath.empty(); + const bool skip = !IsQuotingEnabled() || TopicWriteQuotaResourcePath.empty(); if (size_t quotaRequestSize = skip ? 0 : GetQuotaRequestSize(*request)) { // Request with data. We should check before attempting to write data whether we have enough quota. Y_VERIFY(!WaitingForPreviousBlobQuota()); @@ -4712,31 +4712,31 @@ void TPartition::WriteBlobWithQuota(THolder<TEvKeyValue::TEvRequest>&& request) void TPartition::CalcTopicWriteQuotaParams() { - const auto& pqConfig = AppData()->PQConfig; - const auto& quotingConfig = pqConfig.GetQuotingConfig(); - if (IsQuotingEnabled()) { // Mirrored topics are not quoted in local dc. + const auto& pqConfig = AppData()->PQConfig; + const auto& quotingConfig = pqConfig.GetQuotingConfig(); + if (IsQuotingEnabled()) { // Mirrored topics are not quoted in local dc. Y_VERIFY(quotingConfig.GetTopicWriteQuotaEntityToLimit() != NKikimrPQ::TPQConfig::TQuotingConfig::UNSPECIFIED); - TString topicPath = TopicPath.empty() ? TopicName : TopicPath; - TFsPath fsPath(topicPath); - if (fsPath.IsSubpathOf(pqConfig.GetRoot())) { - topicPath = fsPath.RelativePath(TFsPath(pqConfig.GetRoot())).GetPath(); - } - topicPath = NPersQueue::GetTopicPath(topicPath); - auto topicParts = SplitPath(topicPath); // account/folder/topic // account is first element - if (topicParts.size() < 2) { + TString topicPath = TopicPath.empty() ? TopicName : TopicPath; + TFsPath fsPath(topicPath); + if (fsPath.IsSubpathOf(pqConfig.GetRoot())) { + topicPath = fsPath.RelativePath(TFsPath(pqConfig.GetRoot())).GetPath(); + } + topicPath = NPersQueue::GetTopicPath(topicPath); + auto topicParts = SplitPath(topicPath); // account/folder/topic // account is first element + if (topicParts.size() < 2) { LOG_WARN_S(TActivationContext::AsActorContext(), NKikimrServices::PERSQUEUE, - "tablet " << TabletID << " topic '" << topicPath << "' Bad topic name. Disable quoting for topic"); + "tablet " << TabletID << " topic '" << topicPath << "' Bad topic name. Disable quoting for topic"); return; } - const TString account = topicParts[0]; - topicParts[0] = WRITE_QUOTA_ROOT_PATH; // write-quota/folder/topic + const TString account = topicParts[0]; + topicParts[0] = WRITE_QUOTA_ROOT_PATH; // write-quota/folder/topic - TopicWriteQuotaResourcePath = JoinPath(topicParts); + TopicWriteQuotaResourcePath = JoinPath(topicParts); TopicWriteQuoterPath = TStringBuilder() << quotingConfig.GetQuotersDirectoryPath() << "/" << account; - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::PERSQUEUE, - "topicWriteQuutaResourcePath " << TopicWriteQuotaResourcePath << " topicWriteQuoterPath '" << TopicWriteQuoterPath << " account " << account); + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::PERSQUEUE, + "topicWriteQuutaResourcePath " << TopicWriteQuotaResourcePath << " topicWriteQuoterPath '" << TopicWriteQuoterPath << " account " << account); } } @@ -4747,11 +4747,11 @@ void TPartition::CreateMirrorerActor() { ); } -bool TPartition::IsQuotingEnabled() const { - const auto& pqConfig = AppData()->PQConfig; - const auto& quotingConfig = pqConfig.GetQuotingConfig(); - return LocalDC && !pqConfig.GetTopicsAreFirstClassCitizen() && quotingConfig.GetEnableQuoting(); -} - +bool TPartition::IsQuotingEnabled() const { + const auto& pqConfig = AppData()->PQConfig; + const auto& quotingConfig = pqConfig.GetQuotingConfig(); + return LocalDC && !pqConfig.GetTopicsAreFirstClassCitizen() && quotingConfig.GetEnableQuoting(); +} + }// NPQ }// NKikimr diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index 2cf75fc6f8..2eec26f251 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -208,8 +208,8 @@ private: void ProcessReserveRequests(const TActorContext& ctx); void CreateMirrorerActor(); - bool IsQuotingEnabled() const; - + bool IsQuotingEnabled() const; + void SetupTopicCounters(const TActorContext& ctx); void SetupStreamCounters(const TActorContext& ctx); @@ -220,7 +220,7 @@ public: TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, const TActorId& blobCache, const TString& topicName, const TString& topicPath, const bool localDC, TString dcId, - const NKikimrPQ::TPQTabletConfig& config, const TTabletCountersBase& counters, + const NKikimrPQ::TPQTabletConfig& config, const TTabletCountersBase& counters, const TActorContext& ctx, bool newPartition = false); void Bootstrap(const TActorContext& ctx); @@ -416,7 +416,7 @@ private: ui32 Partition; NKikimrPQ::TPQTabletConfig Config; TString TopicName; - TString TopicPath; + TString TopicPath; bool LocalDC; TString DCId; diff --git a/ydb/core/persqueue/percentile_counter.cpp b/ydb/core/persqueue/percentile_counter.cpp index badbc95432..a6414e70bd 100644 --- a/ydb/core/persqueue/percentile_counter.cpp +++ b/ydb/core/persqueue/percentile_counter.cpp @@ -46,7 +46,7 @@ TVector<TLabelsInfo> GetLabels(const TString& topic) return GetLabels(origDC, topic.substr(pos + 2)); } - + TVector<TLabelsInfo> GetLabels(const TString& cluster, const TString& realTopic) { TString origDC = cluster; diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index bd7dd5915f..9937dbe561 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -157,8 +157,8 @@ private: Y_VERIFY(partResp->GetResult(partResp->ResultSize() - 1).GetSeqNo() == res.GetResult(i).GetSeqNo()); auto rr = partResp->MutableResult(partResp->ResultSize() - 1); (*rr->MutableData()) += res.GetResult(i).GetData(); - rr->SetPartitionKey(res.GetResult(i).GetPartitionKey()); - rr->SetExplicitHash(res.GetResult(i).GetExplicitHash()); + rr->SetPartitionKey(res.GetResult(i).GetPartitionKey()); + rr->SetExplicitHash(res.GetResult(i).GetExplicitHash()); rr->SetPartNo(res.GetResult(i).GetPartNo()); rr->SetUncompressedSize(rr->GetUncompressedSize() + res.GetResult(i).GetUncompressedSize()); if (res.GetResult(i).GetPartNo() + 1 == res.GetResult(i).GetTotalParts()) { @@ -595,8 +595,8 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) cacheSize = Config.GetCacheSize(); if (TopicName.empty()) { // it's the first time - TopicName = Config.GetTopicName(); - TopicPath = Config.GetTopicPath(); + TopicName = Config.GetTopicName(); + TopicPath = Config.GetTopicPath(); LocalDC = Config.GetLocalDC(); KeySchema.clear(); @@ -608,7 +608,7 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) Y_VERIFY(TopicName.size(), "Need topic name here"); CacheActor = ctx.Register(new TPQCacheProxy(ctx.SelfID, TopicName, cacheSize)); } else { - Y_VERIFY(TopicName == Config.GetTopicName(), "Changing topic name is not supported"); + Y_VERIFY(TopicName == Config.GetTopicName(), "Changing topic name is not supported"); ctx.Send(CacheActor, new TEvPQ::TEvChangeCacheConfig(cacheSize)); } @@ -703,7 +703,7 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& } } - TopicName = Config.GetTopicName(); + TopicName = Config.GetTopicName(); LocalDC = Config.GetLocalDC(); KeySchema.clear(); @@ -1415,17 +1415,17 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom(); - if (!req.GetIsDirectWrite()) { - if (!req.HasMessageNo()) { - ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "MessageNo must be set for writes"); - return; - } - - - if (!mirroredPartition && !req.HasOwnerCookie()) { - ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "OwnerCookie must be set for writes"); - return; - } + if (!req.GetIsDirectWrite()) { + if (!req.HasMessageNo()) { + ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "MessageNo must be set for writes"); + return; + } + + + if (!mirroredPartition && !req.HasOwnerCookie()) { + ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "OwnerCookie must be set for writes"); + return; + } } if (req.HasCmdWriteOffset() && req.GetCmdWriteOffset() < 0) { @@ -1457,14 +1457,14 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p } TString errorStr = ""; - if (!cmd.HasSeqNo() && !req.GetIsDirectWrite()) { + if (!cmd.HasSeqNo() && !req.GetIsDirectWrite()) { errorStr = "no SeqNo"; } else if (!cmd.HasData() || cmd.GetData().empty()){ errorStr = "empty Data"; - } else if ((!cmd.HasSourceId() || cmd.GetSourceId().empty()) && !req.GetIsDirectWrite()) { + } else if ((!cmd.HasSourceId() || cmd.GetSourceId().empty()) && !req.GetIsDirectWrite()) { errorStr = "empty SourceId"; - } else if (cmd.GetPartitionKey().size() > 256) { - errorStr = "too long partition key"; + } else if (cmd.GetPartitionKey().size() > 256) { + errorStr = "too long partition key"; } else if (cmd.GetSeqNo() < 0) { errorStr = "SeqNo must be >= 0"; } else if (cmd.HasPartNo() && (cmd.GetPartNo() < 0 || cmd.GetPartNo() >= Max<ui16>())) { @@ -1526,7 +1526,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p diff = 0; msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), partNo, totalParts, totalSize, createTimestampMs, receiveTimestampMs, - disableDeduplication, writeTimestampMs, data, uncompressedSize, + disableDeduplication, writeTimestampMs, data, uncompressedSize, cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation() }); partNo++; diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index b1ec7bd9ae..bc3bfb10ba 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -145,7 +145,7 @@ private: bool NewConfigShouldBeApplied; TString TopicName; - TString TopicPath; + TString TopicPath; bool LocalDC; TString DCId; TVector<NScheme::TTypeId> KeySchema; diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h index bebb95846e..3db93bc104 100644 --- a/ydb/core/persqueue/pq_ut.h +++ b/ydb/core/persqueue/pq_ut.h @@ -228,7 +228,7 @@ void PQTabletPrepare(ui32 mcip, ui64 msip, ui32 deleteTime, const TVector<std::p request->Record.MutableTabletConfig()->SetCacheSize(10*1024*1024); request->Record.SetTxId(12345); auto tabletConfig = request->Record.MutableTabletConfig(); - tabletConfig->SetTopicName("topic"); + tabletConfig->SetTopicName("topic"); tabletConfig->SetVersion(version); tabletConfig->SetLocalDC(localDC); tabletConfig->AddReadRules("user"); diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp index 79995ee314..4eb9c7206d 100644 --- a/ydb/core/persqueue/read_balancer.cpp +++ b/ydb/core/persqueue/read_balancer.cpp @@ -740,7 +740,7 @@ TPersQueueReadBalancer::TClientGroupInfo& TPersQueueReadBalancer::TClientInfo::A clientInfo.ClientId = ClientId; clientInfo.Topic = Topic; clientInfo.TabletId = TabletId; - clientInfo.Path = Path; + clientInfo.Path = Path; clientInfo.Generation = Generation; clientInfo.Step = &Step; @@ -833,7 +833,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvRegisterReadSession::TPtr& it->second.ClientId = record.GetClientId(); it->second.Topic = Topic; it->second.TabletId = TabletID(); - it->second.Path = Path; + it->second.Path = Path; it->second.Generation = Generation; it->second.Step = 0; } @@ -1142,7 +1142,7 @@ void TPersQueueReadBalancer::TClientGroupInfo::LockPartition(const TActorId pipe res->Record.SetSession(pipeInfo.Session); res->Record.SetPartition(partition); res->Record.SetTopic(Topic); - res->Record.SetPath(Path); + res->Record.SetPath(Path); res->Record.SetGeneration(Generation); res->Record.SetStep(++(*Step)); res->Record.SetClientId(ClientId); @@ -1166,7 +1166,7 @@ void TPersQueueReadBalancer::TClientGroupInfo::ReleasePartition(const TActorId p THolder<TEvPersQueue::TEvReleasePartition> res{new TEvPersQueue::TEvReleasePartition}; res->Record.SetSession(sessionInfo.Session); res->Record.SetTopic(Topic); - res->Record.SetPath(Path); + res->Record.SetPath(Path); res->Record.SetGeneration(Generation); res->Record.SetClientId(ClientId); res->Record.SetCount(count); diff --git a/ydb/core/persqueue/read_balancer.h b/ydb/core/persqueue/read_balancer.h index 67a5e48e1e..6a07edcb78 100644 --- a/ydb/core/persqueue/read_balancer.h +++ b/ydb/core/persqueue/read_balancer.h @@ -357,7 +357,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa TString ClientId; TString Topic; ui64 TabletId; - TString Path; + TString Path; ui32 Generation = 0; ui64 RandomNumber = 0; ui32* Step = nullptr; @@ -396,7 +396,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa TString ClientId; TString Topic; ui64 TabletId; - TString Path; + TString Path; ui32 Generation = 0; ui32 Step = 0; diff --git a/ydb/core/protos/http_config.proto b/ydb/core/protos/http_config.proto index 5b195c07ab..b4a3657d4a 100644 --- a/ydb/core/protos/http_config.proto +++ b/ydb/core/protos/http_config.proto @@ -1,13 +1,13 @@ -package NKikimrConfig; -option java_package = "ru.yandex.kikimr.proto"; - -message THttpServerConfig { +package NKikimrConfig; +option java_package = "ru.yandex.kikimr.proto"; + +message THttpServerConfig { // Port number is specifed via cli arg --sqs-port, don't set up it in the config - optional uint32 Port = 1 [default = 8771]; - optional uint32 Threads = 2 [default = 5]; - optional uint32 MaxConnections = 3 [default = 1000]; // 0 - unlimited - optional uint32 MaxQueueSize = 4 [default = 500]; // 0 - unlimited - optional bool EnableKeepAlive = 5 [default = true]; -} - - + optional uint32 Port = 1 [default = 8771]; + optional uint32 Threads = 2 [default = 5]; + optional uint32 MaxConnections = 3 [default = 1000]; // 0 - unlimited + optional uint32 MaxQueueSize = 4 [default = 500]; // 0 - unlimited + optional bool EnableKeepAlive = 5 [default = true]; +} + + diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 98bdadf11f..849b292383 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -21,9 +21,9 @@ message TPersQueuePartitionRequest { optional bool MirrorerRequest = 10 [default = false]; optional string ClientDC = 11; - - optional string PartitionKey = 12; - optional string ExplicitHash = 13; + + optional string PartitionKey = 12; + optional string ExplicitHash = 13; optional bool ExternalOperation = 14 [default = false]; } @@ -65,9 +65,9 @@ message TPersQueuePartitionRequest { optional int32 UncompressedSize = 12; //fill it for all parts optional string ClientDC = 11; - - optional string PartitionKey = 13; - optional bytes ExplicitHash = 14; + + optional string PartitionKey = 13; + optional bytes ExplicitHash = 14; optional bool ExternalOperation = 15 [ default = false ]; } @@ -136,7 +136,7 @@ message TPersQueuePartitionRequest { optional TCmdUpdateWriteTimestamp CmdUpdateWriteTimestamp = 17; - optional bool IsDirectWrite = 18 [default = false]; + optional bool IsDirectWrite = 18 [default = false]; optional uint64 PutUnitsSize = 19; } @@ -365,9 +365,9 @@ message TCmdReadResult { optional uint64 WriteTimestampMS = 8; optional uint64 CreateTimestampMS = 9; optional uint32 UncompressedSize = 10; - - optional string PartitionKey = 11; - optional bytes ExplicitHash = 12; + + optional string PartitionKey = 11; + optional bytes ExplicitHash = 12; } optional uint64 MaxOffset = 2; repeated TResult Result = 3; diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index b93b45f306..7c85927449 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -102,11 +102,11 @@ message TPQConfig { repeated TChannelProfile ChannelProfiles = 31; - optional bool TopicsAreFirstClassCitizen = 32 [default = false]; + optional bool TopicsAreFirstClassCitizen = 32 [default = false]; - optional string SourceIdTablePath = 33 [default = "/Root/PQ/SourceIdMeta2"]; - - repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34; + optional string SourceIdTablePath = 33 [default = "/Root/PQ/SourceIdMeta2"]; + + repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34; message TReadMeteringConfig { message TPricing { @@ -228,7 +228,7 @@ message TPQTabletConfig { optional uint64 CacheSize = 1 [default = 104857600]; //100Mb, per tablet optional TPartitionConfig PartitionConfig = 2; //mandatory repeated uint32 PartitionIds = 3; //will be filled by schemeshard, don't touch. Deprecated by Partitions (id: 31) - optional string TopicName = 4; // also filled by schemeshard + optional string TopicName = 4; // also filled by schemeshard optional uint32 Version = 5; //also filled by schemeshard optional bool LocalDC = 6 [default = false]; optional bool RequireAuthWrite = 7 [default = false]; @@ -255,7 +255,7 @@ message TPQTabletConfig { repeated uint64 ReadRuleVersions = 22; repeated uint64 ReadRuleGenerations = 32; - optional string TopicPath = 23; + optional string TopicPath = 23; optional uint64 AbcId = 24; optional string AbcSlug = 25; @@ -457,7 +457,7 @@ message TLockPartition { optional string Session = 6; optional string ClientId = 7; optional NActorsProto.TActorId PipeClient = 8; - optional string Path = 9; + optional string Path = 9; } message TReleasePartition { @@ -468,7 +468,7 @@ message TReleasePartition { optional uint32 Count = 5; optional NActorsProto.TActorId PipeClient = 6; optional uint32 Group = 7; - optional string Path = 8; + optional string Path = 8; } message TPartitionReleased { @@ -652,7 +652,7 @@ message TBatchHeader { ECompressed = 1; } optional uint32 Format = 7; //for EPayloadFormat - optional bool HasKinesis = 8; + optional bool HasKinesis = 8; } message TUserInfo { diff --git a/ydb/core/protos/serverless_proxy_config.proto b/ydb/core/protos/serverless_proxy_config.proto index 74a921996b..43ed3c331d 100644 --- a/ydb/core/protos/serverless_proxy_config.proto +++ b/ydb/core/protos/serverless_proxy_config.proto @@ -1,33 +1,33 @@ import "ydb/core/protos/http_config.proto"; import "ydb/core/protos/config.proto"; -package NKikimrConfig; -option java_package = "ru.yandex.kikimr.proto"; - -message TServerlessProxyConfig { - message TDatabaseEndpoints { - optional string Database = 1; - optional string DiscoveryEndpoint = 2; - } - - optional THttpServerConfig HttpServerConfig = 1; +package NKikimrConfig; +option java_package = "ru.yandex.kikimr.proto"; + +message TServerlessProxyConfig { + message TDatabaseEndpoints { + optional string Database = 1; + optional string DiscoveryEndpoint = 2; + } + + optional THttpServerConfig HttpServerConfig = 1; optional TGRpcConfig GRpcConfig = 9; repeated string YandexCloudServiceRegion = 2; - - repeated TDatabaseEndpoints KnownEndpoints = 3; - - optional string CaCert = 4; - - optional string IamTokenServiceEndpoint = 5; - optional string AccessServiceEndpoint = 6; - - optional string DatabaseServiceEndpoint = 7; - - optional bool TestMode = 8; + + repeated TDatabaseEndpoints KnownEndpoints = 3; + + optional string CaCert = 4; + + optional string IamTokenServiceEndpoint = 5; + optional string AccessServiceEndpoint = 6; + + optional string DatabaseServiceEndpoint = 7; + + optional bool TestMode = 8; optional uint32 DatabaseListRefreshPeriodSeconds = 10 [default = 10]; optional TLogConfig LogConfig = 11; -} - +} + diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 2db555efe1..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -816,7 +816,7 @@ message TActivity { TEST_SHARD_ACTOR = 511; FRONT_TEST_SHARD_REQUEST = 512; STATE_SERVER_INTERFACE_ACTOR = 513; - IAM_TOKEN_SERVICE_ACTOR = 514; + IAM_TOKEN_SERVICE_ACTOR = 514; YQL_EXECUTE_PLAN_ACTOR = 515; YQL_DB_POOL_ACTOR = 516; YQL_RUN_ACTOR = 517; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 7cfd8d9dce..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -61,7 +61,7 @@ SRCS( flat_scheme_op.proto health.proto hive.proto - http_config.proto + http_config.proto import.proto index_builder.proto issue_id.proto @@ -88,7 +88,7 @@ SRCS( scheme_log.proto scheme_type_metadata.proto scheme_type_operation.proto - serverless_proxy_config.proto + serverless_proxy_config.proto services.proto shared_cache.proto sqs.proto diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index 5b72e38945..5e90eb5992 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -1225,12 +1225,12 @@ Y_UNIT_TEST_SUITE(SystemView) { .Build(); auto result = session.CreateTable("/Root/.sys/partition_stats", std::move(desc)).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { auto result = session.CopyTable("/Root/.sys/partition_stats", "/Root/Table0").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { @@ -1238,19 +1238,19 @@ Y_UNIT_TEST_SUITE(SystemView) { .AppendDropColumns("OwnerId"); auto result = session.AlterTable("/Root/.sys/partition_stats", settings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { auto result = session.DropTable("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { auto result = session.ExecuteSchemeQuery(R"( DROP TABLE `/Root/.sys/partition_stats`; )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { @@ -1278,23 +1278,23 @@ Y_UNIT_TEST_SUITE(SystemView) { } { auto result = schemeClient.MakeDirectory("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { auto result = schemeClient.RemoveDirectory("/Root/.sys").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { auto result = schemeClient.RemoveDirectory("/Root/.sys/partition_stats").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } { TModifyPermissionsSettings settings; auto result = schemeClient.ModifyPermissions("/Root/.sys/partition_stats", settings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); result.GetIssues().PrintTo(Cerr); } } diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 2c0f7fad8b..1885b26e09 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -1749,7 +1749,7 @@ TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletLabeledCo continue; } if (groupNames[j] == "Client") { - group = group->GetSubgroup("ConsumerPath", NPersQueue::ConvertOldConsumerName(groups[j], ctx)); + group = group->GetSubgroup("ConsumerPath", NPersQueue::ConvertOldConsumerName(groups[j], ctx)); continue; } } @@ -2407,7 +2407,7 @@ public: if (groups.size() == 1) { //topic case ff = groups[0]; } else if (groups.size() == 3) { //client important topic - res = NPersQueue::ConvertOldConsumerName(groups[0], ctx) + "|" + groups[1] + "|"; + res = NPersQueue::ConvertOldConsumerName(groups[0], ctx) + "|" + groups[1] + "|"; ff = groups[2]; } else { continue; diff --git a/ydb/core/testlib/service_mocks/access_service_mock.h b/ydb/core/testlib/service_mocks/access_service_mock.h index 33df4592ce..f18ac92f0a 100644 --- a/ydb/core/testlib/service_mocks/access_service_mock.h +++ b/ydb/core/testlib/service_mocks/access_service_mock.h @@ -31,18 +31,18 @@ public: virtual grpc::Status Authenticate( grpc::ServerContext* ctx, const yandex::cloud::priv::servicecontrol::v1::AuthenticateRequest* request, - yandex::cloud::priv::servicecontrol::v1::AuthenticateResponse* response) override - { - TString key; - if (request->has_signature()) { - key = request->signature().v4_parameters().service(); - } else { - key = request->iam_token(); - } - auto it = AuthenticateData.find(key); + yandex::cloud::priv::servicecontrol::v1::AuthenticateResponse* response) override + { + TString key; + if (request->has_signature()) { + key = request->signature().v4_parameters().service(); + } else { + key = request->iam_token(); + } + auto it = AuthenticateData.find(key); if (it != AuthenticateData.end()) { response->CopyFrom(it->second.Response); - CheckRequestId(ctx, it->second, key); + CheckRequestId(ctx, it->second, key); return it->second.Status; } else { return grpc::Status(grpc::StatusCode::PERMISSION_DENIED, "Permission Denied"); diff --git a/ydb/core/testlib/service_mocks/database_service_mock.h b/ydb/core/testlib/service_mocks/database_service_mock.h index aa4e454c90..e232c22d36 100644 --- a/ydb/core/testlib/service_mocks/database_service_mock.h +++ b/ydb/core/testlib/service_mocks/database_service_mock.h @@ -1,73 +1,73 @@ -#pragma once - -#include <cloud/bitbucket/private-api/yandex/cloud/priv/ydb/v1/database_service.grpc.pb.h> - -class TDatabaseServiceMock : public yandex::cloud::priv::ydb::v1::DatabaseService::Service { -public: - THashMap<TString, yandex::cloud::priv::ydb::v1::Database> PersistentDatabases; - THashMap<TString, yandex::cloud::priv::ydb::v1::Database> TemporaryDatabases; - TString Identity; - - TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { - if (!Identity.empty()) { - auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); - UNIT_ASSERT_C(reqIdBegin != reqIdEnd, "Authorization is expected."); - if (Identity != TStringBuf(reqIdBegin->second.cbegin(), reqIdBegin->second.cend())) { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, - TStringBuilder() << "Access for user " << Identity << " is forbidden"); - } - } - - return Nothing(); - } - - virtual grpc::Status GetByPath(grpc::ServerContext* context, - const yandex::cloud::priv::ydb::v1::GetDatabaseByPathRequest* request, - yandex::cloud::priv::ydb::v1::Database* response) override - { - auto status = CheckAuthorization(context); - auto parts = SplitString(request->path(), "/"); - Y_ENSURE(parts.size() >= 3); - TString canonizedPath = "/" + JoinRange("/", parts.begin(), parts.begin() + 3); - - if (auto itPersistent = PersistentDatabases.find(canonizedPath); itPersistent != PersistentDatabases.end()) { - *response = itPersistent->second; - return grpc::Status::OK; - } else { - auto it = TemporaryDatabases.find(canonizedPath); - if (it == TemporaryDatabases.end()) { - return grpc::Status(grpc::StatusCode::NOT_FOUND, TStringBuilder() << " database with name " << request->path() << " not found"); - } else { - *response = it->second; - return grpc::Status::OK; - } - } - } - - virtual grpc::Status ListAll(grpc::ServerContext* context, - const yandex::cloud::priv::ydb::v1::ListAllDatabasesRequest* request, - yandex::cloud::priv::ydb::v1::ListAllDatabasesResponse* response) override - { - auto status = CheckAuthorization(context); - if (status.Defined()) { - return *status; - } - - if (PersistentDatabases.empty()) { - return grpc::Status::OK; - } - auto it = PersistentDatabases.begin(); - if (!request->page_token().empty()) { - it = PersistentDatabases.find(request->page_token()); - } - Y_ENSURE(it != PersistentDatabases.end()); - *response->add_databases() = it->second; - it++; - if (it != PersistentDatabases.end()) { - response->set_next_page_token(it->first); - } - return grpc::Status::OK; - } - -}; - +#pragma once + +#include <cloud/bitbucket/private-api/yandex/cloud/priv/ydb/v1/database_service.grpc.pb.h> + +class TDatabaseServiceMock : public yandex::cloud::priv::ydb::v1::DatabaseService::Service { +public: + THashMap<TString, yandex::cloud::priv::ydb::v1::Database> PersistentDatabases; + THashMap<TString, yandex::cloud::priv::ydb::v1::Database> TemporaryDatabases; + TString Identity; + + TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { + if (!Identity.empty()) { + auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); + UNIT_ASSERT_C(reqIdBegin != reqIdEnd, "Authorization is expected."); + if (Identity != TStringBuf(reqIdBegin->second.cbegin(), reqIdBegin->second.cend())) { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, + TStringBuilder() << "Access for user " << Identity << " is forbidden"); + } + } + + return Nothing(); + } + + virtual grpc::Status GetByPath(grpc::ServerContext* context, + const yandex::cloud::priv::ydb::v1::GetDatabaseByPathRequest* request, + yandex::cloud::priv::ydb::v1::Database* response) override + { + auto status = CheckAuthorization(context); + auto parts = SplitString(request->path(), "/"); + Y_ENSURE(parts.size() >= 3); + TString canonizedPath = "/" + JoinRange("/", parts.begin(), parts.begin() + 3); + + if (auto itPersistent = PersistentDatabases.find(canonizedPath); itPersistent != PersistentDatabases.end()) { + *response = itPersistent->second; + return grpc::Status::OK; + } else { + auto it = TemporaryDatabases.find(canonizedPath); + if (it == TemporaryDatabases.end()) { + return grpc::Status(grpc::StatusCode::NOT_FOUND, TStringBuilder() << " database with name " << request->path() << " not found"); + } else { + *response = it->second; + return grpc::Status::OK; + } + } + } + + virtual grpc::Status ListAll(grpc::ServerContext* context, + const yandex::cloud::priv::ydb::v1::ListAllDatabasesRequest* request, + yandex::cloud::priv::ydb::v1::ListAllDatabasesResponse* response) override + { + auto status = CheckAuthorization(context); + if (status.Defined()) { + return *status; + } + + if (PersistentDatabases.empty()) { + return grpc::Status::OK; + } + auto it = PersistentDatabases.begin(); + if (!request->page_token().empty()) { + it = PersistentDatabases.find(request->page_token()); + } + Y_ENSURE(it != PersistentDatabases.end()); + *response->add_databases() = it->second; + it++; + if (it != PersistentDatabases.end()) { + response->set_next_page_token(it->first); + } + return grpc::Status::OK; + } + +}; + diff --git a/ydb/core/testlib/service_mocks/datastreams_service_mock.h b/ydb/core/testlib/service_mocks/datastreams_service_mock.h index c5b2105c88..3f65899cfc 100644 --- a/ydb/core/testlib/service_mocks/datastreams_service_mock.h +++ b/ydb/core/testlib/service_mocks/datastreams_service_mock.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h> -#include "access_service_mock.h" -#include "datastreams_service_mock.h" - -class TDataStreamsServiceMock : public Ydb::DataStreams::V1::DataStreamsService::Service { -public: - virtual grpc::Status PutRecords(grpc::ServerContext*, - const Ydb::DataStreams::V1::PutRecordsRequest* request, - Ydb::DataStreams::V1::PutRecordsResponse* response) override - { - Y_UNUSED(response); - for (const auto& record : request->records()) { - if (record.partition_key() == "Sleep") { - Sleep(TDuration::Seconds(3)); - } else if (record.partition_key() == "InvalidArgument") { - return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid argument"); - } else if (record.partition_key() == "Unauthenticated") { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access denied"); - } - } - response->mutable_operation()->set_status(Ydb::StatusIds::SUCCESS); - response->mutable_operation()->set_ready(true); - response->mutable_operation()->set_id("12345"); - - return grpc::Status::OK; - } - -}; - +#include "access_service_mock.h" +#include "datastreams_service_mock.h" + +class TDataStreamsServiceMock : public Ydb::DataStreams::V1::DataStreamsService::Service { +public: + virtual grpc::Status PutRecords(grpc::ServerContext*, + const Ydb::DataStreams::V1::PutRecordsRequest* request, + Ydb::DataStreams::V1::PutRecordsResponse* response) override + { + Y_UNUSED(response); + for (const auto& record : request->records()) { + if (record.partition_key() == "Sleep") { + Sleep(TDuration::Seconds(3)); + } else if (record.partition_key() == "InvalidArgument") { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid argument"); + } else if (record.partition_key() == "Unauthenticated") { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access denied"); + } + } + response->mutable_operation()->set_status(Ydb::StatusIds::SUCCESS); + response->mutable_operation()->set_ready(true); + response->mutable_operation()->set_id("12345"); + + return grpc::Status::OK; + } + +}; + diff --git a/ydb/core/testlib/service_mocks/iam_token_service_mock.h b/ydb/core/testlib/service_mocks/iam_token_service_mock.h index fa926a04d8..f8f6ea0f30 100644 --- a/ydb/core/testlib/service_mocks/iam_token_service_mock.h +++ b/ydb/core/testlib/service_mocks/iam_token_service_mock.h @@ -1,43 +1,43 @@ -#pragma once - -#include <yandex/cloud/priv/iam/v1/iam_token_service.grpc.pb.h> - -class TIamTokenServiceMock : public yandex::cloud::priv::iam::v1::IamTokenService::Service { -public: - THashMap<TString, yandex::cloud::priv::iam::v1::CreateIamTokenResponse> IamTokens; - TString Identity; - - TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { - if (!Identity.empty()) { - auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); - UNIT_ASSERT_C(reqIdBegin != reqIdEnd, "Authorization is expected."); +#pragma once + +#include <yandex/cloud/priv/iam/v1/iam_token_service.grpc.pb.h> + +class TIamTokenServiceMock : public yandex::cloud::priv::iam::v1::IamTokenService::Service { +public: + THashMap<TString, yandex::cloud::priv::iam::v1::CreateIamTokenResponse> IamTokens; + TString Identity; + + TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { + if (!Identity.empty()) { + auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); + UNIT_ASSERT_C(reqIdBegin != reqIdEnd, "Authorization is expected."); if (Identity != TStringBuf(reqIdBegin->second.cbegin(), reqIdBegin->second.cend())) { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, - TStringBuilder() << "Access for user " << Identity << " is forbidden"); - } - } - - return Nothing(); - } - - virtual grpc::Status CreateForServiceAccount(grpc::ServerContext* context, - const yandex::cloud::priv::iam::v1::CreateIamTokenForServiceAccountRequest* request, - yandex::cloud::priv::iam::v1::CreateIamTokenResponse* response) override - { - auto status = CheckAuthorization(context); - if (status.Defined()) { - return *status; - } - - TString id = request->service_account_id(); - auto it = IamTokens.find(id); - if (it != IamTokens.end()) { - response->CopyFrom(it->second); - return grpc::Status::OK; - } else { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Iam token not found"); - } - } - -}; - + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, + TStringBuilder() << "Access for user " << Identity << " is forbidden"); + } + } + + return Nothing(); + } + + virtual grpc::Status CreateForServiceAccount(grpc::ServerContext* context, + const yandex::cloud::priv::iam::v1::CreateIamTokenForServiceAccountRequest* request, + yandex::cloud::priv::iam::v1::CreateIamTokenResponse* response) override + { + auto status = CheckAuthorization(context); + if (status.Defined()) { + return *status; + } + + TString id = request->service_account_id(); + auto it = IamTokens.find(id); + if (it != IamTokens.end()) { + response->CopyFrom(it->second); + return grpc::Status::OK; + } else { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Iam token not found"); + } + } + +}; + diff --git a/ydb/core/testlib/service_mocks/service_account_service_mock.h b/ydb/core/testlib/service_mocks/service_account_service_mock.h index 14f592f03a..583396fc0f 100644 --- a/ydb/core/testlib/service_mocks/service_account_service_mock.h +++ b/ydb/core/testlib/service_mocks/service_account_service_mock.h @@ -5,34 +5,34 @@ class TServiceAccountServiceMock : public yandex::cloud::priv::iam::v1::ServiceAccountService::Service { public: THashMap<TString, yandex::cloud::priv::iam::v1::ServiceAccount> ServiceAccountData; - THashMap<TString, yandex::cloud::priv::iam::v1::IamToken> IamTokens; - TString Identity; + THashMap<TString, yandex::cloud::priv::iam::v1::IamToken> IamTokens; + TString Identity; - TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { - if (!Identity.empty()) { - auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); - if (reqIdBegin == reqIdEnd) { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, - TStringBuilder() << "Authorization data is not provided"); + TMaybe<grpc::Status> CheckAuthorization(grpc::ServerContext* context) { + if (!Identity.empty()) { + auto[reqIdBegin, reqIdEnd] = context->client_metadata().equal_range("authorization"); + if (reqIdBegin == reqIdEnd) { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, + TStringBuilder() << "Authorization data is not provided"); } else if (Identity != TStringBuf(reqIdBegin->second.cbegin(), reqIdBegin->second.cend())) { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, - TStringBuilder() << "Access for user " << Identity << " is forbidden"); - } - } - - return Nothing(); - } - - virtual grpc::Status Get(grpc::ServerContext* context, + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, + TStringBuilder() << "Access for user " << Identity << " is forbidden"); + } + } + + return Nothing(); + } + + virtual grpc::Status Get(grpc::ServerContext* context, const yandex::cloud::priv::iam::v1::GetServiceAccountRequest* request, - yandex::cloud::priv::iam::v1::ServiceAccount* response) override - { - auto status = CheckAuthorization(context); - if (status.Defined()) { - return *status; - } - - TString id = request->service_account_id(); + yandex::cloud::priv::iam::v1::ServiceAccount* response) override + { + auto status = CheckAuthorization(context); + if (status.Defined()) { + return *status; + } + + TString id = request->service_account_id(); auto it = ServiceAccountData.find(id); if (it != ServiceAccountData.end()) { response->CopyFrom(it->second); @@ -41,25 +41,25 @@ public: return grpc::Status(grpc::StatusCode::NOT_FOUND, "Not Found"); } } - - virtual grpc::Status IssueToken(grpc::ServerContext* context, - const yandex::cloud::priv::iam::v1::IssueTokenRequest* request, - yandex::cloud::priv::iam::v1::IamToken* response) override - { - auto status = CheckAuthorization(context); - if (status.Defined()) { - return *status; - } - - TString id = request->service_account_id(); - auto it = IamTokens.find(id); - if (it != IamTokens.end()) { - response->CopyFrom(it->second); - return grpc::Status::OK; - } else { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Iam token not found"); - } - } - + + virtual grpc::Status IssueToken(grpc::ServerContext* context, + const yandex::cloud::priv::iam::v1::IssueTokenRequest* request, + yandex::cloud::priv::iam::v1::IamToken* response) override + { + auto status = CheckAuthorization(context); + if (status.Defined()) { + return *status; + } + + TString id = request->service_account_id(); + auto it = IamTokens.find(id); + if (it != IamTokens.end()) { + response->CopyFrom(it->second); + return grpc::Status::OK; + } else { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Iam token not found"); + } + } + }; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 1afac63910..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -273,7 +273,7 @@ namespace Tests { } desc->ServedDatabases.insert(desc->ServedDatabases.end(), rootDomains.begin(), rootDomains.end()); - TVector<TString> grpcServices = {"yql", "clickhouse_internal", "datastreams", "table_service", "scripting", "experimental", "discovery", "pqcd", "pq", "pqv1" }; + TVector<TString> grpcServices = {"yql", "clickhouse_internal", "datastreams", "table_service", "scripting", "experimental", "discovery", "pqcd", "pq", "pqv1" }; desc->ServedServices.insert(desc->ServedServices.end(), grpcServices.begin(), grpcServices.end()); system->Register(NGRpcService::CreateGrpcEndpointPublishActor(desc.Get()), TMailboxType::ReadAsFilled, appData.UserPoolId); @@ -316,7 +316,7 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcYdbS3InternalService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NQuoter::TRateLimiterGRpcService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbLongTxService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcDataStreamsService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcDataStreamsService(system, counters, grpcRequestProxyId)); if (Settings->EnableYq) { GRpcServer->AddService(new NGRpcService::TGRpcYandexQueryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId)); diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h index a8d80c1c3d..f059923250 100644 --- a/ydb/core/testlib/test_pq_client.h +++ b/ydb/core/testlib/test_pq_client.h @@ -462,15 +462,15 @@ struct TPQTestClusterInfo { ui64 Weight = 1000; }; -static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = { - {"dc1", {"localhost", true}}, - {"dc2", {"dc2.logbroker.yandex.net", true}} -}; - -static THashMap<TString, TPQTestClusterInfo> CLUSTERS_LIST_ONE_DC = { - {"dc1", {"localhost", true}} -}; - +static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = { + {"dc1", {"localhost", true}}, + {"dc2", {"dc2.logbroker.yandex.net", true}} +}; + +static THashMap<TString, TPQTestClusterInfo> CLUSTERS_LIST_ONE_DC = { + {"dc1", {"localhost", true}} +}; + class TFlatMsgBusPQClient : public NFlatTests::TFlatMsgBusClient { private: static constexpr ui32 FlatDomain = 0; @@ -523,12 +523,12 @@ public: , GRpcPort(grpc) , Kikimr(GetClientConfig()) { - auto driverConfig = NYdb::TDriverConfig() + auto driverConfig = NYdb::TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << GRpcPort) .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - if (databaseName) { - driverConfig.SetDatabase(*databaseName); - } + if (databaseName) { + driverConfig.SetDatabase(*databaseName); + } Driver.Reset(MakeHolder<NYdb::TDriver>(driverConfig)); } @@ -563,9 +563,9 @@ public: } void InitSourceIds(const TString& path = DEFAULT_SRC_IDS_PATH) { - TFsPath fsPath(path); - CreateTable(fsPath.Dirname(), - "Name: \"" + fsPath.Basename() + "\"" + TFsPath fsPath(path); + CreateTable(fsPath.Dirname(), + "Name: \"" + fsPath.Basename() + "\"" "Columns { Name: \"Hash\" Type: \"Uint32\"}" "Columns { Name: \"SourceId\" Type: \"Utf8\"}" "Columns { Name: \"Topic\" Type: \"Utf8\"}" @@ -576,12 +576,12 @@ public: ); } - void InsertSourceId(ui32 hash, TString sourceId, ui64 accessTime, const TString& path = "/Root/PQ/SourceIdMeta2") { + void InsertSourceId(ui32 hash, TString sourceId, ui64 accessTime, const TString& path = "/Root/PQ/SourceIdMeta2") { TString query = "DECLARE $Hash AS Uint32; " "DECLARE $SourceId AS Utf8; " "DECLARE $AccessTime AS Uint64; " - "UPSERT INTO [" + path + "] (Hash, SourceId, Topic, Partition, CreateTime, AccessTime) " + "UPSERT INTO [" + path + "] (Hash, SourceId, Topic, Partition, CreateTime, AccessTime) " "VALUES($Hash, $SourceId, \"1\", 0, 0, $AccessTime); "; NYdb::TParamsBuilder builder; @@ -594,8 +594,8 @@ public: RunYqlDataQueryWithParams(query, params); } - THashMap<TString, TInstant> ListSourceIds(const TString& path = "/Root/PQ/SourceIdMeta2") { - auto result = RunYqlDataQuery("SELECT SourceId, AccessTime FROM [" + path + "];"); + THashMap<TString, TInstant> ListSourceIds(const TString& path = "/Root/PQ/SourceIdMeta2") { + auto result = RunYqlDataQuery("SELECT SourceId, AccessTime FROM [" + path + "];"); NYdb::TResultSetParser parser(*result); THashMap<TString, TInstant> sourceIds; while(parser.TryNextRow()) { @@ -606,7 +606,7 @@ public: return sourceIds; } - void InitDCs(THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST, const TString& localCluster = TString()) { + void InitDCs(THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST, const TString& localCluster = TString()) { MkDir("/Root/PQ", "Config"); MkDir("/Root/PQ/Config", "V2"); RunYqlSchemeQuery(R"___( @@ -896,25 +896,25 @@ public: const TInstant start = TInstant::Now(); THolder<NMsgBusProxy::TBusPersQueue> request = createRequest.GetRequest(); - + ui32 prevVersion = TopicCreated(createRequest.Topic); TAutoPtr<NBus::TBusMessage> reply; const NMsgBusProxy::TBusResponse* response = SendAndGetReply(request, reply); UNIT_ASSERT(response); UNIT_ASSERT_VALUES_EQUAL_C((ui32)response->Record.GetErrorCode(), (ui32)NPersQueue::NErrorCode::OK, "proxy failure"); - + AddTopic(createRequest.Topic); while (doWait && TopicRealCreated(createRequest.Topic) != prevVersion + 1) { Sleep(TDuration::MilliSeconds(500)); UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); - } + } while (doWait && TopicCreated(createRequest.Topic, prevVersion) != prevVersion + 1) { Sleep(TDuration::MilliSeconds(500)); UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); } - } - + } + void CreateTopic( const TString& name, ui32 nParts, diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index a6ff818f07..137fa829f3 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -22,13 +22,13 @@ SRCS( tenant_runtime.h test_client.cpp test_client.h - service_mocks/access_service_mock.h - service_mocks/datastreams_service_mock.h - service_mocks/iam_token_service_mock.h - service_mocks/user_account_service_mock.h - service_mocks/database_service_mock.h - service_mocks/folder_service_mock.h - service_mocks/service_account_service_mock.h + service_mocks/access_service_mock.h + service_mocks/datastreams_service_mock.h + service_mocks/iam_token_service_mock.h + service_mocks/user_account_service_mock.h + service_mocks/database_service_mock.h + service_mocks/folder_service_mock.h + service_mocks/service_account_service_mock.h ) PEERDIR( diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_common.h index 8d1fd0d3ca..5a3da32cad 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.h @@ -1017,8 +1017,8 @@ public: bool parseOk = ParseFromStringNoSizeLimit(*event->Record.MutableTabletConfig(), *tabletConfig); Y_VERIFY(parseOk); } - event->Record.MutableTabletConfig()->SetTopicName(topicName); - event->Record.MutableTabletConfig()->SetTopicPath(TPath::Init(txState->TargetPathId, context.SS).PathString()); + event->Record.MutableTabletConfig()->SetTopicName(topicName); + event->Record.MutableTabletConfig()->SetTopicPath(TPath::Init(txState->TargetPathId, context.SS).PathString()); event->Record.MutableTabletConfig()->MutablePartitionConfig()->SetTotalPartitions(pqGroup->AlterData ? pqGroup->AlterData->TotalGroupCount : pqGroup->TotalGroupCount); event->Record.MutableTabletConfig()->SetVersion(pqGroup->AlterVersion + 1); @@ -1070,7 +1070,7 @@ public: event->Record.SetTopicName(topicName); event->Record.SetPathId(txState->TargetPathId.LocalPathId); - event->Record.SetPath(TPath::Init(txState->TargetPathId, context.SS).PathString()); + event->Record.SetPath(TPath::Init(txState->TargetPathId, context.SS).PathString()); event->Record.SetPartitionPerTablet(pqGroup->AlterData ? pqGroup->AlterData->MaxPartsPerTablet : pqGroup->MaxPartsPerTablet); event->Record.SetSchemeShardId(context.SS->TabletID()); diff --git a/ydb/core/viewer/json_labeledcounters.h b/ydb/core/viewer/json_labeledcounters.h index 4c2e154934..6fce18b32b 100644 --- a/ydb/core/viewer/json_labeledcounters.h +++ b/ydb/core/viewer/json_labeledcounters.h @@ -47,7 +47,7 @@ public: Topic = NPersQueue::ConvertNewTopicName(params.Get("topic")); if (Topic.empty()) Topic = "*"; - Consumer = NPersQueue::ConvertNewConsumerName(params.Get("consumer"), ctx); + Consumer = NPersQueue::ConvertNewConsumerName(params.Get("consumer"), ctx); DC = params.Get("dc"); if (DC.empty()) DC = "*"; diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h index 611e948081..b98dfc2632 100644 --- a/ydb/core/viewer/json_pqconsumerinfo.h +++ b/ydb/core/viewer/json_pqconsumerinfo.h @@ -50,7 +50,7 @@ public: Client = params.Get("client"); if (Version >= 3) { Topic = "rt3." + DC + "--" + NPersQueue::ConvertNewTopicName(Topic); - Client = NPersQueue::ConvertNewConsumerName(Client, ctx); + Client = NPersQueue::ConvertNewConsumerName(Client, ctx); } else { size_t pos = Topic.rfind('/'); if (pos != TString::npos) { diff --git a/ydb/core/ymq/actor/events.h b/ydb/core/ymq/actor/events.h index 02178d4f14..9fff62c7cf 100644 --- a/ydb/core/ymq/actor/events.h +++ b/ydb/core/ymq/actor/events.h @@ -6,9 +6,9 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/protos/config.pb.h> #include <ydb/core/protos/msgbus.pb.h> - + #include <ydb/library/http_proxy/error/error.h> - + #include <ydb/core/ymq/base/action.h> #include <ydb/core/ymq/base/counters.h> #include <ydb/core/ymq/base/processed_request_attributes.h> diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp index 70cd812a31..5d0963bfb2 100644 --- a/ydb/core/ymq/http/http.cpp +++ b/ydb/core/ymq/http/http.cpp @@ -1,6 +1,6 @@ #include "http.h" #include "xml.h" - + #include <ydb/core/protos/services.pb.h> #include <ydb/library/http_proxy/authorization/auth_helpers.h> #include <ydb/core/ymq/actor/actor.h> @@ -1035,7 +1035,7 @@ void TAsyncHttpServer::OnException() { } THttpServerOptions TAsyncHttpServer::MakeHttpServerOptions(const NKikimrConfig::TSqsConfig& config) { - const auto& cfg = config.GetHttpServerConfig(); + const auto& cfg = config.GetHttpServerConfig(); THttpServerOptions options; options.SetThreads(cfg.GetThreads()); options.SetPort(cfg.GetPort()); diff --git a/ydb/core/ymq/http/http.h b/ydb/core/ymq/http/http.h index e7ee1785a3..01a2b0edc4 100644 --- a/ydb/core/ymq/http/http.h +++ b/ydb/core/ymq/http/http.h @@ -8,7 +8,7 @@ #include <ydb/library/http_proxy/authorization/signature.h> #include <ydb/core/ymq/base/counters.h> - + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/http/server/http.h> @@ -117,7 +117,7 @@ private: TString SecurityToken_; TString IamToken_; TString FolderId_; - TString ApiMethod_; + TString ApiMethod_; THolder<TAwsRequestSignV4> AwsSignature_; diff --git a/ydb/library/http_proxy/authorization/ut/auth_helpers_ut.cpp b/ydb/library/http_proxy/authorization/ut/auth_helpers_ut.cpp index 8cf98cf201..0fb753120a 100644 --- a/ydb/library/http_proxy/authorization/ut/auth_helpers_ut.cpp +++ b/ydb/library/http_proxy/authorization/ut/auth_helpers_ut.cpp @@ -1,7 +1,7 @@ #include <ydb/library/http_proxy/authorization/auth_helpers.h> #include <ydb/library/http_proxy/error/error.h> - + #include <library/cpp/testing/unittest/registar.h> namespace NKikimr::NSQS { diff --git a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp index e285a159a4..062ac52146 100644 --- a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp +++ b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp @@ -1,5 +1,5 @@ #include <ydb/library/http_proxy/authorization/signature.h> - + #include <library/cpp/testing/unittest/registar.h> #include <ydb/library/http_proxy/error/error.h> diff --git a/ydb/library/http_proxy/authorization/ut/ya.make b/ydb/library/http_proxy/authorization/ut/ya.make index 36b9a0e17a..4a127f6e8b 100644 --- a/ydb/library/http_proxy/authorization/ut/ya.make +++ b/ydb/library/http_proxy/authorization/ut/ya.make @@ -1,17 +1,17 @@ -OWNER( +OWNER( g:kikimr - g:sqs -) - + g:sqs +) + UNITTEST_FOR(ydb/library/http_proxy/authorization) - + PEERDIR( ydb/library/http_proxy/error ) -SRCS( +SRCS( auth_helpers_ut.cpp signature_ut.cpp -) - -END() +) + +END() diff --git a/ydb/library/http_proxy/authorization/ya.make b/ydb/library/http_proxy/authorization/ya.make index f31ceda0d0..0000c5f2ae 100644 --- a/ydb/library/http_proxy/authorization/ya.make +++ b/ydb/library/http_proxy/authorization/ya.make @@ -1,24 +1,24 @@ -OWNER( +OWNER( g:kikimr - g:sqs -) - -LIBRARY() - -SRCS( - auth_helpers.cpp - signature.cpp -) - -PEERDIR( - contrib/libs/openssl - library/cpp/cgiparam - library/cpp/http/io - library/cpp/http/misc + g:sqs +) + +LIBRARY() + +SRCS( + auth_helpers.cpp + signature.cpp +) + +PEERDIR( + contrib/libs/openssl + library/cpp/cgiparam + library/cpp/http/io + library/cpp/http/misc ydb/library/http_proxy/error -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/http_proxy/error/ya.make b/ydb/library/http_proxy/error/ya.make index 27028e81de..9a8e5ce99b 100644 --- a/ydb/library/http_proxy/error/ya.make +++ b/ydb/library/http_proxy/error/ya.make @@ -1,12 +1,12 @@ -OWNER( +OWNER( g:kikimr - g:sqs -) - -LIBRARY() - -SRCS( - error.cpp -) - -END() + g:sqs +) + +LIBRARY() + +SRCS( + error.cpp +) + +END() diff --git a/ydb/library/http_proxy/ya.make b/ydb/library/http_proxy/ya.make index efc9bcfa88..6137fcc859 100644 --- a/ydb/library/http_proxy/ya.make +++ b/ydb/library/http_proxy/ya.make @@ -1,4 +1,4 @@ -RECURSE( - authorization +RECURSE( + authorization error -) +) diff --git a/ydb/library/naming_conventions/naming_conventions.cpp b/ydb/library/naming_conventions/naming_conventions.cpp index 76722af186..36c8403272 100644 --- a/ydb/library/naming_conventions/naming_conventions.cpp +++ b/ydb/library/naming_conventions/naming_conventions.cpp @@ -1,36 +1,36 @@ -#include "naming_conventions.h" - -#include <util/string/builder.h> - -namespace NKikimr::NNaming { - - TString SnakeToCamelCase(const TString& name) { - TStringBuilder builder; - bool nextUpper = true; - for (char c : name) { - if (c == '_') { - nextUpper = true; - } else { - builder << (char)(nextUpper ? toupper(c) : c); - nextUpper = false; - } - } - return builder; - } - - TString CamelToSnakeCase(const TString& name) { - TStringBuilder builder; - bool first = true; - for (char c : name) { - if (isupper(c)) { - if (!first) { - builder << "_"; - } - } - first = false; - builder << (char)tolower(c); - } - return builder; - } - -}
\ No newline at end of file +#include "naming_conventions.h" + +#include <util/string/builder.h> + +namespace NKikimr::NNaming { + + TString SnakeToCamelCase(const TString& name) { + TStringBuilder builder; + bool nextUpper = true; + for (char c : name) { + if (c == '_') { + nextUpper = true; + } else { + builder << (char)(nextUpper ? toupper(c) : c); + nextUpper = false; + } + } + return builder; + } + + TString CamelToSnakeCase(const TString& name) { + TStringBuilder builder; + bool first = true; + for (char c : name) { + if (isupper(c)) { + if (!first) { + builder << "_"; + } + } + first = false; + builder << (char)tolower(c); + } + return builder; + } + +}
\ No newline at end of file diff --git a/ydb/library/naming_conventions/naming_conventions.h b/ydb/library/naming_conventions/naming_conventions.h index ab03467759..dd7f7f6fc2 100644 --- a/ydb/library/naming_conventions/naming_conventions.h +++ b/ydb/library/naming_conventions/naming_conventions.h @@ -1,10 +1,10 @@ -#pragma once - -#include <util/generic/string.h> - -namespace NKikimr::NNaming { - - TString SnakeToCamelCase(const TString& name); - TString CamelToSnakeCase(const TString& name); - -}
\ No newline at end of file +#pragma once + +#include <util/generic/string.h> + +namespace NKikimr::NNaming { + + TString SnakeToCamelCase(const TString& name); + TString CamelToSnakeCase(const TString& name); + +}
\ No newline at end of file diff --git a/ydb/library/naming_conventions/ut/naming_conventions_ut.cpp b/ydb/library/naming_conventions/ut/naming_conventions_ut.cpp index fd90b2e992..9f86cfe8ea 100644 --- a/ydb/library/naming_conventions/ut/naming_conventions_ut.cpp +++ b/ydb/library/naming_conventions/ut/naming_conventions_ut.cpp @@ -1,24 +1,24 @@ #include <ydb/library/naming_conventions/naming_conventions.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/format.h> -#include <util/stream/str.h> - -using namespace NKikimr::NNaming; - -Y_UNIT_TEST_SUITE(NamingConventionsSuite) { - - Y_UNIT_TEST(TestCamelToSnake) { - UNIT_ASSERT_EQUAL("camel_case", CamelToSnakeCase("CamelCase")); - UNIT_ASSERT_EQUAL("snakesnake", CamelToSnakeCase("snakesnake")); - UNIT_ASSERT_EQUAL("b_f_g", CamelToSnakeCase("BFG")); - } - - Y_UNIT_TEST(TestSnakeToCamel) { - UNIT_ASSERT_EQUAL("CamelCase", SnakeToCamelCase("camel_case")); - UNIT_ASSERT_EQUAL("Snakesnake", SnakeToCamelCase("snakesnake")); - UNIT_ASSERT_EQUAL("Bfg", SnakeToCamelCase("bfg_")); - } - -} + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/format.h> +#include <util/stream/str.h> + +using namespace NKikimr::NNaming; + +Y_UNIT_TEST_SUITE(NamingConventionsSuite) { + + Y_UNIT_TEST(TestCamelToSnake) { + UNIT_ASSERT_EQUAL("camel_case", CamelToSnakeCase("CamelCase")); + UNIT_ASSERT_EQUAL("snakesnake", CamelToSnakeCase("snakesnake")); + UNIT_ASSERT_EQUAL("b_f_g", CamelToSnakeCase("BFG")); + } + + Y_UNIT_TEST(TestSnakeToCamel) { + UNIT_ASSERT_EQUAL("CamelCase", SnakeToCamelCase("camel_case")); + UNIT_ASSERT_EQUAL("Snakesnake", SnakeToCamelCase("snakesnake")); + UNIT_ASSERT_EQUAL("Bfg", SnakeToCamelCase("bfg_")); + } + +} diff --git a/ydb/library/naming_conventions/ut/ya.make b/ydb/library/naming_conventions/ut/ya.make index dc00b5d303..48990837c9 100644 --- a/ydb/library/naming_conventions/ut/ya.make +++ b/ydb/library/naming_conventions/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(ydb/library/naming_conventions) - -OWNER(g:kikimr) - -SRCS( - naming_conventions_ut.cpp -) - -END() + +OWNER(g:kikimr) + +SRCS( + naming_conventions_ut.cpp +) + +END() diff --git a/ydb/library/naming_conventions/ya.make b/ydb/library/naming_conventions/ya.make index d007b6722b..8e3e9c6a30 100644 --- a/ydb/library/naming_conventions/ya.make +++ b/ydb/library/naming_conventions/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - naming_conventions.cpp -) - -END() +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + naming_conventions.cpp +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/persqueue/topic_parser/topic_parser.cpp b/ydb/library/persqueue/topic_parser/topic_parser.cpp index b25edf060a..a9598b25b2 100644 --- a/ydb/library/persqueue/topic_parser/topic_parser.cpp +++ b/ydb/library/persqueue/topic_parser/topic_parser.cpp @@ -1,49 +1,49 @@ #include "topic_parser.h" #include <ydb/core/base/appdata.h> - -#include <util/folder/path.h> - + +#include <util/folder/path.h> + namespace NPersQueue { -namespace { - TString FullPath(const TMaybe<TString> &database, const TString &path) { +namespace { + TString FullPath(const TMaybe<TString> &database, const TString &path) { if (database.Defined() && !path.StartsWith(*database) && !path.Contains('\0')) { try { return (TFsPath(*database) / path).GetPath(); } catch(...) { return path; } - } else { - return path; - } - } -} - -TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath) { - if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - return FullPath(database, topicPath); - } else { - return topicPath; - } -} - -TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx) { - if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - return consumer; - } else { - return ConvertNewConsumerName(consumer); - } -} - -TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx) { - if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - return consumer; - } else { - return ConvertOldConsumerName(consumer); - } -} - + } else { + return path; + } + } +} + +TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath) { + if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + return FullPath(database, topicPath); + } else { + return topicPath; + } +} + +TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx) { + if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + return consumer; + } else { + return ConvertNewConsumerName(consumer); + } +} + +TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx) { + if (NKikimr::AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + return consumer; + } else { + return ConvertOldConsumerName(consumer); + } +} + TString MakeConsumerPath(const TString& consumer) { TStringBuilder res; res.reserve(consumer.size()); diff --git a/ydb/library/persqueue/topic_parser/topic_parser.h b/ydb/library/persqueue/topic_parser/topic_parser.h index 2a8d548e25..eb5a69cc37 100644 --- a/ydb/library/persqueue/topic_parser/topic_parser.h +++ b/ydb/library/persqueue/topic_parser/topic_parser.h @@ -1,7 +1,7 @@ #pragma once -#include <library/cpp/actors/core/actor.h> - +#include <library/cpp/actors/core/actor.h> + #include <util/generic/string.h> #include <util/generic/hash.h> #include <util/string/builder.h> @@ -10,9 +10,9 @@ namespace NPersQueue { -TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath); -TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx); -TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx); +TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath); +TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx); +TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx); TString MakeConsumerPath(const TString& consumer); diff --git a/ydb/public/api/grpc/draft/ya.make b/ydb/public/api/grpc/draft/ya.make index b3c53edcf8..f63be96521 100644 --- a/ydb/public/api/grpc/draft/ya.make +++ b/ydb/public/api/grpc/draft/ya.make @@ -15,7 +15,7 @@ SRCS( dummy.proto ydb_clickhouse_internal_v1.proto ydb_persqueue_v1.proto - ydb_datastreams_v1.proto + ydb_datastreams_v1.proto ydb_experimental_v1.proto ydb_s3_internal_v1.proto ydb_long_tx_v1.proto diff --git a/ydb/public/api/grpc/draft/ydb_datastreams_v1.proto b/ydb/public/api/grpc/draft/ydb_datastreams_v1.proto index 81c08d5cb4..069ee7ad4c 100644 --- a/ydb/public/api/grpc/draft/ydb_datastreams_v1.proto +++ b/ydb/public/api/grpc/draft/ydb_datastreams_v1.proto @@ -1,49 +1,49 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb.DataStreams.V1; - +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.DataStreams.V1; + import "ydb/public/api/protos/draft/datastreams.proto"; - - -option java_package = "com.yandex.ydb.datastreams.v1"; - - -service DataStreamsService { - // Basic stream manipulation methods - rpc CreateStream(CreateStreamRequest) returns (CreateStreamResponse); - rpc ListStreams(ListStreamsRequest) returns (ListStreamsResponse); - rpc DeleteStream(DeleteStreamRequest) returns (DeleteStreamResponse); - rpc DescribeStream(DescribeStreamRequest) returns (DescribeStreamResponse); - rpc ListShards(ListShardsRequest) returns (ListShardsResponse); - rpc SetWriteQuota(SetWriteQuotaRequest) returns (SetWriteQuotaResponse); - rpc UpdateStream(UpdateStreamRequest) returns (UpdateStreamResponse); - - // Data methods - rpc PutRecord(PutRecordRequest) returns (PutRecordResponse); - rpc PutRecords(PutRecordsRequest) returns (PutRecordsResponse); - rpc GetRecords(GetRecordsRequest) returns (GetRecordsResponse); - rpc GetShardIterator(GetShardIteratorRequest) returns (GetShardIteratorResponse); - rpc SubscribeToShard(SubscribeToShardRequest) returns (stream SubscribeToShardResponse); - rpc DescribeLimits(DescribeLimitsRequest) returns (DescribeLimitsResponse); - rpc DescribeStreamSummary(DescribeStreamSummaryRequest) returns (DescribeStreamSummaryResponse); - rpc DecreaseStreamRetentionPeriod(DecreaseStreamRetentionPeriodRequest) returns (DecreaseStreamRetentionPeriodResponse); - rpc IncreaseStreamRetentionPeriod(IncreaseStreamRetentionPeriodRequest) returns (IncreaseStreamRetentionPeriodResponse); - rpc UpdateShardCount(UpdateShardCountRequest) returns (UpdateShardCountResponse); - // stream consumer methods - rpc RegisterStreamConsumer(RegisterStreamConsumerRequest) returns (RegisterStreamConsumerResponse); - rpc DeregisterStreamConsumer(DeregisterStreamConsumerRequest) returns (DeregisterStreamConsumerResponse); - rpc DescribeStreamConsumer(DescribeStreamConsumerRequest) returns (DescribeStreamConsumerResponse); - rpc ListStreamConsumers(ListStreamConsumersRequest) returns (ListStreamConsumersResponse); - - // Least priority methods from API. - rpc AddTagsToStream(AddTagsToStreamRequest) returns (AddTagsToStreamResponse); - rpc DisableEnhancedMonitoring(DisableEnhancedMonitoringRequest) returns (DisableEnhancedMonitoringResponse); - rpc EnableEnhancedMonitoring(EnableEnhancedMonitoringRequest) returns (EnableEnhancedMonitoringResponse); - rpc ListTagsForStream(ListTagsForStreamRequest) returns (ListTagsForStreamResponse); - rpc MergeShards(MergeShardsRequest) returns (MergeShardsResponse); - rpc RemoveTagsFromStream(RemoveTagsFromStreamRequest) returns (RemoveTagsFromStreamResponse); - rpc SplitShard(SplitShardRequest) returns (SplitShardResponse); - rpc StartStreamEncryption(StartStreamEncryptionRequest) returns (StartStreamEncryptionResponse); - rpc StopStreamEncryption(StopStreamEncryptionRequest) returns (StopStreamEncryptionResponse); -} + + +option java_package = "com.yandex.ydb.datastreams.v1"; + + +service DataStreamsService { + // Basic stream manipulation methods + rpc CreateStream(CreateStreamRequest) returns (CreateStreamResponse); + rpc ListStreams(ListStreamsRequest) returns (ListStreamsResponse); + rpc DeleteStream(DeleteStreamRequest) returns (DeleteStreamResponse); + rpc DescribeStream(DescribeStreamRequest) returns (DescribeStreamResponse); + rpc ListShards(ListShardsRequest) returns (ListShardsResponse); + rpc SetWriteQuota(SetWriteQuotaRequest) returns (SetWriteQuotaResponse); + rpc UpdateStream(UpdateStreamRequest) returns (UpdateStreamResponse); + + // Data methods + rpc PutRecord(PutRecordRequest) returns (PutRecordResponse); + rpc PutRecords(PutRecordsRequest) returns (PutRecordsResponse); + rpc GetRecords(GetRecordsRequest) returns (GetRecordsResponse); + rpc GetShardIterator(GetShardIteratorRequest) returns (GetShardIteratorResponse); + rpc SubscribeToShard(SubscribeToShardRequest) returns (stream SubscribeToShardResponse); + rpc DescribeLimits(DescribeLimitsRequest) returns (DescribeLimitsResponse); + rpc DescribeStreamSummary(DescribeStreamSummaryRequest) returns (DescribeStreamSummaryResponse); + rpc DecreaseStreamRetentionPeriod(DecreaseStreamRetentionPeriodRequest) returns (DecreaseStreamRetentionPeriodResponse); + rpc IncreaseStreamRetentionPeriod(IncreaseStreamRetentionPeriodRequest) returns (IncreaseStreamRetentionPeriodResponse); + rpc UpdateShardCount(UpdateShardCountRequest) returns (UpdateShardCountResponse); + // stream consumer methods + rpc RegisterStreamConsumer(RegisterStreamConsumerRequest) returns (RegisterStreamConsumerResponse); + rpc DeregisterStreamConsumer(DeregisterStreamConsumerRequest) returns (DeregisterStreamConsumerResponse); + rpc DescribeStreamConsumer(DescribeStreamConsumerRequest) returns (DescribeStreamConsumerResponse); + rpc ListStreamConsumers(ListStreamConsumersRequest) returns (ListStreamConsumersResponse); + + // Least priority methods from API. + rpc AddTagsToStream(AddTagsToStreamRequest) returns (AddTagsToStreamResponse); + rpc DisableEnhancedMonitoring(DisableEnhancedMonitoringRequest) returns (DisableEnhancedMonitoringResponse); + rpc EnableEnhancedMonitoring(EnableEnhancedMonitoringRequest) returns (EnableEnhancedMonitoringResponse); + rpc ListTagsForStream(ListTagsForStreamRequest) returns (ListTagsForStreamResponse); + rpc MergeShards(MergeShardsRequest) returns (MergeShardsResponse); + rpc RemoveTagsFromStream(RemoveTagsFromStreamRequest) returns (RemoveTagsFromStreamResponse); + rpc SplitShard(SplitShardRequest) returns (SplitShardResponse); + rpc StartStreamEncryption(StartStreamEncryptionRequest) returns (StartStreamEncryptionResponse); + rpc StopStreamEncryption(StopStreamEncryptionRequest) returns (StopStreamEncryptionResponse); +} diff --git a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto index ac159e0c0e..873ee38b3c 100644 --- a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto +++ b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto @@ -93,16 +93,16 @@ service PersQueueService { * Alter topic command. */ rpc AlterTopic(AlterTopicRequest) returns (AlterTopicResponse); - - /* - * Add read rule command. - */ - rpc AddReadRule(AddReadRuleRequest) returns (AddReadRuleResponse); - - /* - * Remove read rule command. - */ - rpc RemoveReadRule(RemoveReadRuleRequest) returns (RemoveReadRuleResponse); + + /* + * Add read rule command. + */ + rpc AddReadRule(AddReadRuleRequest) returns (AddReadRuleResponse); + + /* + * Remove read rule command. + */ + rpc RemoveReadRule(RemoveReadRuleRequest) returns (RemoveReadRuleResponse); } service ClusterDiscoveryService { diff --git a/ydb/public/api/protos/draft/datastreams.proto b/ydb/public/api/protos/draft/datastreams.proto index 618a81d888..5c1354288a 100644 --- a/ydb/public/api/protos/draft/datastreams.proto +++ b/ydb/public/api/protos/draft/datastreams.proto @@ -1,153 +1,153 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/ydb_operation.proto"; - + import "google/protobuf/descriptor.proto"; - + package Ydb.DataStreams.V1; -option java_package = "com.yandex.ydb.datastreams.v1"; - +option java_package = "com.yandex.ydb.datastreams.v1"; + // Extsions to simplify json <-> proto conversion -enum EFieldTransformationType { - TRANSFORM_NONE = 0; - TRANSFORM_BASE64 = 1; +enum EFieldTransformationType { + TRANSFORM_NONE = 0; + TRANSFORM_BASE64 = 1; TRANSFORM_DOUBLE_S_TO_INT_MS = 2; TRANSFORM_EMPTY_TO_NOTHING = 3; -} - -extend google.protobuf.FieldOptions { - EFieldTransformationType FieldTransformer = 58123; -} - +} + +extend google.protobuf.FieldOptions { + EFieldTransformationType FieldTransformer = 58123; +} + // Here and below: Kinesis data types mapped to protobuf - -enum EncryptionType { - ENCRYPTION_UNDEFINED = 0; + +enum EncryptionType { + ENCRYPTION_UNDEFINED = 0; NONE = 1; KMS = 2; -} - -message EnhancedMetrics { +} + +message EnhancedMetrics { // List of shard-level metrics - repeated string shard_level_metrics = 1; -} + repeated string shard_level_metrics = 1; +} -message ChildShard { +message ChildShard { // The range of possible hash key values for the shard - HashKeyRange hash_key_range = 1; + HashKeyRange hash_key_range = 1; // List of parent shards - repeated string parent_shards = 2; + repeated string parent_shards = 2; // Name of the shard string name = 3; -} - +} + // Represents details of consumer -message Consumer { - string consumer_arn = 1; +message Consumer { + string consumer_arn = 1; int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - string consumer_name = 3; - ConsumerDescription.ConsumerStatus consumer_status = 4; -} + string consumer_name = 3; + ConsumerDescription.ConsumerStatus consumer_status = 4; +} // The range of possible hash key values for the shard -message HashKeyRange { - string starting_hash_key = 1; - string ending_hash_key = 2; -} +message HashKeyRange { + string starting_hash_key = 1; + string ending_hash_key = 2; +} -message Record { +message Record { // Timestamp that the record was inserted into the stream int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Data blob bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; // Encryption type used on record - EncryptionType encryption = 3; + EncryptionType encryption = 3; // Identifies shard in the stream the record is assigned to - string partition_key = 4; + string partition_key = 4; // Unique id of the record within shard - string sequence_number = 5; -} + string sequence_number = 5; +} // Represents the stream description -message StreamDescription { +message StreamDescription { // Represents current status of the stream being described - enum StreamStatus { - STATUS_UNDEFINED = 0; + enum StreamStatus { + STATUS_UNDEFINED = 0; // Stream is being created - CREATING = 1; + CREATING = 1; // Stream is being deleted - DELETING = 2; + DELETING = 2; // Stream is ready for read & write operations or deletion - ACTIVE = 3; + ACTIVE = 3; // Shards in the stream are being merged ot split - UPDATING = 4; - } - + UPDATING = 4; + } + // Encryption type used on record - EncryptionType encryption_type = 1; + EncryptionType encryption_type = 1; // List of enhanced monitoring settings - repeated EnhancedMetrics enhanced_monitoring = 2; + repeated EnhancedMetrics enhanced_monitoring = 2; // If true, more shards are available to describe - bool has_more_shards = 3; + bool has_more_shards = 3; // Id of the key used to encryption string key_id = 4; // Retention period in hours - int32 retention_period_hours = 5; + int32 retention_period_hours = 5; // List of shards - repeated Shard shards = 6; - string stream_arn = 7; + repeated Shard shards = 6; + string stream_arn = 7; // Timestamp that the stream was created int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Current status of the stream - StreamStatus stream_status = 9; + StreamStatus stream_status = 9; // Name of the stream - string stream_name = 10; - int32 write_quota_kb_per_sec = 11; - string owner = 12; -} - + string stream_name = 10; + int32 write_quota_kb_per_sec = 11; + string owner = 12; +} + // Represents range of possible sequence numbers for the shard -message SequenceNumberRange { - string starting_sequence_number = 1; - string ending_sequence_number = 2; -} - +message SequenceNumberRange { + string starting_sequence_number = 1; + string ending_sequence_number = 2; +} + // Represents shard details -message Shard { +message Shard { // Id of the shard adjacent to the shard's parent string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; // The range of possible hash key values for the shard - HashKeyRange hash_key_range = 2; + HashKeyRange hash_key_range = 2; // Id of the shard's parent string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; // The range of possible sequence numbers for the shard SequenceNumberRange sequence_number_range = 4; // Unique id of the shard within stream - string shard_id = 5; -} - + string shard_id = 5; +} + // Represents consumer details -message ConsumerDescription { - enum ConsumerStatus { - STATUS_UNKNOWN = 0; - CREATING = 1; - DELETING = 2; - ACTIVE = 3; - } - - string consumer_arn = 1; +message ConsumerDescription { + enum ConsumerStatus { + STATUS_UNKNOWN = 0; + CREATING = 1; + DELETING = 2; + ACTIVE = 3; + } + + string consumer_arn = 1; // Timestamp that the consumer was created int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the consumer - string consumer_name = 3; - ConsumerStatus consumer_status = 4; - string stream_arn = 5; -} - + string consumer_name = 3; + ConsumerStatus consumer_status = 4; + string stream_arn = 5; +} + // Determines how the shard iterator is used to start reading records -enum ShardIteratorType { - SHARD_ITERATOR_UNDEFINED = 0; +enum ShardIteratorType { + SHARD_ITERATOR_UNDEFINED = 0; // Start reading exactly from the position denoted by a sequence number AT_SEQUENCE_NUMBER = 1; // Start reading right after the specified position @@ -158,165 +158,165 @@ enum ShardIteratorType { TRIM_HORIZON = 4; // Start reading latest records LATEST = 5; -} - +} + // Parameters used to filter out the response of ListShards -message ShardFilter { - enum ShardFilterType { - SHARD_TYPE_UNDEFINED = 0; +message ShardFilter { + enum ShardFilterType { + SHARD_TYPE_UNDEFINED = 0; // Include all the shards, starting with the shard whose id immediately follows the shard_id - AFTER_SHARD_ID = 1; + AFTER_SHARD_ID = 1; // Include all the shards that were open at TRIM_HORIZON - AT_TRIM_HORIZON = 2; + AT_TRIM_HORIZON = 2; // Include all the shards within the retention period - FROM_TRIM_HORIZON = 3; + FROM_TRIM_HORIZON = 3; // Include only the currently open shards - AT_LATEST = 4; + AT_LATEST = 4; // Include all the shards whose start timestamp is <= to the given - AT_TIMESTAMP = 5; + AT_TIMESTAMP = 5; // Include all closed shards whose end timestamp >= to the given - FROM_TIMESTAMP = 6; - } - + FROM_TIMESTAMP = 6; + } + // Exclusive id. Can only be used if AFTER_SHARD_ID is specified - string shard_id = 1; + string shard_id = 1; // Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified. int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; ShardFilterType type = 3; -} +} // Represents starting position in the stream from which to start reading -message StartingPosition { +message StartingPosition { // Timestamp of the record from which to start reading int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Sequence number of the record from which to start reading - string sequence_number = 2; - ShardIteratorType type = 3; -} - -message StreamDescriptionSummary { + string sequence_number = 2; + ShardIteratorType type = 3; +} + +message StreamDescriptionSummary { // Number of enhanced fan-out consumers - int32 consumer_count = 1; + int32 consumer_count = 1; // Encryption type used - EncryptionType encryption_type = 2; + EncryptionType encryption_type = 2; // List of enhanced monitoring settings - repeated EnhancedMetrics enhanced_monitoring = 3; + repeated EnhancedMetrics enhanced_monitoring = 3; // Id of the key used to encryption - string key_id = 4; + string key_id = 4; // Number of open shards in the stream - int32 open_shard_count = 5; + int32 open_shard_count = 5; // Retention period in hours - int32 retention_period_hours = 6; - string stream_arn = 7; + int32 retention_period_hours = 6; + string stream_arn = 7; // Timestamp that the stream was created int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the stream - string stream_name = 9; + string stream_name = 9; // Current status of the stream - StreamDescription.StreamStatus stream_status = 10; -} + StreamDescription.StreamStatus stream_status = 10; +} // Server sends events of this type after call SubscribeToShard -message SubscribeToShardEvent { +message SubscribeToShardEvent { // List of child shards - repeated ChildShard child_shards = 1; + repeated ChildShard child_shards = 1; // Used in next call to SubscribeToShards - string continuation_sequence_number = 2; + string continuation_sequence_number = 2; // Number of milliseconds the read records are from the tip of the stream. // Zero indicates that record processing is caught up. - int32 millis_behind_latest = 3; + int32 millis_behind_latest = 3; // List of records - repeated Record records = 4; -} - -message SubscribeToShardEventStream { - message ExceptionEvent { - } - - oneof event { - ExceptionEvent internal_failure_exception = 1; - ExceptionEvent kms_access_denied_exception = 2; - ExceptionEvent kms_disabled_exception = 3; - ExceptionEvent kms_invalid_state_exception = 4; - ExceptionEvent kms_not_found_exception = 5; - ExceptionEvent kms_opt_in_required = 6; - ExceptionEvent kms_throttling_exception = 7; - ExceptionEvent resource_in_use_exception = 8; - ExceptionEvent resource_not_found_exception = 9; - SubscribeToShardEvent subscribe_to_shard_event = 10; - } -} + repeated Record records = 4; +} + +message SubscribeToShardEventStream { + message ExceptionEvent { + } + + oneof event { + ExceptionEvent internal_failure_exception = 1; + ExceptionEvent kms_access_denied_exception = 2; + ExceptionEvent kms_disabled_exception = 3; + ExceptionEvent kms_invalid_state_exception = 4; + ExceptionEvent kms_not_found_exception = 5; + ExceptionEvent kms_opt_in_required = 6; + ExceptionEvent kms_throttling_exception = 7; + ExceptionEvent resource_in_use_exception = 8; + ExceptionEvent resource_not_found_exception = 9; + SubscribeToShardEvent subscribe_to_shard_event = 10; + } +} // Represents metadata assigned to the stream (key-value pair) -message Tag { - string key = 1; - string value = 2; -} - -message CreateStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; +message Tag { + string key = 1; + string value = 2; +} + +message CreateStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; + string stream_name = 2; // Shards count for a new stream - int32 shard_count = 3; - + int32 shard_count = 3; + // Datastreams extension. Allow to set retention and write quota upon stream creation - int32 write_quota_kb_per_sec = 4; - int32 retention_period_hours = 5; -} - -message CreateStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message CreateStreamResult { -} - -message DeleteStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + int32 write_quota_kb_per_sec = 4; + int32 retention_period_hours = 5; +} + +message CreateStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message CreateStreamResult { +} + +message DeleteStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // If unset or false, and the stream has registered consumers, the call to DeleteStream fails - bool enforce_consumer_deletion = 2; + bool enforce_consumer_deletion = 2; // Name of the stream - string stream_name = 3; -} - -message DeleteStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DeleteStreamResult { -} - -message ListStreamsRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 3; +} + +message DeleteStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DeleteStreamResult { +} + +message ListStreamsRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of stream to start from string exclusive_start_stream_name = 2; // Max number of entries to return int32 limit = 3; // Will make recurse listing if set to true. Otherwise will return only streams from root directory. bool recurse = 4; -} +} -message ListStreamsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message ListStreamsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message ListStreamsResult { +message ListStreamsResult { // True if more streams are available bool has_more_streams = 1; // Name of streams repeated string stream_names = 2; -} - -message ListShardsRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message ListShardsRequest { + Ydb.Operations.OperationParams operation_params = 1; // Id of the shard to start from - string exclusive_start_shard_id = 2; + string exclusive_start_shard_id = 2; // Max number of entries to return - int32 max_results = 3; + int32 max_results = 3; // Pagination token string next_token = 4; // Filter out response @@ -324,189 +324,189 @@ message ListShardsRequest { // Used to distinguish streams that have the same name int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the stream - string stream_name = 7; -} + string stream_name = 7; +} -message ListShardsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message ListShardsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message ListShardsResult { +message ListShardsResult { // Pagination token - string next_token = 1; + string next_token = 1; // List of shards - repeated Shard shards = 2; -} - -message SetWriteQuotaRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated Shard shards = 2; +} + +message SetWriteQuotaRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; - int32 write_quota_kb_per_sec = 3; -} + string stream_name = 2; + int32 write_quota_kb_per_sec = 3; +} -message SetWriteQuotaResponse { +message SetWriteQuotaResponse { // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message SetWriteQuotaResult { -} - -message UpdateStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + Ydb.Operations.Operation operation = 1; +} + +message SetWriteQuotaResult { +} + +message UpdateStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; - int32 write_quota_kb_per_sec = 3; + string stream_name = 2; + int32 write_quota_kb_per_sec = 3; // New number of shards - int32 target_shard_count = 4; + int32 target_shard_count = 4; // Retention period in hours - int32 retention_period_hours = 5; -} + int32 retention_period_hours = 5; +} -message UpdateStreamResponse { +message UpdateStreamResponse { // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message UpdateStreamResult { -} - -message ListStreamConsumersRequest { - Ydb.Operations.OperationParams operation_params = 1; + Ydb.Operations.Operation operation = 1; +} + +message UpdateStreamResult { +} + +message ListStreamConsumersRequest { + Ydb.Operations.OperationParams operation_params = 1; // Max number of entries to return - int32 max_results = 2; + int32 max_results = 2; // Pagination token - string next_token = 3; - string stream_arn = 4; + string next_token = 3; + string stream_arn = 4; // Used to distinguish streams that have the same name int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; -} +} -message ListStreamConsumersResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message ListStreamConsumersResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message ListStreamConsumersResult { +message ListStreamConsumersResult { // Pagination token - string next_token = 1; + string next_token = 1; // List of consumers - repeated Consumer consumers = 2; -} - -message DescribeStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated Consumer consumers = 2; +} + +message DescribeStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Id of the shard to start from - string exclusive_start_shard_id = 2; + string exclusive_start_shard_id = 2; // Max number of entries to return - int32 limit = 3; + int32 limit = 3; // Name of the stream to describe - string stream_name = 4; -} - -message DescribeStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DescribeStreamResult { + string stream_name = 4; +} + +message DescribeStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DescribeStreamResult { StreamDescription stream_description = 1; -} - -message RegisterStreamConsumerRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message RegisterStreamConsumerRequest { + Ydb.Operations.OperationParams operation_params = 1; // Unique name of the consumer within the stream - string consumer_name = 2; - string stream_arn = 3; -} - -message RegisterStreamConsumerResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message RegisterStreamConsumerResult { - Consumer consumer = 1; -} - -message DeregisterStreamConsumerRequest { - Ydb.Operations.OperationParams operation_params = 1; - string consumer_arn = 2; + string consumer_name = 2; + string stream_arn = 3; +} + +message RegisterStreamConsumerResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message RegisterStreamConsumerResult { + Consumer consumer = 1; +} + +message DeregisterStreamConsumerRequest { + Ydb.Operations.OperationParams operation_params = 1; + string consumer_arn = 2; // Name of consumer - string consumer_name = 3; - string stream_arn = 4; -} - -message DeregisterStreamConsumerResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DeregisterStreamConsumerResult { -} - -message DescribeStreamConsumerRequest { - Ydb.Operations.OperationParams operation_params = 1; - string consumer_arn = 2; + string consumer_name = 3; + string stream_arn = 4; +} + +message DeregisterStreamConsumerResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DeregisterStreamConsumerResult { +} + +message DescribeStreamConsumerRequest { + Ydb.Operations.OperationParams operation_params = 1; + string consumer_arn = 2; // Name of consumer - string consumer_name = 3; - string stream_arn = 4; -} - -message DescribeStreamConsumerResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DescribeStreamConsumerResult { - ConsumerDescription consumer_description = 1; -} - -message PutRecordsRequestEntry { + string consumer_name = 3; + string stream_arn = 4; +} + +message DescribeStreamConsumerResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DescribeStreamConsumerResult { + ConsumerDescription consumer_description = 1; +} + +message PutRecordsRequestEntry { // Data blob - bytes data = 1 [(FieldTransformer) = TRANSFORM_BASE64]; + bytes data = 1 [(FieldTransformer) = TRANSFORM_BASE64]; // Hash value used to explicitly determine the shard - string explicit_hash_key = 2; + string explicit_hash_key = 2; // Used as input to hash function that maps partition key to a specific shard - string partition_key = 3; -} - + string partition_key = 3; +} + // Represents result of an individual record -message PutRecordsResultEntry { +message PutRecordsResultEntry { string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; - string sequence_number = 4; + string sequence_number = 4; string shard_id = 5; -} - -message GetRecordsRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message GetRecordsRequest { + Ydb.Operations.OperationParams operation_params = 1; // Max number of records to return int32 limit = 2; // Iterator to shard string shard_iterator = 3; } - -message GetRecordsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} -message GetRecordsResult { +message GetRecordsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message GetRecordsResult { // List of child shards if SplitShard was called on a given shard repeated ChildShard child_shards = 1; // Number of milliseconds the read records are from the tip of the stream. // Zero indicates that record processing is caught up. - int64 millis_behind_latest = 2; - string next_shard_iterator = 3; + int64 millis_behind_latest = 2; + string next_shard_iterator = 3; // List of records repeated Record records = 4; -} - -message PutRecordRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message PutRecordRequest { + Ydb.Operations.OperationParams operation_params = 1; // Data blob bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; // Hash value used to explicitly determine the shard @@ -516,343 +516,343 @@ message PutRecordRequest { string sequence_number_for_ordering = 5; // Name of the stream to put record to string stream_name = 6; -} +} -message PutRecordResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message PutRecordResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message PutRecordResult { +message PutRecordResult { // Encryption type used - EncryptionType encryption_type = 1; + EncryptionType encryption_type = 1; // Unique id of the record within shard - string sequence_number = 2; + string sequence_number = 2; // Id of the shard were the data was inserted string shard_id = 3; -} - -message PutRecordsRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message PutRecordsRequest { + Ydb.Operations.OperationParams operation_params = 1; // List of records - repeated PutRecordsRequestEntry records = 2; + repeated PutRecordsRequestEntry records = 2; // Name of the stream to put records to - string stream_name = 3; -} + string stream_name = 3; +} -message PutRecordsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message PutRecordsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message PutRecordsResult { +message PutRecordsResult { // Encryption type used - EncryptionType encryption_type = 1; + EncryptionType encryption_type = 1; // Number of unsuccessfully processed records - int32 failed_record_count = 2; + int32 failed_record_count = 2; // List of results - repeated PutRecordsResultEntry records = 3; -} - -message GetShardIteratorRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated PutRecordsResultEntry records = 3; +} + +message GetShardIteratorRequest { + Ydb.Operations.OperationParams operation_params = 1; // Id of the shard to get iterator for - string shard_id = 2; - ShardIteratorType shard_iterator_type = 3; + string shard_id = 2; + ShardIteratorType shard_iterator_type = 3; // Used with shard iterator type AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER - string starting_sequence_number = 4; + string starting_sequence_number = 4; // Name of the stream - string stream_name = 5; + string stream_name = 5; // Used with shard iterator type AT_TIMESTAMP int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; -} +} -message GetShardIteratorResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message GetShardIteratorResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message GetShardIteratorResult { +message GetShardIteratorResult { string shard_iterator = 1; -} - -message SubscribeToShardRequest { - Ydb.Operations.OperationParams operation_params = 1; - string consumer_arn = 2; +} + +message SubscribeToShardRequest { + Ydb.Operations.OperationParams operation_params = 1; + string consumer_arn = 2; // Id of the shard to subscribe to - string shard_id = 3; - StartingPosition starting_position = 4; -} - -message SubscribeToShardResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message SubscribeToShardResult { - SubscribeToShardEventStream event_stream = 1; -} - -message AddTagsToStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + string shard_id = 3; + StartingPosition starting_position = 4; +} + +message SubscribeToShardResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message SubscribeToShardResult { + SubscribeToShardEventStream event_stream = 1; +} + +message AddTagsToStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; + string stream_name = 2; // List of tags to add to stream - repeated string tags = 3; -} - -message AddTagsToStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message AddTagsToStreamResult { -} - -message DecreaseStreamRetentionPeriodRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated string tags = 3; +} + +message AddTagsToStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message AddTagsToStreamResult { +} + +message DecreaseStreamRetentionPeriodRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; - // Retention period in hours - int32 retention_period_hours = 3; -} - -message DecreaseStreamRetentionPeriodResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DecreaseStreamRetentionPeriodResult { -} - -message DescribeLimitsRequest { - Ydb.Operations.OperationParams operation_params = 1; -} - -message DescribeLimitsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message DescribeLimitsResult { + string stream_name = 2; + // Retention period in hours + int32 retention_period_hours = 3; +} + +message DecreaseStreamRetentionPeriodResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DecreaseStreamRetentionPeriodResult { +} + +message DescribeLimitsRequest { + Ydb.Operations.OperationParams operation_params = 1; +} + +message DescribeLimitsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message DescribeLimitsResult { // Number of open shards - int32 open_shard_count = 1; + int32 open_shard_count = 1; // Max number of shards - int32 shard_limit = 2; -} - -message DescribeStreamSummaryRequest { - Ydb.Operations.OperationParams operation_params = 1; + int32 shard_limit = 2; +} + +message DescribeStreamSummaryRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; -} + string stream_name = 2; +} -message DescribeStreamSummaryResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message DescribeStreamSummaryResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message DescribeStreamSummaryResult { +message DescribeStreamSummaryResult { // Stream description sumary StreamDescriptionSummary stream_description_summary = 1; -} - -message DisableEnhancedMonitoringRequest { - Ydb.Operations.OperationParams operation_params = 1; +} + +message DisableEnhancedMonitoringRequest { + Ydb.Operations.OperationParams operation_params = 1; // List of shard-level metrics to disable - repeated string shard_level_metrics = 2; + repeated string shard_level_metrics = 2; // Name of the stream - string stream_name = 3; -} + string stream_name = 3; +} -message DisableEnhancedMonitoringResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message DisableEnhancedMonitoringResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message DisableEnhancedMonitoringResult { +message DisableEnhancedMonitoringResult { // Represents current state before the operation - repeated string current_shard_level_metrics = 1; + repeated string current_shard_level_metrics = 1; // Represents list of metrics after the operation - repeated string desired_shard_level_metrics = 2; + repeated string desired_shard_level_metrics = 2; // Name of the stream - string stream_name = 3; -} - -message EnableEnhancedMonitoringRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 3; +} + +message EnableEnhancedMonitoringRequest { + Ydb.Operations.OperationParams operation_params = 1; // List of shard-level metrics to enable - repeated string shard_level_metrics = 2; + repeated string shard_level_metrics = 2; // Name of the stream - string stream_name = 3; -} + string stream_name = 3; +} -message EnableEnhancedMonitoringResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message EnableEnhancedMonitoringResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message EnableEnhancedMonitoringResult { +message EnableEnhancedMonitoringResult { // Represents current state before the operation - repeated string current_shard_level_metrics = 1; + repeated string current_shard_level_metrics = 1; // Represents list of metrics after the operation - repeated string desired_shard_level_metrics = 2; + repeated string desired_shard_level_metrics = 2; // Name of the stream - string stream_name = 3; -} - -message IncreaseStreamRetentionPeriodRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 3; +} + +message IncreaseStreamRetentionPeriodRequest { + Ydb.Operations.OperationParams operation_params = 1; // Retention period in hours - int32 retention_period_hours = 2; + int32 retention_period_hours = 2; // Name of the stream - string stream_name = 3; -} - -message IncreaseStreamRetentionPeriodResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message IncreaseStreamRetentionPeriodResult { -} - -message ListTagsForStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 3; +} + +message IncreaseStreamRetentionPeriodResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message IncreaseStreamRetentionPeriodResult { +} + +message ListTagsForStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Key to start from - string exclusive_start_tag_key = 2; + string exclusive_start_tag_key = 2; // Max number of entries to return - int32 limit = 3; + int32 limit = 3; // Name of the stream - string stream_name = 4; -} + string stream_name = 4; +} -message ListTagsForStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message ListTagsForStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message ListTagsForStreamResult { +message ListTagsForStreamResult { // If true, more tags are available - bool has_more_tags = 1; + bool has_more_tags = 1; // List of tags - repeated Tag tags = 2; -} - -message MergeShardsRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated Tag tags = 2; +} + +message MergeShardsRequest { + Ydb.Operations.OperationParams operation_params = 1; // Id of the adjacent shard for the merge - string adjacent_shard_to_merge = 2; + string adjacent_shard_to_merge = 2; // Id of the shard to combine with the adjacent - string shard_to_merge = 3; + string shard_to_merge = 3; // Name of the stream - string stream_name = 4; -} - -message MergeShardsResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message MergeShardsResult { -} - -message RemoveTagsFromStreamRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 4; +} + +message MergeShardsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message MergeShardsResult { +} + +message RemoveTagsFromStreamRequest { + Ydb.Operations.OperationParams operation_params = 1; // Name of the stream - string stream_name = 2; + string stream_name = 2; // List of tag keys - repeated string tag_keys = 3; -} - -message RemoveTagsFromStreamResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message RemoveTagsFromStreamResult { -} - -message SplitShardRequest { - Ydb.Operations.OperationParams operation_params = 1; + repeated string tag_keys = 3; +} + +message RemoveTagsFromStreamResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message RemoveTagsFromStreamResult { +} + +message SplitShardRequest { + Ydb.Operations.OperationParams operation_params = 1; // Hash value for the starting hash key of one of the new child shards - string new_starting_hash_key = 2; + string new_starting_hash_key = 2; // Id of the shard to split - string shard_to_split = 3; + string shard_to_split = 3; // Name of the stream - string stream_name = 4; -} - -message SplitShardResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message SplitShardResult { -} - -message StartStreamEncryptionRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 4; +} + +message SplitShardResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message SplitShardResult { +} + +message StartStreamEncryptionRequest { + Ydb.Operations.OperationParams operation_params = 1; // Encryption type to use - EncryptionType encryption_type = 2; + EncryptionType encryption_type = 2; // Id of the key to use to encryption - string key_id = 3; + string key_id = 3; // Name of the stream - string stream_name = 4; -} - -message StartStreamEncryptionResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message StartStreamEncryptionResult { -} - -message StopStreamEncryptionRequest { - Ydb.Operations.OperationParams operation_params = 1; + string stream_name = 4; +} + +message StartStreamEncryptionResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message StartStreamEncryptionResult { +} + +message StopStreamEncryptionRequest { + Ydb.Operations.OperationParams operation_params = 1; // Encryption type - EncryptionType encryption_type = 2; + EncryptionType encryption_type = 2; // Id of the key to use to encryption - string key_id = 3; + string key_id = 3; // Name of the stream - string stream_name = 4; -} - -message StopStreamEncryptionResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -message StopStreamEncryptionResult { -} - -message UpdateShardCountRequest { - enum ScalingType { - SCALING_TYPE_UNKNOWN = 0; + string stream_name = 4; +} + +message StopStreamEncryptionResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message StopStreamEncryptionResult { +} + +message UpdateShardCountRequest { + enum ScalingType { + SCALING_TYPE_UNKNOWN = 0; // Creates shards of equal size - UNIFORM_SCALING = 1; - } - - Ydb.Operations.OperationParams operation_params = 1; + UNIFORM_SCALING = 1; + } + + Ydb.Operations.OperationParams operation_params = 1; // Scaling type - ScalingType scaling_type = 2; + ScalingType scaling_type = 2; // Name of the stream - string stream_name = 3; + string stream_name = 3; // New number of shards - int32 target_shard_count = 4; -} + int32 target_shard_count = 4; +} -message UpdateShardCountResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} +message UpdateShardCountResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} -message UpdateShardCountResult { +message UpdateShardCountResult { // Current number of shards - int32 current_shard_count = 1; + int32 current_shard_count = 1; // Name of the stream - string stream_name = 2; + string stream_name = 2; // Updated number of shards - int32 target_shard_count = 3; -} + int32 target_shard_count = 3; +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index 27d3ec8fd3..8a156403a3 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -14,7 +14,7 @@ PEERDIR( ) SRCS( - draft/datastreams.proto + draft/datastreams.proto draft/persqueue_common.proto draft/persqueue_error_codes.proto draft/ydb_long_tx.proto diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto index 5bfffd32a7..93a7fb6c79 100644 --- a/ydb/public/api/protos/ydb_persqueue_v1.proto +++ b/ydb/public/api/protos/ydb_persqueue_v1.proto @@ -910,9 +910,9 @@ message MigrationStreamingReadServerMessage { bytes data = 5; // Uncompressed size of client message body. uint64 uncompressed_size = 6; - // kinesis data - string partition_key = 7; - bytes explicit_hash = 8; + // kinesis data + string partition_key = 7; + bytes explicit_hash = 8; } // Representation of sequence of client messages from one write session. @@ -1262,58 +1262,58 @@ message AlterTopicResult { } /** - * Add read rules for existing topic request. - */ -message AddReadRuleRequest { - Ydb.Operations.OperationParams operation_params = 1; - // Topic path. - string path = 2 [(required) = true]; - // read rules to add - TopicSettings.ReadRule read_rule = 3; -} - -/** - * Add read rules for existing topic response. - */ -message AddReadRuleResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -/** - * Add read rules result message that will be inside AddReadRuleReponse.operation. - */ -message AddReadRuleResult { -} - - -/** - * Remove read rules request for existing topic. - */ -message RemoveReadRuleRequest { - Ydb.Operations.OperationParams operation_params = 1; - // Topic path. - string path = 2 [(required) = true]; - // read rules to remove. Only consumer names - string consumer_name = 3; -} - -/** - * Remove read rules response for existing topic. - */ -message RemoveReadRuleResponse { - // Result of request will be inside operation. - Ydb.Operations.Operation operation = 1; -} - -/** - * Remove read rules result message that will be inside RemoveReadRuleReponse.operation. - */ -message RemoveReadRuleResult { -} - - -/** + * Add read rules for existing topic request. + */ +message AddReadRuleRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Topic path. + string path = 2 [(required) = true]; + // read rules to add + TopicSettings.ReadRule read_rule = 3; +} + +/** + * Add read rules for existing topic response. + */ +message AddReadRuleResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +/** + * Add read rules result message that will be inside AddReadRuleReponse.operation. + */ +message AddReadRuleResult { +} + + +/** + * Remove read rules request for existing topic. + */ +message RemoveReadRuleRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Topic path. + string path = 2 [(required) = true]; + // read rules to remove. Only consumer names + string consumer_name = 3; +} + +/** + * Remove read rules response for existing topic. + */ +message RemoveReadRuleResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +/** + * Remove read rules result message that will be inside RemoveReadRuleReponse.operation. + */ +message RemoveReadRuleResult { +} + + +/** * Describe topic request sent from client to server. */ @@ -1344,4 +1344,4 @@ message DescribeTopicResult { TopicSettings settings = 2; } - + diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp index 584621c711..f8297f4323 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp @@ -1,105 +1,105 @@ -#include "datastreams.h" - -#define INCLUDE_YDB_INTERNAL_H +#include "datastreams.h" + +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - -namespace NYdb::NDataStreams::V1 { - - class TDataStreamsClient::TImpl : public TClientImplCommon<TDataStreamsClient::TImpl> { - public: - TImpl(std::shared_ptr <TGRpcConnectionsImpl> &&connections, const TCommonClientSettings &settings) - : TClientImplCommon(std::move(connections), settings) {} - - template<class TProtoResult, class TResultWrapper> - auto MakeResultExtractor(NThreading::TPromise <TResultWrapper> promise) { + +namespace NYdb::NDataStreams::V1 { + + class TDataStreamsClient::TImpl : public TClientImplCommon<TDataStreamsClient::TImpl> { + public: + TImpl(std::shared_ptr <TGRpcConnectionsImpl> &&connections, const TCommonClientSettings &settings) + : TClientImplCommon(std::move(connections), settings) {} + + template<class TProtoResult, class TResultWrapper> + auto MakeResultExtractor(NThreading::TPromise <TResultWrapper> promise) { return [promise = std::move(promise)] - (google::protobuf::Any *any, TPlainStatus status) mutable { - std::unique_ptr <TProtoResult> result; - if (any) { - result.reset(new TProtoResult); - any->UnpackTo(result.get()); - } - - promise.SetValue( - TResultWrapper( + (google::protobuf::Any *any, TPlainStatus status) mutable { + std::unique_ptr <TProtoResult> result; + if (any) { + result.reset(new TProtoResult); + any->UnpackTo(result.get()); + } + + promise.SetValue( + TResultWrapper( TStatus(std::move(status)), - std::move(result))); - }; - } - - template<class TProtoService, class TProtoRequest, class TProtoResponse, class TProtoResult, class TSettings, class TFillRequestFn, class TAsyncCall> - NThreading::TFuture<TProtoResultWrapper<TProtoResult>> CallImpl(const TSettings& settings, TAsyncCall grpcCall, TFillRequestFn fillRequest) { - using TResultWrapper = TProtoResultWrapper<TProtoResult>; - auto request = MakeOperationRequest<TProtoRequest>(settings); - fillRequest(request); - - auto promise = NThreading::NewPromise<TResultWrapper>(); - auto future = promise.GetFuture(); - - auto extractor = MakeResultExtractor<TProtoResult, TResultWrapper>(std::move(promise)); - - Connections_->RunDeferred<TProtoService, TProtoRequest, TProtoResponse>( - std::move(request), - std::move(extractor), - grpcCall, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return future; - - } - - template<class TProtoService, class TProtoRequest, class TProtoResponse, class TProtoResult, class TSettings, class TAsyncCall> - NThreading::TFuture<TProtoResultWrapper<TProtoResult>> CallImpl(const TSettings& settings, TAsyncCall grpcCall) { - return CallImpl<TProtoService, TProtoRequest, TProtoResponse, TProtoResult>(settings, grpcCall, [](TProtoRequest&) {}); - } - - TAsyncCreateStreamResult CreateStream(const TString &path, TCreateStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::CreateStreamRequest, - Ydb::DataStreams::V1::CreateStreamResponse, - Ydb::DataStreams::V1::CreateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream, - [&](Ydb::DataStreams::V1::CreateStreamRequest& req) { - req.set_stream_name(path); - req.set_shard_count(settings.ShardCount_); - req.set_retention_period_hours(settings.RetentionPeriodHours_); - req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); - }); - } - - TAsyncListStreamsResult ListStreams(TListStreamsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::ListStreamsRequest, - Ydb::DataStreams::V1::ListStreamsResponse, - Ydb::DataStreams::V1::ListStreamsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams, - [&](Ydb::DataStreams::V1::ListStreamsRequest& req) { - req.set_exclusive_start_stream_name(settings.ExclusiveStartStreamName_); - req.set_limit(settings.Limit_); + std::move(result))); + }; + } + + template<class TProtoService, class TProtoRequest, class TProtoResponse, class TProtoResult, class TSettings, class TFillRequestFn, class TAsyncCall> + NThreading::TFuture<TProtoResultWrapper<TProtoResult>> CallImpl(const TSettings& settings, TAsyncCall grpcCall, TFillRequestFn fillRequest) { + using TResultWrapper = TProtoResultWrapper<TProtoResult>; + auto request = MakeOperationRequest<TProtoRequest>(settings); + fillRequest(request); + + auto promise = NThreading::NewPromise<TResultWrapper>(); + auto future = promise.GetFuture(); + + auto extractor = MakeResultExtractor<TProtoResult, TResultWrapper>(std::move(promise)); + + Connections_->RunDeferred<TProtoService, TProtoRequest, TProtoResponse>( + std::move(request), + std::move(extractor), + grpcCall, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return future; + + } + + template<class TProtoService, class TProtoRequest, class TProtoResponse, class TProtoResult, class TSettings, class TAsyncCall> + NThreading::TFuture<TProtoResultWrapper<TProtoResult>> CallImpl(const TSettings& settings, TAsyncCall grpcCall) { + return CallImpl<TProtoService, TProtoRequest, TProtoResponse, TProtoResult>(settings, grpcCall, [](TProtoRequest&) {}); + } + + TAsyncCreateStreamResult CreateStream(const TString &path, TCreateStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::CreateStreamRequest, + Ydb::DataStreams::V1::CreateStreamResponse, + Ydb::DataStreams::V1::CreateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream, + [&](Ydb::DataStreams::V1::CreateStreamRequest& req) { + req.set_stream_name(path); + req.set_shard_count(settings.ShardCount_); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); + }); + } + + TAsyncListStreamsResult ListStreams(TListStreamsSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::ListStreamsRequest, + Ydb::DataStreams::V1::ListStreamsResponse, + Ydb::DataStreams::V1::ListStreamsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams, + [&](Ydb::DataStreams::V1::ListStreamsRequest& req) { + req.set_exclusive_start_stream_name(settings.ExclusiveStartStreamName_); + req.set_limit(settings.Limit_); req.set_recurse(settings.Recurse_); - }); - } - - TAsyncDescribeStreamResult DescribeStream(TDescribeStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DescribeStreamRequest, - Ydb::DataStreams::V1::DescribeStreamResponse, - Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream); - } - + }); + } + + TAsyncDescribeStreamResult DescribeStream(TDescribeStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DescribeStreamRequest, + Ydb::DataStreams::V1::DescribeStreamResponse, + Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream); + } + TAsyncListShardsResult ListShards(const TString &path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::ListShardsRequest, - Ydb::DataStreams::V1::ListShardsResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::ListShardsRequest, + Ydb::DataStreams::V1::ListShardsResponse, Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards, [&](Ydb::DataStreams::V1::ListShardsRequest& req) { req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); @@ -109,726 +109,726 @@ namespace NYdb::NDataStreams::V1 { req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_); req.set_stream_name(path); }); - } - - TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::PutRecordsRequest, - Ydb::DataStreams::V1::PutRecordsResponse, - Ydb::DataStreams::V1::PutRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords, - [&](Ydb::DataStreams::V1::PutRecordsRequest& req) { - req.set_stream_name(path); - for (const auto& record : records) { - auto* protoRecord = req.add_records(); - protoRecord->set_partition_key(record.PartitionKey); - protoRecord->set_data(record.Data); - protoRecord->set_explicit_hash_key(record.ExplicitHashDecimal); - } - }); - } - + } + + TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::PutRecordsRequest, + Ydb::DataStreams::V1::PutRecordsResponse, + Ydb::DataStreams::V1::PutRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords, + [&](Ydb::DataStreams::V1::PutRecordsRequest& req) { + req.set_stream_name(path); + for (const auto& record : records) { + auto* protoRecord = req.add_records(); + protoRecord->set_partition_key(record.PartitionKey); + protoRecord->set_data(record.Data); + protoRecord->set_explicit_hash_key(record.ExplicitHashDecimal); + } + }); + } + TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::GetRecordsRequest, - Ydb::DataStreams::V1::GetRecordsResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::GetRecordsRequest, + Ydb::DataStreams::V1::GetRecordsResponse, Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords, [&](Ydb::DataStreams::V1::GetRecordsRequest& req) { req.set_shard_iterator(shardIterator); req.set_limit(settings.Limit_); }); - } - + } + TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::GetShardIteratorRequest, - Ydb::DataStreams::V1::GetShardIteratorResponse, - Ydb::DataStreams::V1::GetShardIteratorResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator, - [&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) { - req.set_stream_name(path); + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::GetShardIteratorRequest, + Ydb::DataStreams::V1::GetShardIteratorResponse, + Ydb::DataStreams::V1::GetShardIteratorResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator, + [&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) { + req.set_stream_name(path); req.set_shard_id(shardId); req.set_shard_iterator_type(shardIteratorType); req.set_starting_sequence_number(settings.StartingSequenceNumber_); req.set_timestamp(settings.Timestamp_); - }); - } - - /*TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::SubscribeToShardRequest, - Ydb::DataStreams::V1::SubscribeToShardResponse, - Ydb::DataStreams::V1::SubscribeToShardResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSubscribeToShard); - }*/ - - TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DescribeLimitsRequest, - Ydb::DataStreams::V1::DescribeLimitsResponse, - Ydb::DataStreams::V1::DescribeLimitsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits); - } - + }); + } + + /*TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::SubscribeToShardRequest, + Ydb::DataStreams::V1::SubscribeToShardResponse, + Ydb::DataStreams::V1::SubscribeToShardResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSubscribeToShard); + }*/ + + TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DescribeLimitsRequest, + Ydb::DataStreams::V1::DescribeLimitsResponse, + Ydb::DataStreams::V1::DescribeLimitsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits); + } + TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DescribeStreamSummaryRequest, - Ydb::DataStreams::V1::DescribeStreamSummaryResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DescribeStreamSummaryRequest, + Ydb::DataStreams::V1::DescribeStreamSummaryResponse, Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary, [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) { req.set_stream_name(path); }); - } - - TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod, - [&](Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& req) { - req.set_stream_name(path); - req.set_retention_period_hours(settings.RetentionPeriodHours_); - }); - - } - - TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod, - [&](Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& req) { - req.set_stream_name(path); - req.set_retention_period_hours(settings.RetentionPeriodHours_); - }); - - } - - TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::UpdateShardCountRequest, - Ydb::DataStreams::V1::UpdateShardCountResponse, - Ydb::DataStreams::V1::UpdateShardCountResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount, - [&](Ydb::DataStreams::V1::UpdateShardCountRequest& req) { - req.set_stream_name(path); - req.set_target_shard_count(settings.TargetShardCount_); - }); - } - + } + + TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod, + [&](Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& req) { + req.set_stream_name(path); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + }); + + } + + TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod, + [&](Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& req) { + req.set_stream_name(path); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + }); + + } + + TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::UpdateShardCountRequest, + Ydb::DataStreams::V1::UpdateShardCountResponse, + Ydb::DataStreams::V1::UpdateShardCountResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount, + [&](Ydb::DataStreams::V1::UpdateShardCountRequest& req) { + req.set_stream_name(path); + req.set_target_shard_count(settings.TargetShardCount_); + }); + } + TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::RegisterStreamConsumerRequest, - Ydb::DataStreams::V1::RegisterStreamConsumerResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::RegisterStreamConsumerRequest, + Ydb::DataStreams::V1::RegisterStreamConsumerResponse, Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer, [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) { req.set_stream_arn(path); req.set_consumer_name(consumer_name); }); - } - + } + TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, - Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, + Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer, [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) { req.set_stream_arn(path); req.set_consumer_name(consumer_name); }); - } - - TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DescribeStreamConsumerRequest, - Ydb::DataStreams::V1::DescribeStreamConsumerResponse, - Ydb::DataStreams::V1::DescribeStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer); - } - + } + + TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DescribeStreamConsumerRequest, + Ydb::DataStreams::V1::DescribeStreamConsumerResponse, + Ydb::DataStreams::V1::DescribeStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer); + } + TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::ListStreamConsumersRequest, - Ydb::DataStreams::V1::ListStreamConsumersResponse, + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::ListStreamConsumersRequest, + Ydb::DataStreams::V1::ListStreamConsumersResponse, Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) { req.set_stream_arn(path); req.set_next_token(settings.NextToken_); req.set_max_results(settings.MaxResults_); }); - } - - TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::AddTagsToStreamRequest, - Ydb::DataStreams::V1::AddTagsToStreamResponse, - Ydb::DataStreams::V1::AddTagsToStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream); - } - - TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, - Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, - Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring); - } - - TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, - Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, - Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring); - } - - TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::ListTagsForStreamRequest, - Ydb::DataStreams::V1::ListTagsForStreamResponse, - Ydb::DataStreams::V1::ListTagsForStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream); - } - - TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::MergeShardsRequest, - Ydb::DataStreams::V1::MergeShardsResponse, - Ydb::DataStreams::V1::MergeShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards); - } - - TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, - Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, - Ydb::DataStreams::V1::RemoveTagsFromStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream); - } - - TAsyncSplitShardResult SplitShard(TSplitShardSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::SplitShardRequest, - Ydb::DataStreams::V1::SplitShardResponse, - Ydb::DataStreams::V1::SplitShardResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard); - } - - TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::StartStreamEncryptionRequest, - Ydb::DataStreams::V1::StartStreamEncryptionResponse, - Ydb::DataStreams::V1::StartStreamEncryptionResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption); - } - - TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::StopStreamEncryptionRequest, - Ydb::DataStreams::V1::StopStreamEncryptionResponse, - Ydb::DataStreams::V1::StopStreamEncryptionResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption); - } - - TAsyncUpdateStreamResult UpdateStream(const TString& streamName, TUpdateStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::UpdateStreamRequest, - Ydb::DataStreams::V1::UpdateStreamResponse, - Ydb::DataStreams::V1::UpdateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream, - [&](Ydb::DataStreams::V1::UpdateStreamRequest& req) { - req.set_stream_name(streamName); - req.set_target_shard_count(settings.TargetShardCount_); - req.set_retention_period_hours(settings.RetentionPeriodHours_); - req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); - }); - } - - TAsyncDeleteStreamResult DeleteStream(const TString &path, TDeleteStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DeleteStreamRequest, - Ydb::DataStreams::V1::DeleteStreamResponse, + } + + TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::AddTagsToStreamRequest, + Ydb::DataStreams::V1::AddTagsToStreamResponse, + Ydb::DataStreams::V1::AddTagsToStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream); + } + + TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring); + } + + TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring); + } + + TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::ListTagsForStreamRequest, + Ydb::DataStreams::V1::ListTagsForStreamResponse, + Ydb::DataStreams::V1::ListTagsForStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream); + } + + TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::MergeShardsRequest, + Ydb::DataStreams::V1::MergeShardsResponse, + Ydb::DataStreams::V1::MergeShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards); + } + + TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, + Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, + Ydb::DataStreams::V1::RemoveTagsFromStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream); + } + + TAsyncSplitShardResult SplitShard(TSplitShardSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::SplitShardRequest, + Ydb::DataStreams::V1::SplitShardResponse, + Ydb::DataStreams::V1::SplitShardResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard); + } + + TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::StartStreamEncryptionRequest, + Ydb::DataStreams::V1::StartStreamEncryptionResponse, + Ydb::DataStreams::V1::StartStreamEncryptionResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption); + } + + TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::StopStreamEncryptionRequest, + Ydb::DataStreams::V1::StopStreamEncryptionResponse, + Ydb::DataStreams::V1::StopStreamEncryptionResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption); + } + + TAsyncUpdateStreamResult UpdateStream(const TString& streamName, TUpdateStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::UpdateStreamRequest, + Ydb::DataStreams::V1::UpdateStreamResponse, + Ydb::DataStreams::V1::UpdateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream, + [&](Ydb::DataStreams::V1::UpdateStreamRequest& req) { + req.set_stream_name(streamName); + req.set_target_shard_count(settings.TargetShardCount_); + req.set_retention_period_hours(settings.RetentionPeriodHours_); + req.set_write_quota_kb_per_sec(settings.WriteQuotaKbPerSec_); + }); + } + + TAsyncDeleteStreamResult DeleteStream(const TString &path, TDeleteStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DeleteStreamRequest, + Ydb::DataStreams::V1::DeleteStreamResponse, Ydb::DataStreams::V1::DeleteStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream, [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) { req.set_stream_name(path); req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_); }); - } - - TAsyncDescribeStreamResult DescribeStream(const TString &path, TDescribeStreamSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::DescribeStreamRequest, - Ydb::DataStreams::V1::DescribeStreamResponse, - Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream, - [&](Ydb::DataStreams::V1::DescribeStreamRequest& req) { - req.set_stream_name(path); - req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); - req.set_limit(settings.Limit_); - }); - } - - TAsyncPutRecordResult PutRecord(const TString &path, const TDataRecord& record, TPutRecordSettings settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, - Ydb::DataStreams::V1::PutRecordRequest, - Ydb::DataStreams::V1::PutRecordResponse, - Ydb::DataStreams::V1::PutRecordResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord, - [&](Ydb::DataStreams::V1::PutRecordRequest& req) { - req.set_stream_name(path); - req.set_explicit_hash_key(record.ExplicitHashDecimal); - req.set_partition_key(record.PartitionKey); - req.set_data(record.Data); - }); - } - - template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> - NThreading::TFuture<TProtoResultWrapper<TProtoResult>> DoProtoRequest(const TProtoRequest& proto, TMethod method, const TProtoRequestSettings& settings) { - return CallImpl<Ydb::DataStreams::V1::DataStreamsService, TProtoRequest, TProtoResponse, TProtoResult>(settings, method, - [&](TProtoRequest& req) { - req.CopyFrom(proto); - }); - } - - }; - - TDataStreamsClient::TDataStreamsClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) - {} - - TAsyncCreateStreamResult TDataStreamsClient::CreateStream(const TString& path, TCreateStreamSettings settings) { - return Impl_->CreateStream(path, settings); - } - - TAsyncDeleteStreamResult TDataStreamsClient::DeleteStream(const TString& path, TDeleteStreamSettings settings) { - return Impl_->DeleteStream(path, settings); - } - - TAsyncDescribeStreamResult TDataStreamsClient::DescribeStream(const TString& path, TDescribeStreamSettings settings) { - return Impl_->DescribeStream(path, settings); - } - - TAsyncPutRecordResult TDataStreamsClient::PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings) { - return Impl_->PutRecord(path, record, settings); - } - - TAsyncListStreamsResult TDataStreamsClient::ListStreams(TListStreamsSettings settings) { - return Impl_->ListStreams(settings); - } - + } + + TAsyncDescribeStreamResult DescribeStream(const TString &path, TDescribeStreamSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::DescribeStreamRequest, + Ydb::DataStreams::V1::DescribeStreamResponse, + Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream, + [&](Ydb::DataStreams::V1::DescribeStreamRequest& req) { + req.set_stream_name(path); + req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); + req.set_limit(settings.Limit_); + }); + } + + TAsyncPutRecordResult PutRecord(const TString &path, const TDataRecord& record, TPutRecordSettings settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, + Ydb::DataStreams::V1::PutRecordRequest, + Ydb::DataStreams::V1::PutRecordResponse, + Ydb::DataStreams::V1::PutRecordResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord, + [&](Ydb::DataStreams::V1::PutRecordRequest& req) { + req.set_stream_name(path); + req.set_explicit_hash_key(record.ExplicitHashDecimal); + req.set_partition_key(record.PartitionKey); + req.set_data(record.Data); + }); + } + + template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> + NThreading::TFuture<TProtoResultWrapper<TProtoResult>> DoProtoRequest(const TProtoRequest& proto, TMethod method, const TProtoRequestSettings& settings) { + return CallImpl<Ydb::DataStreams::V1::DataStreamsService, TProtoRequest, TProtoResponse, TProtoResult>(settings, method, + [&](TProtoRequest& req) { + req.CopyFrom(proto); + }); + } + + }; + + TDataStreamsClient::TDataStreamsClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) + {} + + TAsyncCreateStreamResult TDataStreamsClient::CreateStream(const TString& path, TCreateStreamSettings settings) { + return Impl_->CreateStream(path, settings); + } + + TAsyncDeleteStreamResult TDataStreamsClient::DeleteStream(const TString& path, TDeleteStreamSettings settings) { + return Impl_->DeleteStream(path, settings); + } + + TAsyncDescribeStreamResult TDataStreamsClient::DescribeStream(const TString& path, TDescribeStreamSettings settings) { + return Impl_->DescribeStream(path, settings); + } + + TAsyncPutRecordResult TDataStreamsClient::PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings) { + return Impl_->PutRecord(path, record, settings); + } + + TAsyncListStreamsResult TDataStreamsClient::ListStreams(TListStreamsSettings settings) { + return Impl_->ListStreams(settings); + } + TAsyncListShardsResult TDataStreamsClient::ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings) { return Impl_->ListShards(path, shardFilter, settings); - } - - TAsyncPutRecordsResult TDataStreamsClient::PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { - return Impl_->PutRecords(path, records, settings); - } - + } + + TAsyncPutRecordsResult TDataStreamsClient::PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { + return Impl_->PutRecords(path, records, settings); + } + TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { return Impl_->GetRecords(shardIterator, settings); - } - + } + TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) { return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings); - } - - /* TAsyncSubscribeToShardResult TDataStreamsClient::SubscribeToShard(TSubscribeToShardSettings settings) { - return Impl_->SubscribeToShard(settings); - } */ - - TAsyncDescribeLimitsResult TDataStreamsClient::DescribeLimits(TDescribeLimitsSettings settings) { - return Impl_->DescribeLimits(settings); - } - + } + + /* TAsyncSubscribeToShardResult TDataStreamsClient::SubscribeToShard(TSubscribeToShardSettings settings) { + return Impl_->SubscribeToShard(settings); + } */ + + TAsyncDescribeLimitsResult TDataStreamsClient::DescribeLimits(TDescribeLimitsSettings settings) { + return Impl_->DescribeLimits(settings); + } + TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { return Impl_->DescribeStreamSummary(path, settings); - } - - TAsyncDecreaseStreamRetentionPeriodResult TDataStreamsClient::DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { - return Impl_->DecreaseStreamRetentionPeriod(path, settings); - } - - TAsyncIncreaseStreamRetentionPeriodResult TDataStreamsClient::IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings) { - return Impl_->IncreaseStreamRetentionPeriod(path, settings); - } - - TAsyncUpdateShardCountResult TDataStreamsClient::UpdateShardCount(const TString& path, TUpdateShardCountSettings settings) { - return Impl_->UpdateShardCount(path, settings); - } - + } + + TAsyncDecreaseStreamRetentionPeriodResult TDataStreamsClient::DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { + return Impl_->DecreaseStreamRetentionPeriod(path, settings); + } + + TAsyncIncreaseStreamRetentionPeriodResult TDataStreamsClient::IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings) { + return Impl_->IncreaseStreamRetentionPeriod(path, settings); + } + + TAsyncUpdateShardCountResult TDataStreamsClient::UpdateShardCount(const TString& path, TUpdateShardCountSettings settings) { + return Impl_->UpdateShardCount(path, settings); + } + TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const TString& path, const TString& consumer_name, const TRegisterStreamConsumerSettings settings) { return Impl_->RegisterStreamConsumer(path, consumer_name, settings); - } - + } + TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { return Impl_->DeregisterStreamConsumer(path, consumer_name, settings); - } - - TAsyncDescribeStreamConsumerResult TDataStreamsClient::DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { - return Impl_->DescribeStreamConsumer(settings); - } - + } + + TAsyncDescribeStreamConsumerResult TDataStreamsClient::DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { + return Impl_->DescribeStreamConsumer(settings); + } + TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { return Impl_->ListStreamConsumers(path, settings); - } - - TAsyncAddTagsToStreamResult TDataStreamsClient::AddTagsToStream(TAddTagsToStreamSettings settings) { - return Impl_->AddTagsToStream(settings); - } - - TAsyncDisableEnhancedMonitoringResult TDataStreamsClient::DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { - return Impl_->DisableEnhancedMonitoring(settings); - } - - TAsyncEnableEnhancedMonitoringResult TDataStreamsClient::EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { - return Impl_->EnableEnhancedMonitoring(settings); - } - - TAsyncListTagsForStreamResult TDataStreamsClient::ListTagsForStream(TListTagsForStreamSettings settings) { - return Impl_->ListTagsForStream(settings); - } - - TAsyncMergeShardsResult TDataStreamsClient::MergeShards(TMergeShardsSettings settings) { - return Impl_->MergeShards(settings); - } - - TAsyncRemoveTagsFromStreamResult TDataStreamsClient::RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { - return Impl_->RemoveTagsFromStream(settings); - } - - TAsyncSplitShardResult TDataStreamsClient::SplitShard(TSplitShardSettings settings) { - return Impl_->SplitShard(settings); - } - - TAsyncStartStreamEncryptionResult TDataStreamsClient::StartStreamEncryption(TStartStreamEncryptionSettings settings) { - return Impl_->StartStreamEncryption(settings); - } - - TAsyncStopStreamEncryptionResult TDataStreamsClient::StopStreamEncryption(TStopStreamEncryptionSettings settings) { - return Impl_->StopStreamEncryption(settings); - } - - TAsyncUpdateStreamResult TDataStreamsClient::UpdateStream(const TString& streamName, TUpdateStreamSettings settings) { - return Impl_->UpdateStream(streamName, settings); - } - - template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> - NThreading::TFuture<TProtoResultWrapper<TProtoResult>> TDataStreamsClient::DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings) { - return Impl_->DoProtoRequest<TProtoRequest, TProtoResponse, TProtoResult, TMethod>(request, method, settings); - } - - - // Instantiate template protobuf methods - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::PutRecordsRequest, - Ydb::DataStreams::V1::PutRecordsResponse, - Ydb::DataStreams::V1::PutRecordsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) - >( - const Ydb::DataStreams::V1::PutRecordsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordResult>>TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::PutRecordRequest, - Ydb::DataStreams::V1::PutRecordResponse, - Ydb::DataStreams::V1::PutRecordResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) - >( - const Ydb::DataStreams::V1::PutRecordRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::ListStreamsRequest, - Ydb::DataStreams::V1::ListStreamsResponse, - Ydb::DataStreams::V1::ListStreamsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) - >( - const Ydb::DataStreams::V1::ListStreamsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::CreateStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::CreateStreamRequest, - Ydb::DataStreams::V1::CreateStreamResponse, - Ydb::DataStreams::V1::CreateStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) - >( - const Ydb::DataStreams::V1::CreateStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DeleteStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DeleteStreamRequest, - Ydb::DataStreams::V1::DeleteStreamResponse, - Ydb::DataStreams::V1::DeleteStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) - >( - const Ydb::DataStreams::V1::DeleteStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DescribeStreamRequest, - Ydb::DataStreams::V1::DescribeStreamResponse, - Ydb::DataStreams::V1::DescribeStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) - >( - const Ydb::DataStreams::V1::DescribeStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListShardsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::ListShardsRequest, - Ydb::DataStreams::V1::ListShardsResponse, - Ydb::DataStreams::V1::ListShardsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) - >( - const Ydb::DataStreams::V1::ListShardsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::GetRecordsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::GetRecordsRequest, - Ydb::DataStreams::V1::GetRecordsResponse, - Ydb::DataStreams::V1::GetRecordsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) - >( - const Ydb::DataStreams::V1::GetRecordsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::GetShardIteratorResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::GetShardIteratorRequest, - Ydb::DataStreams::V1::GetShardIteratorResponse, - Ydb::DataStreams::V1::GetShardIteratorResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) - >( - const Ydb::DataStreams::V1::GetShardIteratorRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeLimitsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DescribeLimitsRequest, - Ydb::DataStreams::V1::DescribeLimitsResponse, - Ydb::DataStreams::V1::DescribeLimitsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) - >( - const Ydb::DataStreams::V1::DescribeLimitsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, - Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) - >( - const Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, - Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) - >( - const Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::UpdateShardCountResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::UpdateShardCountRequest, - Ydb::DataStreams::V1::UpdateShardCountResponse, - Ydb::DataStreams::V1::UpdateShardCountResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) - >( - const Ydb::DataStreams::V1::UpdateShardCountRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::RegisterStreamConsumerResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::RegisterStreamConsumerRequest, - Ydb::DataStreams::V1::RegisterStreamConsumerResponse, - Ydb::DataStreams::V1::RegisterStreamConsumerResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) - >( - const Ydb::DataStreams::V1::RegisterStreamConsumerRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DeregisterStreamConsumerResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, - Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, - Ydb::DataStreams::V1::DeregisterStreamConsumerResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) - >( - const Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamConsumerResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DescribeStreamConsumerRequest, - Ydb::DataStreams::V1::DescribeStreamConsumerResponse, - Ydb::DataStreams::V1::DescribeStreamConsumerResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) - >( - const Ydb::DataStreams::V1::DescribeStreamConsumerRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamConsumersResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::ListStreamConsumersRequest, - Ydb::DataStreams::V1::ListStreamConsumersResponse, - Ydb::DataStreams::V1::ListStreamConsumersResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) - >( - const Ydb::DataStreams::V1::ListStreamConsumersRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::AddTagsToStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::AddTagsToStreamRequest, - Ydb::DataStreams::V1::AddTagsToStreamResponse, - Ydb::DataStreams::V1::AddTagsToStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) - >( - const Ydb::DataStreams::V1::AddTagsToStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, - Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, - Ydb::DataStreams::V1::DisableEnhancedMonitoringResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) - >( - const Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, - Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, - Ydb::DataStreams::V1::EnableEnhancedMonitoringResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) - >( - const Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::MergeShardsResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::MergeShardsRequest, - Ydb::DataStreams::V1::MergeShardsResponse, - Ydb::DataStreams::V1::MergeShardsResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) - >( - const Ydb::DataStreams::V1::MergeShardsRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListTagsForStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::ListTagsForStreamRequest, - Ydb::DataStreams::V1::ListTagsForStreamResponse, - Ydb::DataStreams::V1::ListTagsForStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) - >( - const Ydb::DataStreams::V1::ListTagsForStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::RemoveTagsFromStreamResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, - Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, - Ydb::DataStreams::V1::RemoveTagsFromStreamResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) - >( - const Ydb::DataStreams::V1::RemoveTagsFromStreamRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::SplitShardResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::SplitShardRequest, - Ydb::DataStreams::V1::SplitShardResponse, - Ydb::DataStreams::V1::SplitShardResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) - >( - const Ydb::DataStreams::V1::SplitShardRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::StartStreamEncryptionResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::StartStreamEncryptionRequest, - Ydb::DataStreams::V1::StartStreamEncryptionResponse, - Ydb::DataStreams::V1::StartStreamEncryptionResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) - >( - const Ydb::DataStreams::V1::StartStreamEncryptionRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::StopStreamEncryptionResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::StopStreamEncryptionRequest, - Ydb::DataStreams::V1::StopStreamEncryptionResponse, - Ydb::DataStreams::V1::StopStreamEncryptionResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) - >( - const Ydb::DataStreams::V1::StopStreamEncryptionRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) method, - TProtoRequestSettings settings - ); - - template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamSummaryResult>> TDataStreamsClient::DoProtoRequest - < - Ydb::DataStreams::V1::DescribeStreamSummaryRequest, - Ydb::DataStreams::V1::DescribeStreamSummaryResponse, - Ydb::DataStreams::V1::DescribeStreamSummaryResult, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) - >( - const Ydb::DataStreams::V1::DescribeStreamSummaryRequest& request, - decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) method, - TProtoRequestSettings settings - ); - - -} - + } + + TAsyncAddTagsToStreamResult TDataStreamsClient::AddTagsToStream(TAddTagsToStreamSettings settings) { + return Impl_->AddTagsToStream(settings); + } + + TAsyncDisableEnhancedMonitoringResult TDataStreamsClient::DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings) { + return Impl_->DisableEnhancedMonitoring(settings); + } + + TAsyncEnableEnhancedMonitoringResult TDataStreamsClient::EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings) { + return Impl_->EnableEnhancedMonitoring(settings); + } + + TAsyncListTagsForStreamResult TDataStreamsClient::ListTagsForStream(TListTagsForStreamSettings settings) { + return Impl_->ListTagsForStream(settings); + } + + TAsyncMergeShardsResult TDataStreamsClient::MergeShards(TMergeShardsSettings settings) { + return Impl_->MergeShards(settings); + } + + TAsyncRemoveTagsFromStreamResult TDataStreamsClient::RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings) { + return Impl_->RemoveTagsFromStream(settings); + } + + TAsyncSplitShardResult TDataStreamsClient::SplitShard(TSplitShardSettings settings) { + return Impl_->SplitShard(settings); + } + + TAsyncStartStreamEncryptionResult TDataStreamsClient::StartStreamEncryption(TStartStreamEncryptionSettings settings) { + return Impl_->StartStreamEncryption(settings); + } + + TAsyncStopStreamEncryptionResult TDataStreamsClient::StopStreamEncryption(TStopStreamEncryptionSettings settings) { + return Impl_->StopStreamEncryption(settings); + } + + TAsyncUpdateStreamResult TDataStreamsClient::UpdateStream(const TString& streamName, TUpdateStreamSettings settings) { + return Impl_->UpdateStream(streamName, settings); + } + + template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> + NThreading::TFuture<TProtoResultWrapper<TProtoResult>> TDataStreamsClient::DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings) { + return Impl_->DoProtoRequest<TProtoRequest, TProtoResponse, TProtoResult, TMethod>(request, method, settings); + } + + + // Instantiate template protobuf methods + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::PutRecordsRequest, + Ydb::DataStreams::V1::PutRecordsResponse, + Ydb::DataStreams::V1::PutRecordsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) + >( + const Ydb::DataStreams::V1::PutRecordsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordResult>>TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::PutRecordRequest, + Ydb::DataStreams::V1::PutRecordResponse, + Ydb::DataStreams::V1::PutRecordResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) + >( + const Ydb::DataStreams::V1::PutRecordRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListStreamsRequest, + Ydb::DataStreams::V1::ListStreamsResponse, + Ydb::DataStreams::V1::ListStreamsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) + >( + const Ydb::DataStreams::V1::ListStreamsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::CreateStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::CreateStreamRequest, + Ydb::DataStreams::V1::CreateStreamResponse, + Ydb::DataStreams::V1::CreateStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) + >( + const Ydb::DataStreams::V1::CreateStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DeleteStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DeleteStreamRequest, + Ydb::DataStreams::V1::DeleteStreamResponse, + Ydb::DataStreams::V1::DeleteStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) + >( + const Ydb::DataStreams::V1::DeleteStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamRequest, + Ydb::DataStreams::V1::DescribeStreamResponse, + Ydb::DataStreams::V1::DescribeStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) + >( + const Ydb::DataStreams::V1::DescribeStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListShardsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListShardsRequest, + Ydb::DataStreams::V1::ListShardsResponse, + Ydb::DataStreams::V1::ListShardsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) + >( + const Ydb::DataStreams::V1::ListShardsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::GetRecordsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::GetRecordsRequest, + Ydb::DataStreams::V1::GetRecordsResponse, + Ydb::DataStreams::V1::GetRecordsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) + >( + const Ydb::DataStreams::V1::GetRecordsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::GetShardIteratorResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::GetShardIteratorRequest, + Ydb::DataStreams::V1::GetShardIteratorResponse, + Ydb::DataStreams::V1::GetShardIteratorResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) + >( + const Ydb::DataStreams::V1::GetShardIteratorRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeLimitsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeLimitsRequest, + Ydb::DataStreams::V1::DescribeLimitsResponse, + Ydb::DataStreams::V1::DescribeLimitsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) + >( + const Ydb::DataStreams::V1::DescribeLimitsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) + >( + const Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, + Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) + >( + const Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::UpdateShardCountResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::UpdateShardCountRequest, + Ydb::DataStreams::V1::UpdateShardCountResponse, + Ydb::DataStreams::V1::UpdateShardCountResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) + >( + const Ydb::DataStreams::V1::UpdateShardCountRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::RegisterStreamConsumerResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::RegisterStreamConsumerRequest, + Ydb::DataStreams::V1::RegisterStreamConsumerResponse, + Ydb::DataStreams::V1::RegisterStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) + >( + const Ydb::DataStreams::V1::RegisterStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DeregisterStreamConsumerResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, + Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, + Ydb::DataStreams::V1::DeregisterStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) + >( + const Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamConsumerResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamConsumerRequest, + Ydb::DataStreams::V1::DescribeStreamConsumerResponse, + Ydb::DataStreams::V1::DescribeStreamConsumerResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) + >( + const Ydb::DataStreams::V1::DescribeStreamConsumerRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamConsumersResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListStreamConsumersRequest, + Ydb::DataStreams::V1::ListStreamConsumersResponse, + Ydb::DataStreams::V1::ListStreamConsumersResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) + >( + const Ydb::DataStreams::V1::ListStreamConsumersRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::AddTagsToStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::AddTagsToStreamRequest, + Ydb::DataStreams::V1::AddTagsToStreamResponse, + Ydb::DataStreams::V1::AddTagsToStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) + >( + const Ydb::DataStreams::V1::AddTagsToStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncAddTagsToStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::DisableEnhancedMonitoringResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) + >( + const Ydb::DataStreams::V1::DisableEnhancedMonitoringRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDisableEnhancedMonitoring) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResponse, + Ydb::DataStreams::V1::EnableEnhancedMonitoringResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) + >( + const Ydb::DataStreams::V1::EnableEnhancedMonitoringRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncEnableEnhancedMonitoring) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::MergeShardsResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::MergeShardsRequest, + Ydb::DataStreams::V1::MergeShardsResponse, + Ydb::DataStreams::V1::MergeShardsResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) + >( + const Ydb::DataStreams::V1::MergeShardsRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncMergeShards) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::ListTagsForStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::ListTagsForStreamRequest, + Ydb::DataStreams::V1::ListTagsForStreamResponse, + Ydb::DataStreams::V1::ListTagsForStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) + >( + const Ydb::DataStreams::V1::ListTagsForStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListTagsForStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::RemoveTagsFromStreamResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::RemoveTagsFromStreamRequest, + Ydb::DataStreams::V1::RemoveTagsFromStreamResponse, + Ydb::DataStreams::V1::RemoveTagsFromStreamResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) + >( + const Ydb::DataStreams::V1::RemoveTagsFromStreamRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRemoveTagsFromStream) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::SplitShardResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::SplitShardRequest, + Ydb::DataStreams::V1::SplitShardResponse, + Ydb::DataStreams::V1::SplitShardResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) + >( + const Ydb::DataStreams::V1::SplitShardRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncSplitShard) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::StartStreamEncryptionResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::StartStreamEncryptionRequest, + Ydb::DataStreams::V1::StartStreamEncryptionResponse, + Ydb::DataStreams::V1::StartStreamEncryptionResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) + >( + const Ydb::DataStreams::V1::StartStreamEncryptionRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStartStreamEncryption) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::StopStreamEncryptionResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::StopStreamEncryptionRequest, + Ydb::DataStreams::V1::StopStreamEncryptionResponse, + Ydb::DataStreams::V1::StopStreamEncryptionResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) + >( + const Ydb::DataStreams::V1::StopStreamEncryptionRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncStopStreamEncryption) method, + TProtoRequestSettings settings + ); + + template NThreading::TFuture<TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamSummaryResult>> TDataStreamsClient::DoProtoRequest + < + Ydb::DataStreams::V1::DescribeStreamSummaryRequest, + Ydb::DataStreams::V1::DescribeStreamSummaryResponse, + Ydb::DataStreams::V1::DescribeStreamSummaryResult, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) + >( + const Ydb::DataStreams::V1::DescribeStreamSummaryRequest& request, + decltype(&Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary) method, + TProtoRequestSettings settings + ); + + +} + diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h index 2d208fded9..783f51080d 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h @@ -1,116 +1,116 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h> - -namespace NYdb::NDataStreams::V1 { - - template<class TProtoResult> - class TProtoResultWrapper : public NYdb::TStatus { - friend class TDataStreamsClient; - - private: - TProtoResultWrapper( - NYdb::TStatus&& status, - std::unique_ptr<TProtoResult> result) - : TStatus(std::move(status)) - , Result(std::move(result)) - { } - - public: - const TProtoResult& GetResult() const { - Y_VERIFY(Result, "Uninitialized result"); - return *Result; - } - - private: - std::unique_ptr<TProtoResult> Result; - }; - - using TCreateStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::CreateStreamResult>; - using TDeleteStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::DeleteStreamResult>; - using TDescribeStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamResult>; - using TPutRecordResult = TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordResult>; - using TRegisterStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::RegisterStreamConsumerResult>; - using TDeregisterStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::DeregisterStreamConsumerResult>; - using TDescribeStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamConsumerResult>; - using TListStreamsResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamsResult>; - using TListShardsResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListShardsResult>; - using TPutRecordsResult = TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordsResult>; - using TGetRecordsResult = TProtoResultWrapper<Ydb::DataStreams::V1::GetRecordsResult>; - using TGetShardIteratorResult = TProtoResultWrapper<Ydb::DataStreams::V1::GetShardIteratorResult>; - // using TSubscribeToShardResult = TProtoResultWrapper<Ydb::DataStreams::V1::SubscribeToShardResult>; - using TDescribeLimitsResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeLimitsResult>; - using TDescribeStreamSummaryResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamSummaryResult>; - using TDecreaseStreamRetentionPeriodResult = TProtoResultWrapper<Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>; - using TIncreaseStreamRetentionPeriodResult = TProtoResultWrapper<Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>; - using TUpdateShardCountResult = TProtoResultWrapper<Ydb::DataStreams::V1::UpdateShardCountResult>; - using TListStreamConsumersResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamConsumersResult>; - using TAddTagsToStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::AddTagsToStreamResult>; - using TDisableEnhancedMonitoringResult = TProtoResultWrapper<Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>; - using TEnableEnhancedMonitoringResult = TProtoResultWrapper<Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>; - using TListTagsForStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListTagsForStreamResult>; - using TMergeShardsResult = TProtoResultWrapper<Ydb::DataStreams::V1::MergeShardsResult>; - using TRemoveTagsFromStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::RemoveTagsFromStreamResult>; - using TSplitShardResult = TProtoResultWrapper<Ydb::DataStreams::V1::SplitShardResult>; - using TStartStreamEncryptionResult = TProtoResultWrapper<Ydb::DataStreams::V1::StartStreamEncryptionResult>; - using TStopStreamEncryptionResult = TProtoResultWrapper<Ydb::DataStreams::V1::StopStreamEncryptionResult>; - using TUpdateStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::UpdateStreamResult>; - - using TAsyncCreateStreamResult = NThreading::TFuture<TCreateStreamResult>; - using TAsyncDeleteStreamResult = NThreading::TFuture<TDeleteStreamResult>; - using TAsyncDescribeStreamResult = NThreading::TFuture<TDescribeStreamResult>; - using TAsyncPutRecordResult = NThreading::TFuture<TPutRecordResult>; - using TAsyncRegisterStreamConsumerResult = NThreading::TFuture<TRegisterStreamConsumerResult>; - using TAsyncDeregisterStreamConsumerResult = NThreading::TFuture<TDeregisterStreamConsumerResult>; - using TAsyncDescribeStreamConsumerResult = NThreading::TFuture<TDescribeStreamConsumerResult>; - using TAsyncListStreamsResult = NThreading::TFuture<TListStreamsResult>; - using TAsyncListShardsResult = NThreading::TFuture<TListShardsResult>; - using TAsyncPutRecordsResult = NThreading::TFuture<TPutRecordsResult>; - using TAsyncGetRecordsResult = NThreading::TFuture<TGetRecordsResult>; - using TAsyncGetShardIteratorResult = NThreading::TFuture<TGetShardIteratorResult>; - // using TAsyncSubscribeToShardResult = NThreading::TFuture<TSubscribeToShardResult>; - using TAsyncDescribeLimitsResult = NThreading::TFuture<TDescribeLimitsResult>; - using TAsyncDescribeStreamSummaryResult = NThreading::TFuture<TDescribeStreamSummaryResult>; - using TAsyncDecreaseStreamRetentionPeriodResult = NThreading::TFuture<TDecreaseStreamRetentionPeriodResult>; - using TAsyncIncreaseStreamRetentionPeriodResult = NThreading::TFuture<TIncreaseStreamRetentionPeriodResult>; - using TAsyncUpdateShardCountResult = NThreading::TFuture<TUpdateShardCountResult>; - using TAsyncListStreamConsumersResult = NThreading::TFuture<TListStreamConsumersResult>; - using TAsyncAddTagsToStreamResult = NThreading::TFuture<TAddTagsToStreamResult>; - using TAsyncDisableEnhancedMonitoringResult = NThreading::TFuture<TDisableEnhancedMonitoringResult>; - using TAsyncEnableEnhancedMonitoringResult = NThreading::TFuture<TEnableEnhancedMonitoringResult>; - using TAsyncListTagsForStreamResult = NThreading::TFuture<TListTagsForStreamResult>; - using TAsyncMergeShardsResult = NThreading::TFuture<TMergeShardsResult>; - using TAsyncRemoveTagsFromStreamResult = NThreading::TFuture<TRemoveTagsFromStreamResult>; - using TAsyncSplitShardResult = NThreading::TFuture<TSplitShardResult>; - using TAsyncStartStreamEncryptionResult = NThreading::TFuture<TStartStreamEncryptionResult>; - using TAsyncStopStreamEncryptionResult = NThreading::TFuture<TStopStreamEncryptionResult>; - using TAsyncUpdateStreamResult = NThreading::TFuture<TUpdateStreamResult>; - - struct TDataRecord { - TString Data; - TString PartitionKey; - TString ExplicitHashDecimal; - }; - - struct TCreateStreamSettings : public NYdb::TOperationRequestSettings<TCreateStreamSettings> { - FLUENT_SETTING(ui32, ShardCount); - FLUENT_SETTING(ui32, RetentionPeriodHours); - FLUENT_SETTING(ui64, WriteQuotaKbPerSec); - }; - struct TListStreamsSettings : public NYdb::TOperationRequestSettings<TListStreamsSettings> { - FLUENT_SETTING(ui32, Limit); - FLUENT_SETTING(TString, ExclusiveStartStreamName); + +namespace NYdb::NDataStreams::V1 { + + template<class TProtoResult> + class TProtoResultWrapper : public NYdb::TStatus { + friend class TDataStreamsClient; + + private: + TProtoResultWrapper( + NYdb::TStatus&& status, + std::unique_ptr<TProtoResult> result) + : TStatus(std::move(status)) + , Result(std::move(result)) + { } + + public: + const TProtoResult& GetResult() const { + Y_VERIFY(Result, "Uninitialized result"); + return *Result; + } + + private: + std::unique_ptr<TProtoResult> Result; + }; + + using TCreateStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::CreateStreamResult>; + using TDeleteStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::DeleteStreamResult>; + using TDescribeStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamResult>; + using TPutRecordResult = TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordResult>; + using TRegisterStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::RegisterStreamConsumerResult>; + using TDeregisterStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::DeregisterStreamConsumerResult>; + using TDescribeStreamConsumerResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamConsumerResult>; + using TListStreamsResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamsResult>; + using TListShardsResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListShardsResult>; + using TPutRecordsResult = TProtoResultWrapper<Ydb::DataStreams::V1::PutRecordsResult>; + using TGetRecordsResult = TProtoResultWrapper<Ydb::DataStreams::V1::GetRecordsResult>; + using TGetShardIteratorResult = TProtoResultWrapper<Ydb::DataStreams::V1::GetShardIteratorResult>; + // using TSubscribeToShardResult = TProtoResultWrapper<Ydb::DataStreams::V1::SubscribeToShardResult>; + using TDescribeLimitsResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeLimitsResult>; + using TDescribeStreamSummaryResult = TProtoResultWrapper<Ydb::DataStreams::V1::DescribeStreamSummaryResult>; + using TDecreaseStreamRetentionPeriodResult = TProtoResultWrapper<Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>; + using TIncreaseStreamRetentionPeriodResult = TProtoResultWrapper<Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>; + using TUpdateShardCountResult = TProtoResultWrapper<Ydb::DataStreams::V1::UpdateShardCountResult>; + using TListStreamConsumersResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListStreamConsumersResult>; + using TAddTagsToStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::AddTagsToStreamResult>; + using TDisableEnhancedMonitoringResult = TProtoResultWrapper<Ydb::DataStreams::V1::DisableEnhancedMonitoringResult>; + using TEnableEnhancedMonitoringResult = TProtoResultWrapper<Ydb::DataStreams::V1::EnableEnhancedMonitoringResult>; + using TListTagsForStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::ListTagsForStreamResult>; + using TMergeShardsResult = TProtoResultWrapper<Ydb::DataStreams::V1::MergeShardsResult>; + using TRemoveTagsFromStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::RemoveTagsFromStreamResult>; + using TSplitShardResult = TProtoResultWrapper<Ydb::DataStreams::V1::SplitShardResult>; + using TStartStreamEncryptionResult = TProtoResultWrapper<Ydb::DataStreams::V1::StartStreamEncryptionResult>; + using TStopStreamEncryptionResult = TProtoResultWrapper<Ydb::DataStreams::V1::StopStreamEncryptionResult>; + using TUpdateStreamResult = TProtoResultWrapper<Ydb::DataStreams::V1::UpdateStreamResult>; + + using TAsyncCreateStreamResult = NThreading::TFuture<TCreateStreamResult>; + using TAsyncDeleteStreamResult = NThreading::TFuture<TDeleteStreamResult>; + using TAsyncDescribeStreamResult = NThreading::TFuture<TDescribeStreamResult>; + using TAsyncPutRecordResult = NThreading::TFuture<TPutRecordResult>; + using TAsyncRegisterStreamConsumerResult = NThreading::TFuture<TRegisterStreamConsumerResult>; + using TAsyncDeregisterStreamConsumerResult = NThreading::TFuture<TDeregisterStreamConsumerResult>; + using TAsyncDescribeStreamConsumerResult = NThreading::TFuture<TDescribeStreamConsumerResult>; + using TAsyncListStreamsResult = NThreading::TFuture<TListStreamsResult>; + using TAsyncListShardsResult = NThreading::TFuture<TListShardsResult>; + using TAsyncPutRecordsResult = NThreading::TFuture<TPutRecordsResult>; + using TAsyncGetRecordsResult = NThreading::TFuture<TGetRecordsResult>; + using TAsyncGetShardIteratorResult = NThreading::TFuture<TGetShardIteratorResult>; + // using TAsyncSubscribeToShardResult = NThreading::TFuture<TSubscribeToShardResult>; + using TAsyncDescribeLimitsResult = NThreading::TFuture<TDescribeLimitsResult>; + using TAsyncDescribeStreamSummaryResult = NThreading::TFuture<TDescribeStreamSummaryResult>; + using TAsyncDecreaseStreamRetentionPeriodResult = NThreading::TFuture<TDecreaseStreamRetentionPeriodResult>; + using TAsyncIncreaseStreamRetentionPeriodResult = NThreading::TFuture<TIncreaseStreamRetentionPeriodResult>; + using TAsyncUpdateShardCountResult = NThreading::TFuture<TUpdateShardCountResult>; + using TAsyncListStreamConsumersResult = NThreading::TFuture<TListStreamConsumersResult>; + using TAsyncAddTagsToStreamResult = NThreading::TFuture<TAddTagsToStreamResult>; + using TAsyncDisableEnhancedMonitoringResult = NThreading::TFuture<TDisableEnhancedMonitoringResult>; + using TAsyncEnableEnhancedMonitoringResult = NThreading::TFuture<TEnableEnhancedMonitoringResult>; + using TAsyncListTagsForStreamResult = NThreading::TFuture<TListTagsForStreamResult>; + using TAsyncMergeShardsResult = NThreading::TFuture<TMergeShardsResult>; + using TAsyncRemoveTagsFromStreamResult = NThreading::TFuture<TRemoveTagsFromStreamResult>; + using TAsyncSplitShardResult = NThreading::TFuture<TSplitShardResult>; + using TAsyncStartStreamEncryptionResult = NThreading::TFuture<TStartStreamEncryptionResult>; + using TAsyncStopStreamEncryptionResult = NThreading::TFuture<TStopStreamEncryptionResult>; + using TAsyncUpdateStreamResult = NThreading::TFuture<TUpdateStreamResult>; + + struct TDataRecord { + TString Data; + TString PartitionKey; + TString ExplicitHashDecimal; + }; + + struct TCreateStreamSettings : public NYdb::TOperationRequestSettings<TCreateStreamSettings> { + FLUENT_SETTING(ui32, ShardCount); + FLUENT_SETTING(ui32, RetentionPeriodHours); + FLUENT_SETTING(ui64, WriteQuotaKbPerSec); + }; + struct TListStreamsSettings : public NYdb::TOperationRequestSettings<TListStreamsSettings> { + FLUENT_SETTING(ui32, Limit); + FLUENT_SETTING(TString, ExclusiveStartStreamName); FLUENT_SETTING_DEFAULT(bool, Recurse, false); - }; + }; struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings<TDeleteStreamSettings> { FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false); }; - struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings<TDescribeStreamSettings> { - FLUENT_SETTING(ui32, Limit); - FLUENT_SETTING(TString, ExclusiveStartShardId); - }; + struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings<TDescribeStreamSettings> { + FLUENT_SETTING(ui32, Limit); + FLUENT_SETTING(TString, ExclusiveStartShardId); + }; struct TListShardsSettings : public NYdb::TOperationRequestSettings<TListShardsSettings> { FLUENT_SETTING(TString, ExclusiveStartShardId); FLUENT_SETTING(ui32, MaxResults); @@ -124,85 +124,85 @@ namespace NYdb::NDataStreams::V1 { FLUENT_SETTING(TString, StartingSequenceNumber); FLUENT_SETTING(ui64, Timestamp); }; - struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings<TSubscribeToShardSettings> {}; - struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings<TDescribeLimitsSettings> {}; - struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings<TDescribeStreamSummarySettings> {}; - struct TDecreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings<TDecreaseStreamRetentionPeriodSettings> { - FLUENT_SETTING(ui32, RetentionPeriodHours); - }; - struct TIncreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings<TIncreaseStreamRetentionPeriodSettings> { - FLUENT_SETTING(ui32, RetentionPeriodHours); - }; - struct TUpdateShardCountSettings : public NYdb::TOperationRequestSettings<TUpdateShardCountSettings> { - FLUENT_SETTING(ui32, TargetShardCount); - }; - struct TUpdateStreamSettings : public NYdb::TOperationRequestSettings<TUpdateStreamSettings> { - FLUENT_SETTING(ui32, TargetShardCount); - FLUENT_SETTING(ui32, RetentionPeriodHours); - FLUENT_SETTING(ui64, WriteQuotaKbPerSec); - }; - struct TPutRecordSettings : public NYdb::TOperationRequestSettings<TPutRecordSettings> {}; - struct TPutRecordsSettings : public NYdb::TOperationRequestSettings<TPutRecordsSettings> {}; - struct TRegisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TRegisterStreamConsumerSettings> {}; - struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDeregisterStreamConsumerSettings> {}; - struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDescribeStreamConsumerSettings> {}; + struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings<TSubscribeToShardSettings> {}; + struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings<TDescribeLimitsSettings> {}; + struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings<TDescribeStreamSummarySettings> {}; + struct TDecreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings<TDecreaseStreamRetentionPeriodSettings> { + FLUENT_SETTING(ui32, RetentionPeriodHours); + }; + struct TIncreaseStreamRetentionPeriodSettings : public NYdb::TOperationRequestSettings<TIncreaseStreamRetentionPeriodSettings> { + FLUENT_SETTING(ui32, RetentionPeriodHours); + }; + struct TUpdateShardCountSettings : public NYdb::TOperationRequestSettings<TUpdateShardCountSettings> { + FLUENT_SETTING(ui32, TargetShardCount); + }; + struct TUpdateStreamSettings : public NYdb::TOperationRequestSettings<TUpdateStreamSettings> { + FLUENT_SETTING(ui32, TargetShardCount); + FLUENT_SETTING(ui32, RetentionPeriodHours); + FLUENT_SETTING(ui64, WriteQuotaKbPerSec); + }; + struct TPutRecordSettings : public NYdb::TOperationRequestSettings<TPutRecordSettings> {}; + struct TPutRecordsSettings : public NYdb::TOperationRequestSettings<TPutRecordsSettings> {}; + struct TRegisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TRegisterStreamConsumerSettings> {}; + struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDeregisterStreamConsumerSettings> {}; + struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDescribeStreamConsumerSettings> {}; struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings<TListStreamConsumersSettings> { FLUENT_SETTING(ui32, MaxResults); FLUENT_SETTING(TString, NextToken); }; - struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings<TAddTagsToStreamSettings> {}; - struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TDisableEnhancedMonitoringSettings> {}; - struct TEnableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TEnableEnhancedMonitoringSettings> {}; - struct TListTagsForStreamSettings : public NYdb::TOperationRequestSettings<TListTagsForStreamSettings> {}; - struct TMergeShardsSettings : public NYdb::TOperationRequestSettings<TMergeShardsSettings> {}; - struct TRemoveTagsFromStreamSettings : public NYdb::TOperationRequestSettings<TRemoveTagsFromStreamSettings> {}; - struct TSplitShardSettings : public NYdb::TOperationRequestSettings<TSplitShardSettings> {}; - struct TStartStreamEncryptionSettings : public NYdb::TOperationRequestSettings<TStartStreamEncryptionSettings> {}; - struct TStopStreamEncryptionSettings : public NYdb::TOperationRequestSettings<TStopStreamEncryptionSettings> {}; - struct TProtoRequestSettings : public NYdb::TOperationRequestSettings<TProtoRequestSettings> {}; - - class TDataStreamsClient { - class TImpl; - - public: - TDataStreamsClient(const NYdb::TDriver& driver, const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings()); - - TAsyncCreateStreamResult CreateStream(const TString& path, TCreateStreamSettings settings = TCreateStreamSettings()); - TAsyncDeleteStreamResult DeleteStream(const TString& path, TDeleteStreamSettings settings = TDeleteStreamSettings()); - TAsyncDescribeStreamResult DescribeStream(const TString& path, TDescribeStreamSettings settings = TDescribeStreamSettings()); - TAsyncPutRecordResult PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings = TPutRecordSettings()); - TAsyncListStreamsResult ListStreams(TListStreamsSettings settings = TListStreamsSettings()); + struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings<TAddTagsToStreamSettings> {}; + struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TDisableEnhancedMonitoringSettings> {}; + struct TEnableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TEnableEnhancedMonitoringSettings> {}; + struct TListTagsForStreamSettings : public NYdb::TOperationRequestSettings<TListTagsForStreamSettings> {}; + struct TMergeShardsSettings : public NYdb::TOperationRequestSettings<TMergeShardsSettings> {}; + struct TRemoveTagsFromStreamSettings : public NYdb::TOperationRequestSettings<TRemoveTagsFromStreamSettings> {}; + struct TSplitShardSettings : public NYdb::TOperationRequestSettings<TSplitShardSettings> {}; + struct TStartStreamEncryptionSettings : public NYdb::TOperationRequestSettings<TStartStreamEncryptionSettings> {}; + struct TStopStreamEncryptionSettings : public NYdb::TOperationRequestSettings<TStopStreamEncryptionSettings> {}; + struct TProtoRequestSettings : public NYdb::TOperationRequestSettings<TProtoRequestSettings> {}; + + class TDataStreamsClient { + class TImpl; + + public: + TDataStreamsClient(const NYdb::TDriver& driver, const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings()); + + TAsyncCreateStreamResult CreateStream(const TString& path, TCreateStreamSettings settings = TCreateStreamSettings()); + TAsyncDeleteStreamResult DeleteStream(const TString& path, TDeleteStreamSettings settings = TDeleteStreamSettings()); + TAsyncDescribeStreamResult DescribeStream(const TString& path, TDescribeStreamSettings settings = TDescribeStreamSettings()); + TAsyncPutRecordResult PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings = TPutRecordSettings()); + TAsyncListStreamsResult ListStreams(TListStreamsSettings settings = TListStreamsSettings()); TAsyncListShardsResult ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings()); - TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings = TPutRecordsSettings()); + TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings = TPutRecordsSettings()); TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings()); TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr, TGetShardIteratorSettings settings = TGetShardIteratorSettings()); - // TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings = TSubscribeToShardSettings()); - TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings = TDescribeLimitsSettings()); + // TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings = TSubscribeToShardSettings()); + TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings = TDescribeLimitsSettings()); TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings()); - TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings = TDecreaseStreamRetentionPeriodSettings()); - TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings = TIncreaseStreamRetentionPeriodSettings()); - TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings = TUpdateShardCountSettings()); + TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings = TDecreaseStreamRetentionPeriodSettings()); + TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings = TIncreaseStreamRetentionPeriodSettings()); + TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings = TUpdateShardCountSettings()); TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings()); TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings()); - TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings = TDescribeStreamConsumerSettings()); + TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings = TDescribeStreamConsumerSettings()); TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings()); - TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings = TAddTagsToStreamSettings()); - TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings = TDisableEnhancedMonitoringSettings()); - TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings = TEnableEnhancedMonitoringSettings()); - TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings = TListTagsForStreamSettings()); - TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings = TMergeShardsSettings()); - TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings = TRemoveTagsFromStreamSettings()); - TAsyncSplitShardResult SplitShard(TSplitShardSettings settings = TSplitShardSettings()); - TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings = TStartStreamEncryptionSettings()); - TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings = TStopStreamEncryptionSettings()); - TAsyncUpdateStreamResult UpdateStream(const TString& streamName, TUpdateStreamSettings settings = TUpdateStreamSettings()); - - template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> - NThreading::TFuture<TProtoResultWrapper<TProtoResult>> DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings = TProtoRequestSettings()); - - private: - std::shared_ptr<TImpl> Impl_; - }; - -} + TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings = TAddTagsToStreamSettings()); + TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings = TDisableEnhancedMonitoringSettings()); + TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings = TEnableEnhancedMonitoringSettings()); + TAsyncListTagsForStreamResult ListTagsForStream(TListTagsForStreamSettings settings = TListTagsForStreamSettings()); + TAsyncMergeShardsResult MergeShards(TMergeShardsSettings settings = TMergeShardsSettings()); + TAsyncRemoveTagsFromStreamResult RemoveTagsFromStream(TRemoveTagsFromStreamSettings settings = TRemoveTagsFromStreamSettings()); + TAsyncSplitShardResult SplitShard(TSplitShardSettings settings = TSplitShardSettings()); + TAsyncStartStreamEncryptionResult StartStreamEncryption(TStartStreamEncryptionSettings settings = TStartStreamEncryptionSettings()); + TAsyncStopStreamEncryptionResult StopStreamEncryption(TStopStreamEncryptionSettings settings = TStopStreamEncryptionSettings()); + TAsyncUpdateStreamResult UpdateStream(const TString& streamName, TUpdateStreamSettings settings = TUpdateStreamSettings()); + + template<class TProtoRequest, class TProtoResponse, class TProtoResult, class TMethod> + NThreading::TFuture<TProtoResultWrapper<TProtoResult>> DoProtoRequest(const TProtoRequest& request, TMethod method, TProtoRequestSettings settings = TProtoRequestSettings()); + + private: + std::shared_ptr<TImpl> Impl_; + }; + +} diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make index 6caf298299..3e94f96757 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make @@ -3,7 +3,7 @@ LIBRARY() OWNER(g:kikimr) SRCS( - datastreams.cpp + datastreams.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp index ff7089d083..a22ed23cea 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp @@ -158,14 +158,14 @@ TAsyncStatus TPersQueueClient::DropTopic(const TString& path, const TDropTopicSe return Impl_->DropTopic(path, settings); } -TAsyncStatus TPersQueueClient::AddReadRule(const TString& path, const TAddReadRuleSettings& settings) { - return Impl_->AddReadRule(path, settings); -} - -TAsyncStatus TPersQueueClient::RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& settings) { - return Impl_->RemoveReadRule(path, settings); -} - +TAsyncStatus TPersQueueClient::AddReadRule(const TString& path, const TAddReadRuleSettings& settings) { + return Impl_->AddReadRule(path, settings); +} + +TAsyncStatus TPersQueueClient::RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& settings) { + return Impl_->RemoveReadRule(path, settings); +} + TAsyncDescribeTopicResult TPersQueueClient::DescribeTopic(const TString& path, const TDescribeTopicSettings& settings) { return Impl_->DescribeTopic(path, settings); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h index 6636f35cf0..57e9dd43bf 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h @@ -30,19 +30,19 @@ public: { } - template<class TReadRule> - static void ConvertToProtoReadRule(const TReadRule& readRule, Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps) { - rrProps.set_consumer_name(readRule.ConsumerName_); - rrProps.set_important(readRule.Important_); - rrProps.set_starting_message_timestamp_ms(readRule.StartingMessageTimestamp_.MilliSeconds()); - rrProps.set_version(readRule.Version_); - rrProps.set_supported_format(static_cast<Ydb::PersQueue::V1::TopicSettings::Format>(readRule.SupportedFormat_)); - for (const auto& codec : readRule.SupportedCodecs_) { - rrProps.add_supported_codecs((static_cast<Ydb::PersQueue::V1::Codec>(codec))); - } + template<class TReadRule> + static void ConvertToProtoReadRule(const TReadRule& readRule, Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps) { + rrProps.set_consumer_name(readRule.ConsumerName_); + rrProps.set_important(readRule.Important_); + rrProps.set_starting_message_timestamp_ms(readRule.StartingMessageTimestamp_.MilliSeconds()); + rrProps.set_version(readRule.Version_); + rrProps.set_supported_format(static_cast<Ydb::PersQueue::V1::TopicSettings::Format>(readRule.SupportedFormat_)); + for (const auto& codec : readRule.SupportedCodecs_) { + rrProps.add_supported_codecs((static_cast<Ydb::PersQueue::V1::Codec>(codec))); + } rrProps.set_service_type(readRule.ServiceType_); - } - + } + template <class TRequest, class TSettings> static TRequest MakePropsCreateOrAlterRequest(const TString& path, const TSettings& settings) { TRequest request = MakeOperationRequest<TRequest>(settings); @@ -71,9 +71,9 @@ public: Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps = *props.add_read_rules(); - ConvertToProtoReadRule(readRule, rrProps); + ConvertToProtoReadRule(readRule, rrProps); } - + if (settings.RemoteMirrorRule_) { auto rmr = props.mutable_remote_mirror_rule(); rmr->set_endpoint(settings.RemoteMirrorRule_.GetRef().Endpoint_); @@ -140,30 +140,30 @@ public: settings.ClientTimeout_); } - TAsyncStatus AddReadRule(const TString& path, const TAddReadRuleSettings& settings) { - auto request = MakeOperationRequest<Ydb::PersQueue::V1::AddReadRuleRequest>(settings); - request.set_path(path); - ConvertToProtoReadRule(settings.ReadRule_, *request.mutable_read_rule()); - return RunSimple<Ydb::PersQueue::V1::PersQueueService, Ydb::PersQueue::V1::AddReadRuleRequest, Ydb::PersQueue::V1::AddReadRuleResponse>( - std::move(request), - &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncAddReadRule, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - } - - TAsyncStatus RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& settings) { - auto request = MakeOperationRequest<Ydb::PersQueue::V1::RemoveReadRuleRequest>(settings); - request.set_path(path); - - request.set_consumer_name(settings.ConsumerName_); - return RunSimple<Ydb::PersQueue::V1::PersQueueService, Ydb::PersQueue::V1::RemoveReadRuleRequest, Ydb::PersQueue::V1::RemoveReadRuleResponse>( - std::move(request), - &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncRemoveReadRule, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - } - - + TAsyncStatus AddReadRule(const TString& path, const TAddReadRuleSettings& settings) { + auto request = MakeOperationRequest<Ydb::PersQueue::V1::AddReadRuleRequest>(settings); + request.set_path(path); + ConvertToProtoReadRule(settings.ReadRule_, *request.mutable_read_rule()); + return RunSimple<Ydb::PersQueue::V1::PersQueueService, Ydb::PersQueue::V1::AddReadRuleRequest, Ydb::PersQueue::V1::AddReadRuleResponse>( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncAddReadRule, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + } + + TAsyncStatus RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& settings) { + auto request = MakeOperationRequest<Ydb::PersQueue::V1::RemoveReadRuleRequest>(settings); + request.set_path(path); + + request.set_consumer_name(settings.ConsumerName_); + return RunSimple<Ydb::PersQueue::V1::PersQueueService, Ydb::PersQueue::V1::RemoveReadRuleRequest, Ydb::PersQueue::V1::RemoveReadRuleResponse>( + std::move(request), + &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncRemoveReadRule, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + } + + TAsyncDescribeTopicResult DescribeTopic(const TString& path, const TDescribeTopicSettings& settings) { auto request = MakeOperationRequest<Ydb::PersQueue::V1::DescribeTopicRequest>(settings); request.set_path(path); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp index 57e68484ca..e7dd0a87e5 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp @@ -179,7 +179,7 @@ void TReadSession::ProceedWithoutClusterDiscovery() { clusterSessionInfoIter = ClusterSessions.emplace(normalizedName, normalizedName).first; TClusterSessionInfo& clusterSessionInfo = clusterSessionInfoIter->second; clusterSessionInfo.ClusterEndpoint = DbDriverState->DiscoveryEndpoint; - clusterSessionInfo.Topics = Settings.Topics_; + clusterSessionInfo.Topics = Settings.Topics_; CreateClusterSessionsImpl(); } ScheduleDumpCountersToLog(); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h index 1b2e20a589..ee9aa75a0a 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h @@ -170,34 +170,34 @@ using TAsyncDescribeTopicResult = NThreading::TFuture<TDescribeTopicResult>; const TVector<ECodec>& GetDefaultCodecs(); -struct TReadRuleSettings { - TReadRuleSettings() {} - using TSelf = TReadRuleSettings; - FLUENT_SETTING(TString, ConsumerName); - FLUENT_SETTING_DEFAULT(bool, Important, false); - FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); - FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) - FLUENT_SETTING_DEFAULT(TVector<ECodec>, SupportedCodecs, GetDefaultCodecs()); - - FLUENT_SETTING_DEFAULT(ui32, Version, 0); +struct TReadRuleSettings { + TReadRuleSettings() {} + using TSelf = TReadRuleSettings; + FLUENT_SETTING(TString, ConsumerName); + FLUENT_SETTING_DEFAULT(bool, Important, false); + FLUENT_SETTING_DEFAULT(TInstant, StartingMessageTimestamp, TInstant::Zero()); + FLUENT_SETTING_DEFAULT(EFormat, SupportedFormat, EFormat::BASE) + FLUENT_SETTING_DEFAULT(TVector<ECodec>, SupportedCodecs, GetDefaultCodecs()); + + FLUENT_SETTING_DEFAULT(ui32, Version, 0); FLUENT_SETTING(TString, ServiceType); - - TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) { - ConsumerName_ = settings.ConsumerName(); - Important_ = settings.Important(); - StartingMessageTimestamp_ = settings.StartingMessageTimestamp(); - SupportedFormat_ = settings.SupportedFormat(); - SupportedCodecs_.clear(); - for (const auto& codec : settings.SupportedCodecs()) { - SupportedCodecs_.push_back(codec); - } - Version_ = settings.Version(); + + TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) { + ConsumerName_ = settings.ConsumerName(); + Important_ = settings.Important(); + StartingMessageTimestamp_ = settings.StartingMessageTimestamp(); + SupportedFormat_ = settings.SupportedFormat(); + SupportedCodecs_.clear(); + for (const auto& codec : settings.SupportedCodecs()) { + SupportedCodecs_.push_back(codec); + } + Version_ = settings.Version(); ServiceType_ = settings.ServiceType(); - return *this; - } - -}; - + return *this; + } + +}; + // Settings for topic. template <class TDerived> struct TTopicSettings : public TOperationRequestSettings<TDerived> { @@ -294,17 +294,17 @@ struct TDropTopicSettings : public TOperationRequestSettings<TDropTopicSettings> // Settings for describe resource request. struct TDescribeTopicSettings : public TOperationRequestSettings<TDescribeTopicSettings> {}; -// Settings for add read rule request -struct TAddReadRuleSettings : public TTopicSettings<TAddReadRuleSettings> { - FLUENT_SETTING(TReadRuleSettings, ReadRule); -}; - -// Settings for remove read rule request -struct TRemoveReadRuleSettings : public TOperationRequestSettings<TRemoveReadRuleSettings> { - FLUENT_SETTING(TString, ConsumerName); -}; - - +// Settings for add read rule request +struct TAddReadRuleSettings : public TTopicSettings<TAddReadRuleSettings> { + FLUENT_SETTING(TReadRuleSettings, ReadRule); +}; + +// Settings for remove read rule request +struct TRemoveReadRuleSettings : public TOperationRequestSettings<TRemoveReadRuleSettings> { + FLUENT_SETTING(TString, ConsumerName); +}; + + //! Session metainformation. struct TWriteSessionMeta : public TThrRefBase { using TPtr = TIntrusivePtr<TWriteSessionMeta>; @@ -557,9 +557,9 @@ struct TReadSessionEvent { TMessage(const TString& data, std::exception_ptr decompressionException, const TMessageInformation& information, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash); + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash); //! Commits single message. void Commit() override; @@ -1481,11 +1481,11 @@ public: TAsyncStatus DropTopic(const TString& path, const TDropTopicSettings& = {}); // Add topic read rule - TAsyncStatus AddReadRule(const TString& path, const TAddReadRuleSettings& = {}); - - // Remove topic read rule - TAsyncStatus RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& = {}); - + TAsyncStatus AddReadRule(const TString& path, const TAddReadRuleSettings& = {}); + + // Remove topic read rule + TAsyncStatus RemoveReadRule(const TString& path, const TRemoveReadRuleSettings& = {}); + // Describe settings of topic. TAsyncDescribeTopicResult DescribeTopic(const TString& path, const TDescribeTopicSettings& = {}); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp index a2bb14819c..3cf6e7f8b8 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp @@ -1,19 +1,19 @@ #include "data_plane_helpers.h" - -namespace NKikimr::NPersQueueTests { - + +namespace NKikimr::NPersQueueTests { + using namespace NYdb::NPersQueue; - + std::shared_ptr<NYdb::NPersQueue::IWriteSession> CreateWriter( NYdb::TDriver& driver, const NYdb::NPersQueue::TWriteSessionSettings& settings, std::shared_ptr<NYdb::ICredentialsProviderFactory> creds - ) { + ) { TPersQueueClientSettings clientSettings; if (creds) clientSettings.CredentialsProviderFactory(creds); return TPersQueueClient(driver, clientSettings).CreateWriteSession(TWriteSessionSettings(settings).ClusterDiscoveryMode(EClusterDiscoveryMode::Off)); - } - + } + std::shared_ptr<NYdb::NPersQueue::IWriteSession> CreateWriter( NYdb::TDriver& driver, const TString& topic, @@ -22,7 +22,7 @@ namespace NKikimr::NPersQueueTests { std::optional<TString> codec, std::optional<bool> reconnectOnFailure, std::shared_ptr<NYdb::ICredentialsProviderFactory> creds - ) { + ) { auto settings = TWriteSessionSettings().Path(topic).MessageGroupId(sourceId); if (partitionGroup) settings.PartitionGroupId(*partitionGroup); settings.RetryPolicy((reconnectOnFailure && *reconnectOnFailure) ? IRetryPolicy::GetDefaultPolicy() : IRetryPolicy::GetNoRetryPolicy()); @@ -35,15 +35,15 @@ namespace NKikimr::NPersQueueTests { settings.Codec(ECodec::LZOP); } return CreateWriter(driver, settings, creds); - } - + } + std::shared_ptr<NYdb::NPersQueue::ISimpleBlockingWriteSession> CreateSimpleWriter( NYdb::TDriver& driver, const NYdb::NPersQueue::TWriteSessionSettings& settings - ) { + ) { return TPersQueueClient(driver).CreateSimpleBlockingWriteSession(TWriteSessionSettings(settings).ClusterDiscoveryMode(EClusterDiscoveryMode::Off)); - } - + } + std::shared_ptr<NYdb::NPersQueue::ISimpleBlockingWriteSession> CreateSimpleWriter( NYdb::TDriver& driver, const TString& topic, @@ -51,14 +51,14 @@ namespace NKikimr::NPersQueueTests { std::optional<ui32> partitionGroup, std::optional<TString> codec, std::optional<bool> reconnectOnFailure - ) { + ) { Y_UNUSED(codec); auto settings = TWriteSessionSettings().Path(topic).MessageGroupId(sourceId); if (partitionGroup) settings.PartitionGroupId(*partitionGroup); settings.RetryPolicy((reconnectOnFailure && *reconnectOnFailure) ? IRetryPolicy::GetDefaultPolicy() : IRetryPolicy::GetNoRetryPolicy()); return CreateSimpleWriter(driver, settings); - } - + } + std::shared_ptr<NYdb::NPersQueue::IReadSession> CreateReader( NYdb::TDriver& driver, const NYdb::NPersQueue::TReadSessionSettings& settings, @@ -70,7 +70,7 @@ namespace NKikimr::NPersQueueTests { } TMaybe<TReadSessionEvent::TDataReceivedEvent> GetNextMessageSkipAssignment(std::shared_ptr<IReadSession>& reader, TDuration timeout) { - while (true) { + while (true) { auto future = reader->WaitEvent(); future.Wait(timeout); @@ -85,10 +85,10 @@ namespace NKikimr::NPersQueueTests { destroyPartitionStreamEvent->Confirm(); } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&*event)) { return {}; - } - } + } + } return {}; - } - + } + } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h index cea9193c34..8868ce64b2 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> - -namespace NKikimr::NPersQueueTests { - + +namespace NKikimr::NPersQueueTests { + std::shared_ptr<NYdb::NPersQueue::IWriteSession> CreateWriter( NYdb::TDriver& driver, const NYdb::NPersQueue::TWriteSessionSettings& settings, std::shared_ptr<NYdb::ICredentialsProviderFactory> creds = nullptr - ); - + ); + std::shared_ptr<NYdb::NPersQueue::IWriteSession> CreateWriter( NYdb::TDriver& driver, const TString& topic, @@ -20,13 +20,13 @@ namespace NKikimr::NPersQueueTests { std::optional<TString> codec = {}, std::optional<bool> reconnectOnFailure = {}, std::shared_ptr<NYdb::ICredentialsProviderFactory> creds = nullptr - ); - + ); + std::shared_ptr<NYdb::NPersQueue::ISimpleBlockingWriteSession> CreateSimpleWriter( NYdb::TDriver& driver, const NYdb::NPersQueue::TWriteSessionSettings& settings - ); - + ); + std::shared_ptr<NYdb::NPersQueue::ISimpleBlockingWriteSession> CreateSimpleWriter( NYdb::TDriver& driver, const TString& topic, @@ -34,15 +34,15 @@ namespace NKikimr::NPersQueueTests { std::optional<ui32> partitionGroup = {}, std::optional<TString> codec = {}, std::optional<bool> reconnectOnFailure = {} - ); - + ); + std::shared_ptr<NYdb::NPersQueue::IReadSession> CreateReader( NYdb::TDriver& driver, const NYdb::NPersQueue::TReadSessionSettings& settings, std::shared_ptr<NYdb::ICredentialsProviderFactory> creds = nullptr ); - + TMaybe<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent> GetNextMessageSkipAssignment(std::shared_ptr<NYdb::NPersQueue::IReadSession>& reader, TDuration timeout = TDuration::Max()); } diff --git a/ydb/public/tools/lib/cmds/__init__.py b/ydb/public/tools/lib/cmds/__init__.py index a3532d5533..977f0a4aae 100644 --- a/ydb/public/tools/lib/cmds/__init__.py +++ b/ydb/public/tools/lib/cmds/__init__.py @@ -273,7 +273,7 @@ def deploy(arguments): optionals.update({'grpc_tls_data_path': grpc_tls_data_path(arguments)}) optionals.update({'grpc_ssl_enable': enable_tls()}) pdisk_store_path = arguments.ydb_working_dir if arguments.ydb_working_dir else None - + configuration = KikimrConfigGenerator( parse_erasure(arguments), arguments.ydb_binary_path, diff --git a/ydb/services/datastreams/datastreams_proxy.cpp b/ydb/services/datastreams/datastreams_proxy.cpp index db94af7ab0..03a6d06213 100644 --- a/ydb/services/datastreams/datastreams_proxy.cpp +++ b/ydb/services/datastreams/datastreams_proxy.cpp @@ -1,8 +1,8 @@ -#include "datastreams_proxy.h" -#include "put_records_actor.h" +#include "datastreams_proxy.h" +#include "put_records_actor.h" #include "shard_iterator.h" #include "next_token.h" - + #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_deferrable.h> #include <ydb/core/grpc_services/rpc_scheme_base.h> @@ -12,227 +12,227 @@ #include <ydb/services/lib/actors/pq_schema_actor.h> #include <ydb/services/lib/sharding/sharding.h> -#include <util/folder/path.h> - +#include <util/folder/path.h> + #include <iterator> -using namespace NActors; -using namespace NKikimrClient; - -using grpc::Status; - -namespace NKikimr::NDataStreams::V1 { +using namespace NActors; +using namespace NKikimrClient; + +using grpc::Status; + +namespace NKikimr::NDataStreams::V1 { const TString YDS_SERVICE_TYPE = "data-streams"; - - using namespace NGRpcService; - using namespace NGRpcProxy::V1; - - namespace { - - ui32 PartitionWriteSpeedInBytesPerSec(ui32 speedInKbPerSec) { - return speedInKbPerSec == 0 ? 1024 * 1024 : speedInKbPerSec * 1024; - } - - TDuration RetentionPeriod(ui32 retentionPeriodInHours) { - return retentionPeriodInHours == 0 ? TDuration::Hours(24) : TDuration::Hours(retentionPeriodInHours); - } - - template<class TRequest> + + using namespace NGRpcService; + using namespace NGRpcProxy::V1; + + namespace { + + ui32 PartitionWriteSpeedInBytesPerSec(ui32 speedInKbPerSec) { + return speedInKbPerSec == 0 ? 1024 * 1024 : speedInKbPerSec * 1024; + } + + TDuration RetentionPeriod(ui32 retentionPeriodInHours) { + return retentionPeriodInHours == 0 ? TDuration::Hours(24) : TDuration::Hours(retentionPeriodInHours); + } + + template<class TRequest> bool ValidateRetentionPeriod(const TRequest& req, const NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, - TMaybe<bool> increase, TString& errorText) - { - if (req.retention_period_hours() > 24) { - errorText = TStringBuilder() << "Retention period is greater than 24 hours: " - << req.retention_period_hours(); - return false; - } - - if (increase.Defined()) { - auto currentLifetime = TDuration::Seconds(groupConfig.GetPQTabletConfig().GetPartitionConfig().GetLifetimeSeconds()); - auto newLifetime = TDuration::Hours(req.retention_period_hours()); - if (*increase) { - if (newLifetime <= currentLifetime) { - errorText = TStringBuilder() << "Retention period is not greater than provided: " - << currentLifetime.Hours() << " <= " << newLifetime.Hours(); - return false; - } - } else { - if (newLifetime >= currentLifetime) { - errorText = TStringBuilder() << "Retention period is not less than provided: " - << currentLifetime.Hours() << " <= " << newLifetime.Hours(); - return false; - } - } - } - - return true; - } - - template<class TRequest> + TMaybe<bool> increase, TString& errorText) + { + if (req.retention_period_hours() > 24) { + errorText = TStringBuilder() << "Retention period is greater than 24 hours: " + << req.retention_period_hours(); + return false; + } + + if (increase.Defined()) { + auto currentLifetime = TDuration::Seconds(groupConfig.GetPQTabletConfig().GetPartitionConfig().GetLifetimeSeconds()); + auto newLifetime = TDuration::Hours(req.retention_period_hours()); + if (*increase) { + if (newLifetime <= currentLifetime) { + errorText = TStringBuilder() << "Retention period is not greater than provided: " + << currentLifetime.Hours() << " <= " << newLifetime.Hours(); + return false; + } + } else { + if (newLifetime >= currentLifetime) { + errorText = TStringBuilder() << "Retention period is not less than provided: " + << currentLifetime.Hours() << " <= " << newLifetime.Hours(); + return false; + } + } + } + + return true; + } + + template<class TRequest> bool ValidateShardsCount(const TRequest& req, const NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, TString& errorText) { - if (req.target_shard_count() < (i32)groupConfig.GetTotalGroupCount()) { - errorText = TStringBuilder() << "Shard count must be non-decreasing, old value is " - << groupConfig.GetTotalGroupCount() - << " new value is " << req.target_shard_count(); - return false; - } - return true; - } - - template<class TRequest> - bool ValidateWriteSpeedLimit(const TRequest& req, TString& errorText, const TActorContext& ctx) { - THashSet<ui32> validLimits {0}; - if (AppData(ctx)->PQConfig.ValidWriteSpeedLimitsKbPerSecSize() == 0) { - validLimits.insert(128); - validLimits.insert(1024); - } else { - const auto& limits = AppData(ctx)->PQConfig.GetValidWriteSpeedLimitsKbPerSec(); - validLimits.insert(limits.begin(), limits.end()); - } - if (validLimits.find(req.write_quota_kb_per_sec()) == validLimits.end()) { - errorText = TStringBuilder() << "write_quota_kb_per_sec must have values from set {" << JoinSeq(",", validLimits) << ", got " << req.write_quota_kb_per_sec(); - return false; - } - - return true; - } - - } - - class TCreateStreamActor : public TPQGrpcSchemaBase<TCreateStreamActor, NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest> { - using TBase = TPQGrpcSchemaBase<TCreateStreamActor, TEvDataStreamsCreateStreamRequest>; - + if (req.target_shard_count() < (i32)groupConfig.GetTotalGroupCount()) { + errorText = TStringBuilder() << "Shard count must be non-decreasing, old value is " + << groupConfig.GetTotalGroupCount() + << " new value is " << req.target_shard_count(); + return false; + } + return true; + } + + template<class TRequest> + bool ValidateWriteSpeedLimit(const TRequest& req, TString& errorText, const TActorContext& ctx) { + THashSet<ui32> validLimits {0}; + if (AppData(ctx)->PQConfig.ValidWriteSpeedLimitsKbPerSecSize() == 0) { + validLimits.insert(128); + validLimits.insert(1024); + } else { + const auto& limits = AppData(ctx)->PQConfig.GetValidWriteSpeedLimitsKbPerSec(); + validLimits.insert(limits.begin(), limits.end()); + } + if (validLimits.find(req.write_quota_kb_per_sec()) == validLimits.end()) { + errorText = TStringBuilder() << "write_quota_kb_per_sec must have values from set {" << JoinSeq(",", validLimits) << ", got " << req.write_quota_kb_per_sec(); + return false; + } + + return true; + } + + } + + class TCreateStreamActor : public TPQGrpcSchemaBase<TCreateStreamActor, NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest> { + using TBase = TPQGrpcSchemaBase<TCreateStreamActor, TEvDataStreamsCreateStreamRequest>; + TActorId NewSchemeCache; - public: + public: TCreateStreamActor(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest* request, TActorId newSchemeCache); - ~TCreateStreamActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name); - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + ~TCreateStreamActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name); + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx); - }; - + void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx); + }; + TCreateStreamActor::TCreateStreamActor(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest* request, TActorId newSchemeCache) - : TBase(request, request->GetProtoRequest()->stream_name()) + : TBase(request, request->GetProtoRequest()->stream_name()) , NewSchemeCache(newSchemeCache) - { + { Y_UNUSED(NewSchemeCache); - } - - void TCreateStreamActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); + } + + void TCreateStreamActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); SendProposeRequest(ctx); - Become(&TCreateStreamActor::StateWork); - } - + Become(&TCreateStreamActor::StateWork); + } + void TCreateStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); Y_UNUSED(ctx); } - void TCreateStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, - const TActorContext& ctx, const TString& workingDir, const TString& name) - { + void TCreateStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, + const TActorContext& ctx, const TString& workingDir, const TString& name) + { NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); - - Ydb::PersQueue::V1::TopicSettings topicSettings; - topicSettings.set_partitions_count(GetProtoRequest()->shard_count()); - topicSettings.set_retention_period_ms(RetentionPeriod(GetProtoRequest()->retention_period_hours()).MilliSeconds()); - topicSettings.set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + + Ydb::PersQueue::V1::TopicSettings topicSettings; + topicSettings.set_partitions_count(GetProtoRequest()->shard_count()); + topicSettings.set_retention_period_ms(RetentionPeriod(GetProtoRequest()->retention_period_hours()).MilliSeconds()); + topicSettings.set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); topicSettings.add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); - topicSettings.set_max_partition_write_speed(PartitionWriteSpeedInBytesPerSec(GetProtoRequest()->write_quota_kb_per_sec())); + topicSettings.set_max_partition_write_speed(PartitionWriteSpeedInBytesPerSec(GetProtoRequest()->write_quota_kb_per_sec())); topicSettings.set_max_partition_write_burst(PartitionWriteSpeedInBytesPerSec(GetProtoRequest()->write_quota_kb_per_sec())); - + if (workingDir != proposal.Record.GetDatabaseName() && !proposal.Record.GetDatabaseName().empty()) { return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, "streams can be created only at database root", ctx); } - TString error; - auto status = NKikimr::NGRpcProxy::V1::FillProposeRequestImpl(name, topicSettings, modifyScheme, ctx, false, error); - modifyScheme.SetWorkingDir(workingDir); - - if (!error.Empty()) { - return ReplyWithError(status, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - auto pqDescr = modifyScheme.MutableCreatePersQueueGroup(); - pqDescr->SetPartitionPerTablet(1); - - if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx) - || !ValidateRetentionPeriod(*GetProtoRequest(), *pqDescr, Nothing(), error)) - { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - } - - void TCreateStreamActor::Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { - auto msg = ev->Get(); - const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(ev->Get()->Record.GetStatus()); - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete + TString error; + auto status = NKikimr::NGRpcProxy::V1::FillProposeRequestImpl(name, topicSettings, modifyScheme, ctx, false, error); + modifyScheme.SetWorkingDir(workingDir); + + if (!error.Empty()) { + return ReplyWithError(status, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + auto pqDescr = modifyScheme.MutableCreatePersQueueGroup(); + pqDescr->SetPartitionPerTablet(1); + + if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx) + || !ValidateRetentionPeriod(*GetProtoRequest(), *pqDescr, Nothing(), error)) + { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + } + + void TCreateStreamActor::Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx) { + auto msg = ev->Get(); + const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(ev->Get()->Record.GetStatus()); + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete && msg->Record.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusAlreadyExists) - { - return ReplyWithError(Ydb::StatusIds::ALREADY_EXISTS, - Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Stream with name " << GetProtoRequest()->stream_name() << " is already exists", - ctx); - } + { + return ReplyWithError(Ydb::StatusIds::ALREADY_EXISTS, + Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Stream with name " << GetProtoRequest()->stream_name() << " is already exists", + ctx); + } return TBase::TBase::Handle(ev, ctx); - } - - void TCreateStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, Handle); - default: TBase::StateWork(ev, ctx); - } - } - - //----------------------------------------------------------------------------------- - - class TDeleteStreamActor : public TPQGrpcSchemaBase<TDeleteStreamActor, NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest> { - using TBase = TPQGrpcSchemaBase<TDeleteStreamActor, TEvDataStreamsDeleteStreamRequest>; - - public: - TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request); - ~TDeleteStreamActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name); + } + + void TCreateStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, Handle); + default: TBase::StateWork(ev, ctx); + } + } + + //----------------------------------------------------------------------------------- + + class TDeleteStreamActor : public TPQGrpcSchemaBase<TDeleteStreamActor, NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest> { + using TBase = TPQGrpcSchemaBase<TDeleteStreamActor, TEvDataStreamsDeleteStreamRequest>; + + public: + TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request); + ~TDeleteStreamActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name); void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); private: bool EnforceDeletion; - }; - - TDeleteStreamActor::TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request) - : TBase(request, request->GetProtoRequest()->stream_name()) + }; + + TDeleteStreamActor::TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request) + : TBase(request, request->GetProtoRequest()->stream_name()) , EnforceDeletion{request->GetProtoRequest()->enforce_consumer_deletion()} - { - } - - void TDeleteStreamActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); + { + } + + void TDeleteStreamActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); SendDescribeProposeRequest(ctx); - Become(&TDeleteStreamActor::StateWork); - } - + Become(&TDeleteStreamActor::StateWork); + } + void TDeleteStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, const TString& workingDir, const TString& name) - { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "WorkingDir = " << workingDir << ", name = " << name); + { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "WorkingDir = " << workingDir << ", name = " << name); NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); - modifyScheme.SetWorkingDir(workingDir); + modifyScheme.SetWorkingDir(workingDir); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropPersQueueGroup); - modifyScheme.MutableDrop()->SetName(name); - } - + modifyScheme.MutableDrop()->SetName(name); + } + void TDeleteStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { if (ReplyIfNotTopic(ev, ctx)) { @@ -251,30 +251,30 @@ namespace NKikimr::NDataStreams::V1 { SendProposeRequest(ctx); } - //----------------------------------------------------------------------------------------------------------- - - class TUpdateShardCountActor : public TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest> { - using TBase = TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest>; - - public: - TUpdateShardCountActor(TEvDataStreamsUpdateShardCountRequest* request) - : TBase(request, request->GetProtoRequest()->stream_name()) - { - } - - void Bootstrap(const TActorContext& ctx); + //----------------------------------------------------------------------------------------------------------- + + class TUpdateShardCountActor : public TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest> { + using TBase = TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest>; + + public: + TUpdateShardCountActor(TEvDataStreamsUpdateShardCountRequest* request) + : TBase(request, request->GetProtoRequest()->stream_name()) + { + } + + void Bootstrap(const TActorContext& ctx); void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); - }; - - void TUpdateShardCountActor::Bootstrap(const TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TBase::StateWork); - } - + }; + + void TUpdateShardCountActor::Bootstrap(const TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TBase::StateWork); + } + void TUpdateShardCountActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -283,39 +283,39 @@ namespace NKikimr::NDataStreams::V1 { ) { Y_UNUSED(pqGroupDescription); Y_UNUSED(selfInfo); - - TString error; - if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error)) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - - groupConfig.SetTotalGroupCount(GetProtoRequest()->target_shard_count()); - } - - //----------------------------------------------------------------------------------------------------------- - - class TUpdateStreamActor : public TUpdateSchemeActor<TUpdateStreamActor, TEvDataStreamsUpdateStreamRequest> { - using TBase = TUpdateSchemeActor<TUpdateStreamActor, TEvDataStreamsUpdateStreamRequest>; - - public: - TUpdateStreamActor(TEvDataStreamsUpdateStreamRequest* request) - : TBase(request, request->GetProtoRequest()->stream_name()) - { - } - - void Bootstrap(const TActorContext& ctx); + + TString error; + if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error)) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + + groupConfig.SetTotalGroupCount(GetProtoRequest()->target_shard_count()); + } + + //----------------------------------------------------------------------------------------------------------- + + class TUpdateStreamActor : public TUpdateSchemeActor<TUpdateStreamActor, TEvDataStreamsUpdateStreamRequest> { + using TBase = TUpdateSchemeActor<TUpdateStreamActor, TEvDataStreamsUpdateStreamRequest>; + + public: + TUpdateStreamActor(TEvDataStreamsUpdateStreamRequest* request) + : TBase(request, request->GetProtoRequest()->stream_name()) + { + } + + void Bootstrap(const TActorContext& ctx); void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); - }; - - void TUpdateStreamActor::Bootstrap(const TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TBase::StateWork); - } - + }; + + void TUpdateStreamActor::Bootstrap(const TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TBase::StateWork); + } + void TUpdateStreamActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -324,45 +324,45 @@ namespace NKikimr::NDataStreams::V1 { ) { Y_UNUSED(pqGroupDescription); Y_UNUSED(selfInfo); - - TString error; - if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error) - || !ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx) - || !ValidateRetentionPeriod(*GetProtoRequest(), groupConfig, Nothing(), error)) - { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - - groupConfig.SetTotalGroupCount(GetProtoRequest()->target_shard_count()); - groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetWriteSpeedInBytesPerSecond( - PartitionWriteSpeedInBytesPerSec(GetProtoRequest()->write_quota_kb_per_sec())); - groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetLifetimeSeconds(TInstant::Hours(this->GetProtoRequest()->retention_period_hours()).Seconds()); - } - - //----------------------------------------------------------------------------------------------------------- - - class TSetWriteQuotaActor : public TUpdateSchemeActor<TSetWriteQuotaActor, TEvDataStreamsSetWriteQuotaRequest> { - using TBase = TUpdateSchemeActor<TSetWriteQuotaActor, TEvDataStreamsSetWriteQuotaRequest>; - - public: - TSetWriteQuotaActor(TEvDataStreamsSetWriteQuotaRequest* request) - : TBase(request, request->GetProtoRequest()->stream_name()) - { - } - - void Bootstrap(const TActorContext& ctx); + + TString error; + if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error) + || !ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx) + || !ValidateRetentionPeriod(*GetProtoRequest(), groupConfig, Nothing(), error)) + { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + + groupConfig.SetTotalGroupCount(GetProtoRequest()->target_shard_count()); + groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetWriteSpeedInBytesPerSecond( + PartitionWriteSpeedInBytesPerSec(GetProtoRequest()->write_quota_kb_per_sec())); + groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetLifetimeSeconds(TInstant::Hours(this->GetProtoRequest()->retention_period_hours()).Seconds()); + } + + //----------------------------------------------------------------------------------------------------------- + + class TSetWriteQuotaActor : public TUpdateSchemeActor<TSetWriteQuotaActor, TEvDataStreamsSetWriteQuotaRequest> { + using TBase = TUpdateSchemeActor<TSetWriteQuotaActor, TEvDataStreamsSetWriteQuotaRequest>; + + public: + TSetWriteQuotaActor(TEvDataStreamsSetWriteQuotaRequest* request) + : TBase(request, request->GetProtoRequest()->stream_name()) + { + } + + void Bootstrap(const TActorContext& ctx); void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); - }; - - void TSetWriteQuotaActor::Bootstrap(const TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TBase::StateWork); - } - + }; + + void TSetWriteQuotaActor::Bootstrap(const TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TBase::StateWork); + } + void TSetWriteQuotaActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -371,34 +371,34 @@ namespace NKikimr::NDataStreams::V1 { ) { Y_UNUSED(pqGroupDescription); Y_UNUSED(selfInfo); - - TString error; - if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx)) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetWriteSpeedInBytesPerSecond(GetProtoRequest()->write_quota_kb_per_sec() * 1024LL); + + TString error; + if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx)) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetWriteSpeedInBytesPerSecond(GetProtoRequest()->write_quota_kb_per_sec() * 1024LL); groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetBurstSize(GetProtoRequest()->write_quota_kb_per_sec() * 1024LL); - } - - //----------------------------------------------------------------------------------------------------------- - - template<class TEvProto> - class TSetStreamRetentionPeriodActor : public TUpdateSchemeActor<TSetStreamRetentionPeriodActor<TEvProto>, TEvProto> { - using TBase = TUpdateSchemeActor<TSetStreamRetentionPeriodActor<TEvProto>, TEvProto>; - - public: - TSetStreamRetentionPeriodActor(TEvProto* request, bool shouldIncrease) - : TBase(request, request->GetProtoRequest()->stream_name()) - , ShouldIncrease(shouldIncrease) - { - } - - void Bootstrap(const TActorContext& ctx) { - TBase::Bootstrap(ctx); - TBase::SendDescribeProposeRequest(ctx); - TBase::Become(&TBase::StateWork); - } - + } + + //----------------------------------------------------------------------------------------------------------- + + template<class TEvProto> + class TSetStreamRetentionPeriodActor : public TUpdateSchemeActor<TSetStreamRetentionPeriodActor<TEvProto>, TEvProto> { + using TBase = TUpdateSchemeActor<TSetStreamRetentionPeriodActor<TEvProto>, TEvProto>; + + public: + TSetStreamRetentionPeriodActor(TEvProto* request, bool shouldIncrease) + : TBase(request, request->GetProtoRequest()->stream_name()) + , ShouldIncrease(shouldIncrease) + { + } + + void Bootstrap(const TActorContext& ctx) { + TBase::Bootstrap(ctx); + TBase::SendDescribeProposeRequest(ctx); + TBase::Become(&TBase::StateWork); + } + void ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -407,30 +407,30 @@ namespace NKikimr::NDataStreams::V1 { ) { Y_UNUSED(pqGroupDescription); Y_UNUSED(selfInfo); - - TString error; - if (!ValidateRetentionPeriod(*this->GetProtoRequest(), groupConfig, ShouldIncrease, error)) { - return this->ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetLifetimeSeconds(TInstant::Hours(this->GetProtoRequest()->retention_period_hours()).Seconds()); - } - - private: - bool ShouldIncrease; - }; - - //----------------------------------------------------------------------------------- - - class TDescribeStreamActor : public TPQGrpcSchemaBase<TDescribeStreamActor, TEvDataStreamsDescribeStreamRequest> { - using TBase = TPQGrpcSchemaBase<TDescribeStreamActor, TEvDataStreamsDescribeStreamRequest>; - - public: - TDescribeStreamActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest* request); - ~TDescribeStreamActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + + TString error; + if (!ValidateRetentionPeriod(*this->GetProtoRequest(), groupConfig, ShouldIncrease, error)) { + return this->ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + groupConfig.MutablePQTabletConfig()->MutablePartitionConfig()->SetLifetimeSeconds(TInstant::Hours(this->GetProtoRequest()->retention_period_hours()).Seconds()); + } + + private: + bool ShouldIncrease; + }; + + //----------------------------------------------------------------------------------- + + class TDescribeStreamActor : public TPQGrpcSchemaBase<TDescribeStreamActor, TEvDataStreamsDescribeStreamRequest> { + using TBase = TPQGrpcSchemaBase<TDescribeStreamActor, TEvDataStreamsDescribeStreamRequest>; + + public: + TDescribeStreamActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest* request); + ~TDescribeStreamActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { @@ -470,28 +470,28 @@ namespace NKikimr::NDataStreams::V1 { std::vector<TActorId> Pipes; ui32 RequestsInfly = 0; std::map<ui64, std::pair<ui64, ui64>> StartEndOffsetsPerPartition; - }; - - TDescribeStreamActor::TDescribeStreamActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest* request) - : TBase(request, request->GetProtoRequest()->stream_name()) - { - } - - void TDescribeStreamActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TDescribeStreamActor::StateWork); - } - - void TDescribeStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { + }; + + TDescribeStreamActor::TDescribeStreamActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest* request) + : TBase(request, request->GetProtoRequest()->stream_name()) + { + } + + void TDescribeStreamActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TDescribeStreamActor::StateWork); + } + + void TDescribeStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { HFunc(TEvPersQueue::TEvOffsetsResponse, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); - default: TBase::StateWork(ev, ctx); - } - } - + default: TBase::StateWork(ev, ctx); + } + } + void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic @@ -503,7 +503,7 @@ namespace NKikimr::NDataStreams::V1 { } Y_VERIFY(response.PQGroupInfo); - + PQGroup = response.PQGroupInfo->Description; SelfInfo = response.Self->Info; std::set<ui64> tabletIds; @@ -513,9 +513,9 @@ namespace NKikimr::NDataStreams::V1 { if (tabletIds.size() == 0) { ReplyAndDie(ctx); } - + RequestsInfly = tabletIds.size(); - + NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = { .RetryLimitCount = 6, @@ -529,9 +529,9 @@ namespace NKikimr::NDataStreams::V1 { Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets); NTabletPipe::SendData(ctx, Pipes.back(), req.Release()); - } - } - + } + } + void TDescribeStreamActor::ReplyAndDie(const TActorContext& ctx) { Ydb::DataStreams::V1::DescribeStreamResult result; @@ -587,25 +587,25 @@ namespace NKikimr::NDataStreams::V1 { } - //----------------------------------------------------------------------------------- - - class TListStreamsActor : public TRpcSchemeRequestActor<TListStreamsActor, NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest> { - using TBase = TRpcSchemeRequestActor<TListStreamsActor, TEvDataStreamsListStreamsRequest>; - - public: - TListStreamsActor(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest* request, TActorId newSchemeCache); - ~TListStreamsActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - - protected: - void SendNavigateRequest(const TActorContext& ctx, const TString& path); - void SendPendingRequests(const TActorContext& ctx); - void SendResponse(const TActorContext& ctx); - + //----------------------------------------------------------------------------------- + + class TListStreamsActor : public TRpcSchemeRequestActor<TListStreamsActor, NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest> { + using TBase = TRpcSchemeRequestActor<TListStreamsActor, TEvDataStreamsListStreamsRequest>; + + public: + TListStreamsActor(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest* request, TActorId newSchemeCache); + ~TListStreamsActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + + protected: + void SendNavigateRequest(const TActorContext& ctx, const TString& path); + void SendPendingRequests(const TActorContext& ctx); + void SendResponse(const TActorContext& ctx); + void ReplyWithError(Ydb::StatusIds::StatusCode status, Ydb::PersQueue::ErrorCode::ErrorCode pqStatus, const TString& messageText, const NActors::TActorContext& ctx) { this->Request_->RaiseIssue(FillIssue(messageText, pqStatus)); @@ -613,27 +613,27 @@ namespace NKikimr::NDataStreams::V1 { this->Die(ctx); } - private: - static constexpr ui32 MAX_IN_FLIGHT = 5; - - ui32 RequestsInFlight = 0; - std::vector<std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet>> WaitingList; - std::vector<TString> Topics; - TActorId NewSchemeCache; - }; - - TListStreamsActor::TListStreamsActor(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest* request, TActorId newSchemeCache) - : TBase(request) - , NewSchemeCache(newSchemeCache) - { - } - - void TListStreamsActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - if (!Request_->GetDatabaseName()) { + private: + static constexpr ui32 MAX_IN_FLIGHT = 5; + + ui32 RequestsInFlight = 0; + std::vector<std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet>> WaitingList; + std::vector<TString> Topics; + TActorId NewSchemeCache; + }; + + TListStreamsActor::TListStreamsActor(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest* request, TActorId newSchemeCache) + : TBase(request) + , NewSchemeCache(newSchemeCache) + { + } + + void TListStreamsActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + if (!Request_->GetDatabaseName()) { return ReplyWithError(Ydb::StatusIds::BAD_REQUEST,Ydb::PersQueue::ErrorCode::BAD_REQUEST, "Request without dabase is forbiden", ctx); - } + } if (this->Request_->GetInternalToken().empty()) { if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { @@ -642,108 +642,108 @@ namespace NKikimr::NDataStreams::V1 { } } - SendNavigateRequest(ctx, *Request_->GetDatabaseName()); - Become(&TListStreamsActor::StateWork); - } - - void TListStreamsActor::SendPendingRequests(const TActorContext& ctx) { - if (RequestsInFlight < MAX_IN_FLIGHT && WaitingList.size() > 0) { - ctx.Send(NewSchemeCache, WaitingList.back().release()); - WaitingList.pop_back(); - RequestsInFlight++; - } - } - - void TListStreamsActor::SendResponse(const TActorContext& ctx) { - Y_ENSURE(WaitingList.size() == 0 && RequestsInFlight == 0); - - for (TString& topic : Topics) { - topic = TFsPath(topic).RelativePath(*Request_->GetDatabaseName()).GetPath(); - } - Sort(Topics.begin(), Topics.end()); - Ydb::DataStreams::V1::ListStreamsResult result; - - int limit = GetProtoRequest()->limit() == 0 ? 100 : GetProtoRequest()->limit(); - - if (limit > 10000) { - Request_->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); - return Die(ctx); - } - - result.set_has_more_streams(false); - for (const auto& streamName : Topics) { - if (GetProtoRequest()->exclusive_start_stream_name().empty() - || GetProtoRequest()->exclusive_start_stream_name() < streamName) - { - if (result.stream_names().size() >= limit) { - result.set_has_more_streams(true); - break; - } - result.add_stream_names(streamName); - } - } - - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - void TListStreamsActor::SendNavigateRequest(const TActorContext& ctx, const TString &path) { - auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); - schemeCacheRequest->DatabaseName = Request().GetDatabaseName().GetRef(); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = NKikimr::SplitPath(path); - + SendNavigateRequest(ctx, *Request_->GetDatabaseName()); + Become(&TListStreamsActor::StateWork); + } + + void TListStreamsActor::SendPendingRequests(const TActorContext& ctx) { + if (RequestsInFlight < MAX_IN_FLIGHT && WaitingList.size() > 0) { + ctx.Send(NewSchemeCache, WaitingList.back().release()); + WaitingList.pop_back(); + RequestsInFlight++; + } + } + + void TListStreamsActor::SendResponse(const TActorContext& ctx) { + Y_ENSURE(WaitingList.size() == 0 && RequestsInFlight == 0); + + for (TString& topic : Topics) { + topic = TFsPath(topic).RelativePath(*Request_->GetDatabaseName()).GetPath(); + } + Sort(Topics.begin(), Topics.end()); + Ydb::DataStreams::V1::ListStreamsResult result; + + int limit = GetProtoRequest()->limit() == 0 ? 100 : GetProtoRequest()->limit(); + + if (limit > 10000) { + Request_->ReplyWithYdbStatus(Ydb::StatusIds::BAD_REQUEST); + return Die(ctx); + } + + result.set_has_more_streams(false); + for (const auto& streamName : Topics) { + if (GetProtoRequest()->exclusive_start_stream_name().empty() + || GetProtoRequest()->exclusive_start_stream_name() < streamName) + { + if (result.stream_names().size() >= limit) { + result.set_has_more_streams(true); + break; + } + result.add_stream_names(streamName); + } + } + + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + void TListStreamsActor::SendNavigateRequest(const TActorContext& ctx, const TString &path) { + auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); + schemeCacheRequest->DatabaseName = Request().GetDatabaseName().GetRef(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = NKikimr::SplitPath(path); + if (!this->Request_->GetInternalToken().empty()) { schemeCacheRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken()); } - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; - schemeCacheRequest->ResultSet.emplace_back(entry); - WaitingList.push_back(std::make_unique<TEvTxProxySchemeCache::TEvNavigateKeySet>(schemeCacheRequest.release())); - SendPendingRequests(ctx); - } - - void TListStreamsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - default: TBase::StateWork(ev, ctx); - } - } - - void TListStreamsActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); - for (const auto& entry : navigate->ResultSet) { - if (entry.Kind == NSchemeCache::TSchemeCacheNavigate::EKind::KindPath - || entry.Kind == NSchemeCache::TSchemeCacheNavigate::EKind::KindSubdomain) - { - Y_ENSURE(entry.ListNodeEntry, "ListNodeEntry is zero"); - for (const auto& child : entry.ListNodeEntry->Children) { - TString childFullPath = JoinPath({JoinPath(entry.Path), child.Name}); - switch (child.Kind) { - case NSchemeCache::TSchemeCacheNavigate::EKind::KindPath: + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; + schemeCacheRequest->ResultSet.emplace_back(entry); + WaitingList.push_back(std::make_unique<TEvTxProxySchemeCache::TEvNavigateKeySet>(schemeCacheRequest.release())); + SendPendingRequests(ctx); + } + + void TListStreamsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + default: TBase::StateWork(ev, ctx); + } + } + + void TListStreamsActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); + for (const auto& entry : navigate->ResultSet) { + if (entry.Kind == NSchemeCache::TSchemeCacheNavigate::EKind::KindPath + || entry.Kind == NSchemeCache::TSchemeCacheNavigate::EKind::KindSubdomain) + { + Y_ENSURE(entry.ListNodeEntry, "ListNodeEntry is zero"); + for (const auto& child : entry.ListNodeEntry->Children) { + TString childFullPath = JoinPath({JoinPath(entry.Path), child.Name}); + switch (child.Kind) { + case NSchemeCache::TSchemeCacheNavigate::EKind::KindPath: if (GetProtoRequest()->recurse()) { SendNavigateRequest(ctx, childFullPath); } - break; - case NSchemeCache::TSchemeCacheNavigate::EKind::KindTopic: - Topics.push_back(childFullPath); - break; - default: - break; - // ignore all other types - } - } - } - } - RequestsInFlight--; - SendPendingRequests(ctx); - if (RequestsInFlight == 0) { - SendResponse(ctx); - } - } - - //----------------------------------------------------------------------------------- - + break; + case NSchemeCache::TSchemeCacheNavigate::EKind::KindTopic: + Topics.push_back(childFullPath); + break; + default: + break; + // ignore all other types + } + } + } + } + RequestsInFlight--; + SendPendingRequests(ctx); + if (RequestsInFlight == 0) { + SendResponse(ctx); + } + } + + //----------------------------------------------------------------------------------- + class TListStreamConsumersActor : public TPQGrpcSchemaBase<TListStreamConsumersActor, NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest> { using TBase = TPQGrpcSchemaBase<TListStreamConsumersActor, TEvDataStreamsListStreamConsumersRequest>; @@ -1657,276 +1657,276 @@ namespace NKikimr::NDataStreams::V1 { //----------------------------------------------------------------------------------------- - template<class TEvRequest> - class TNotImplementedRequestActor : public TRpcSchemeRequestActor<TNotImplementedRequestActor<TEvRequest>, TEvRequest> { - using TBase = TRpcSchemeRequestActor<TNotImplementedRequestActor, TEvRequest>; - - public: - TNotImplementedRequestActor(TEvRequest* request) - : TBase(request) - { - } - ~TNotImplementedRequestActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - this->Request_->RaiseIssue(FillIssue("Method is not implemented yet", Ydb::PersQueue::ErrorCode::ErrorCode::ERROR)); - this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNSUPPORTED); - this->Die(ctx); - } - }; - + template<class TEvRequest> + class TNotImplementedRequestActor : public TRpcSchemeRequestActor<TNotImplementedRequestActor<TEvRequest>, TEvRequest> { + using TBase = TRpcSchemeRequestActor<TNotImplementedRequestActor, TEvRequest>; + + public: + TNotImplementedRequestActor(TEvRequest* request) + : TBase(request) + { + } + ~TNotImplementedRequestActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + this->Request_->RaiseIssue(FillIssue("Method is not implemented yet", Ydb::PersQueue::ErrorCode::ErrorCode::ERROR)); + this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNSUPPORTED); + this->Die(ctx); + } + }; + //----------------------------------------------------------------------------------- - - IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) { - return new TDataStreamsService(counters, newSchemeCache); - } - - TDataStreamsService::TDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) - : Counters(counters) - , NewSchemeCache(newSchemeCache) - { - } - - void TDataStreamsService::Bootstrap(const TActorContext&) { - Become(&TThis::StateFunc); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx) { + + IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) { + return new TDataStreamsService(counters, newSchemeCache); + } + + TDataStreamsService::TDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) + : Counters(counters) + , NewSchemeCache(newSchemeCache) + { + } + + void TDataStreamsService::Bootstrap(const TActorContext&) { + Become(&TThis::StateFunc); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TCreateStreamActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDeleteStreamActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDescribeStreamActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TDeleteStreamActor(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TDescribeStreamActor(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TRegisterStreamConsumerActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TDeregisterStreamConsumerActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TPutRecordActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TListStreamsActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TPutRecordActor(ev->Release().Release(), NewSchemeCache)); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TListStreamsActor(ev->Release().Release(), NewSchemeCache)); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TListShardsActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TPutRecordsActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TPutRecordsActor(ev->Release().Release(), NewSchemeCache)); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TGetRecordsActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TGetShardIteratorActor(ev->Release().Release(), NewSchemeCache)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TDescribeStreamSummaryActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TSetStreamRetentionPeriodActor<TEvDataStreamsDecreaseStreamRetentionPeriodRequest>(ev->Release().Release(), false)); - } - - void TDataStreamsService::Handle(TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TSetStreamRetentionPeriodActor<TEvDataStreamsIncreaseStreamRetentionPeriodRequest>(ev->Release().Release(), true)); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TUpdateShardCountActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TUpdateStreamActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) { + } + + void TDataStreamsService::Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TSetStreamRetentionPeriodActor<TEvDataStreamsDecreaseStreamRetentionPeriodRequest>(ev->Release().Release(), false)); + } + + void TDataStreamsService::Handle(TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TSetStreamRetentionPeriodActor<TEvDataStreamsIncreaseStreamRetentionPeriodRequest>(ev->Release().Release(), true)); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TUpdateShardCountActor(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TUpdateStreamActor(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) { ctx.Register(new TListStreamConsumersActor(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest>(ev->Release().Release())); - } - - void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest>(ev->Release().Release())); - } - -} - - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSetWriteQuotaRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); -} + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest>(ev->Release().Release())); + } + + void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TNotImplementedRequestActor<NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest>(ev->Release().Release())); + } + +} + + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSetWriteQuotaRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NDataStreams::V1::GetDataStreamsServiceActorID(), ev->Release().Release()); +} diff --git a/ydb/services/datastreams/datastreams_proxy.h b/ydb/services/datastreams/datastreams_proxy.h index 8904d1fd50..1712e171ee 100644 --- a/ydb/services/datastreams/datastreams_proxy.h +++ b/ydb/services/datastreams/datastreams_proxy.h @@ -1,98 +1,98 @@ -#pragma once - -#include "events.h" - +#pragma once + +#include "events.h" + #include <ydb/core/client/server/grpc_base.h> #include <ydb/core/grpc_services/rpc_calls.h> - + #include <library/cpp/grpc/server/grpc_server.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/actorsystem.h> - -namespace NKikimr::NDataStreams::V1 { - - inline TActorId GetDataStreamsServiceActorID() { - return TActorId(0, "PqDsProxy"); - } - - IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache); - - class TDataStreamsService : public NActors::TActorBootstrapped<TDataStreamsService> { - public: - TDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache); - - void Bootstrap(const TActorContext& ctx); - - private: - STFUNC(StateFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest, Handle); - } - } - - void Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); - - private: - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - - TActorId NewSchemeCache; - }; - -} + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/actorsystem.h> + +namespace NKikimr::NDataStreams::V1 { + + inline TActorId GetDataStreamsServiceActorID() { + return TActorId(0, "PqDsProxy"); + } + + IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache); + + class TDataStreamsService : public NActors::TActorBootstrapped<TDataStreamsService> { + public: + TDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache); + + void Bootstrap(const TActorContext& ctx); + + private: + STFUNC(StateFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest, Handle); + } + } + + void Handle(NKikimr::NGRpcService::TEvDataStreamsCreateStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeLimitsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsIncreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateShardCountRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsUpdateStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsDisableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsEnableEnhancedMonitoringRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsListTagsForStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsMergeShardsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsRemoveTagsFromStreamRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsSplitShardRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsStartStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvDataStreamsStopStreamEncryptionRequest::TPtr& ev, const TActorContext& ctx); + + private: + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + + TActorId NewSchemeCache; + }; + +} diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index d348804d79..2ede2a2d05 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -2,40 +2,40 @@ #include <ydb/services/ydb/ydb_common_ut.h> #include <ydb/services/persqueue_v1/ut/persqueue_test_fixture.h> #include <ydb/services/ydb/ydb_keys_ut.h> - + #include <ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h> - + #include <library/cpp/json/json_reader.h> #include <library/cpp/digest/md5/md5.h> #include <random> -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NKikimr::NPersQueueTests; +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NKikimr::NPersQueueTests; using namespace NKikimr::NDataStreams::V1; namespace YDS_V1 = Ydb::DataStreams::V1; namespace NYDS_V1 = NYdb::NDataStreams::V1; -struct WithSslAndAuth : TKikimrTestSettings { - static constexpr bool SSL = true; - static constexpr bool AUTH = true; -}; -using TKikimrWithGrpcAndRootSchemaSecure = NYdb::TBasicKikimrWithGrpcAndRootSchema<WithSslAndAuth>; - - -template<class TKikimr, bool secure> +struct WithSslAndAuth : TKikimrTestSettings { + static constexpr bool SSL = true; + static constexpr bool AUTH = true; +}; +using TKikimrWithGrpcAndRootSchemaSecure = NYdb::TBasicKikimrWithGrpcAndRootSchema<WithSslAndAuth>; + + +template<class TKikimr, bool secure> class TDatastreamsTestServer { -public: +public: TDatastreamsTestServer() { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutablePQConfig()->SetTopicsAreFirstClassCitizen(true); - appConfig.MutablePQConfig()->SetEnabled(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutablePQConfig()->SetTopicsAreFirstClassCitizen(true); + appConfig.MutablePQConfig()->SetEnabled(true); appConfig.MutablePQConfig()->SetMetaCacheRefreshIntervalMilliSeconds(30000); appConfig.MutablePQConfig()->MutableQuotingConfig()->SetEnableQuoting(true); appConfig.MutablePQConfig()->MutableQuotingConfig()->SetQuotaWaitDurationMs(300); @@ -47,51 +47,51 @@ public: MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt"); appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name()); - if (secure) { - appConfig.MutablePQConfig()->SetRequireCredentialsInNewProtocol(true); - } - KikimrServer = std::make_unique<TKikimr>(std::move(appConfig)); - ui16 grpc = KikimrServer->GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + if (secure) { + appConfig.MutablePQConfig()->SetRequireCredentialsInNewProtocol(true); + } + KikimrServer = std::make_unique<TKikimr>(std::move(appConfig)); + ui16 grpc = KikimrServer->GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; auto driverConfig = TDriverConfig().SetEndpoint(location).SetLog(CreateLogBackend("cerr", TLOG_DEBUG)); - if (secure) { - driverConfig.UseSecureConnection(NYdbSslTestData::CaCrt); - } else { - driverConfig.SetDatabase("/Root/"); - } - - Driver = std::make_unique<TDriver>(std::move(driverConfig)); + if (secure) { + driverConfig.UseSecureConnection(NYdbSslTestData::CaCrt); + } else { + driverConfig.SetDatabase("/Root/"); + } + + Driver = std::make_unique<TDriver>(std::move(driverConfig)); DataStreamsClient = std::make_unique<NYDS_V1::TDataStreamsClient>(*Driver, - TCommonClientSettings() - .AuthToken("user@builtin")); - - { - NYdb::NScheme::TSchemeClient schemeClient(*Driver); - NYdb::NScheme::TPermissions permissions("user@builtin", {"ydb.generic.read", "ydb.generic.write"}); - - auto result = schemeClient.ModifyPermissions("/Root", + TCommonClientSettings() + .AuthToken("user@builtin")); + + { + NYdb::NScheme::TSchemeClient schemeClient(*Driver); + NYdb::NScheme::TPermissions permissions("user@builtin", {"ydb.generic.read", "ydb.generic.write"}); + + auto result = schemeClient.ModifyPermissions("/Root", NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); + ).ExtractValueSync(); Cerr << result.GetIssues().ToString() << "\n"; UNIT_ASSERT(result.IsSuccess()); - } + } TClient client(*(KikimrServer->ServerSettings)); UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, client.AlterUserAttributes("/", "Root", {{"folder_id", "somefolder"},{"cloud_id", "somecloud"}, {"database_id", "root"}})); - } - -public: - std::unique_ptr<TKikimr> KikimrServer; - std::unique_ptr<TDriver> Driver; + } + +public: + std::unique_ptr<TKikimr> KikimrServer; + std::unique_ptr<TDriver> Driver; std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient; std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient; THolder<TTempFileHandle> MeteringFile; -}; - +}; + using TInsecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchema, false>; using TSecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchemaSecure, true>; - + void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) { Sleep(TDuration::Seconds(1)); meteringFile->Flush(); @@ -131,29 +131,29 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) #define Y_UNIT_TEST_NAME this->Name_; -Y_UNIT_TEST_SUITE(DataStreams) { - +Y_UNIT_TEST_SUITE(DataStreams) { + Y_UNIT_TEST(TestControlPlaneAndMeteringData) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - // Trying to delete stream that doesn't exist yet - { + // Trying to delete stream that doesn't exist yet + { auto result = testServer.DataStreamsClient->DeleteStream("testfolder/" + streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); } - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -189,10 +189,10 @@ Y_UNIT_TEST_SUITE(DataStreams) { { auto result = testServer.DataStreamsClient->CreateStream("testfolder/" + streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - + } + { // for metering purposes std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { @@ -205,102 +205,102 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } - { + { auto result = testServer.DataStreamsClient->ListStreams().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names(0), streamName); - } - - // cannot decrease the number of shards - { + } + + // cannot decrease the number of shards + { auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, NYDS_V1::TUpdateShardCountSettings().TargetShardCount(2)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, NYDS_V1::TUpdateShardCountSettings().TargetShardCount(15)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + - // now when stream is created delete should work fine - { + // now when stream is created delete should work fine + { auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - // Describe should fail after delete - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + // Describe should fail after delete + { auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName); - } - + } + Y_UNIT_TEST(TestCreateExistingStream) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS); - } - - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS); + } + + } + Y_UNIT_TEST(TestStreamPagination) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) { - for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) { + for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) { + for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) { TStringBuilder streamNameX = TStringBuilder() << folderIdx << streamName << streamIdx; auto result = testServer.DataStreamsClient->CreateStream(streamNameX, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); Cerr << result.GetIssues().ToString() << "\n"; - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - - TString startStream; - THashSet<TString> streams; - for (int i = 0; i < 3; i++) { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + + TString startStream; + THashSet<TString> streams; + for (int i = 0; i < 3; i++) { auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 6); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().has_more_streams(), true); - streams.insert(result.GetResult().stream_names().begin(), result.GetResult().stream_names().end()); - startStream = result.GetResult().stream_names(5); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 6); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().has_more_streams(), true); + streams.insert(result.GetResult().stream_names().begin(), result.GetResult().stream_names().end()); + startStream = result.GetResult().stream_names(5); + } + + { auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().has_more_streams(), false); - streams.insert(result.GetResult().stream_names().begin(), result.GetResult().stream_names().end()); - } - - UNIT_ASSERT_VALUES_EQUAL(streams.size(), 20); - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().has_more_streams(), false); + streams.insert(result.GetResult().stream_names().begin(), result.GetResult().stream_names().end()); + } + + UNIT_ASSERT_VALUES_EQUAL(streams.size(), 20); + } + Y_UNIT_TEST(TestDeleteStream) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; @@ -392,130 +392,130 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestUpdateStream) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - for (ui32 i = 0; i < 2; ++i) { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + for (ui32 i = 0; i < 2; ++i) { auto result = testServer.DataStreamsClient->UpdateStream(streamName, NYDS_V1::TUpdateStreamSettings().RetentionPeriodHours(5).TargetShardCount(20).WriteQuotaKbPerSec(128) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards_size(), 20); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 5); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 128); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().owner(), "user@builtin"); UNIT_ASSERT(result.GetResult().stream_description().stream_creation_timestamp() > 0); - } - } - + } + } + Y_UNIT_TEST(TestStreamRetention) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(50) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { auto result = testServer.DataStreamsClient->DecreaseStreamRetentionPeriod(streamName, NYDS_V1::TDecreaseStreamRetentionPeriodSettings().RetentionPeriodHours(8) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(4) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(15) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + } + Y_UNIT_TEST(TestShardPagination) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { + { auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - // describe stream - { - TString exclusiveStartShardId; - THashSet<TString> describedShards; + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + // describe stream + { + TString exclusiveStartShardId; + THashSet<TString> describedShards; for (int i = 0; i < 8; i += 2) { auto result = testServer.DataStreamsClient->DescribeStream(streamName, NYDS_V1::TDescribeStreamSettings() .Limit(2) - .ExclusiveStartShardId(exclusiveStartShardId) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + .ExclusiveStartShardId(exclusiveStartShardId) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto& description = result.GetResult().stream_description(); UNIT_ASSERT_VALUES_EQUAL(description.shards().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(description.has_more_shards(), true); - for (const auto& shard : description.shards()) { - describedShards.insert(shard.shard_id()); - } + UNIT_ASSERT_VALUES_EQUAL(description.has_more_shards(), true); + for (const auto& shard : description.shards()) { + describedShards.insert(shard.shard_id()); + } exclusiveStartShardId = description.shards(1).shard_id(); - } - - { + } + + { auto result = testServer.DataStreamsClient->DescribeStream(streamName, NYDS_V1::TDescribeStreamSettings() .Limit(2) - .ExclusiveStartShardId( + .ExclusiveStartShardId( exclusiveStartShardId) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto &description = result.GetResult().stream_description(); UNIT_ASSERT_VALUES_EQUAL(description.shards().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(description.has_more_shards(), false); - for (const auto& shard : description.shards()) { - describedShards.insert(shard.shard_id()); - } - } - - // check for total number of shards + UNIT_ASSERT_VALUES_EQUAL(description.has_more_shards(), false); + for (const auto& shard : description.shards()) { + describedShards.insert(shard.shard_id()); + } + } + + // check for total number of shards UNIT_ASSERT_EQUAL(describedShards.size(), 9); - } - - } - + } + + } + #define SET_YDS_LOCALS \ auto& kikimr = testServer.KikimrServer->Server_; \ @@ -528,12 +528,12 @@ Y_UNIT_TEST_SUITE(DataStreams) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; - { + { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); @@ -547,20 +547,20 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); - { + { std::vector<NYDS_V1::TDataRecord> records; - for (ui32 i = 1; i <= 30; ++i) { - TString data = Sprintf("%04u", i); - records.push_back({data, data, ""}); - } + for (ui32 i = 1; i <= 30; ++i) { + TString data = Sprintf("%04u", i); + records.push_back({data, data, ""}); + } auto result = client.PutRecords(streamName, records).ExtractValueSync(); - Cerr << result.GetResult().DebugString() << Endl; - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - + Cerr << result.GetResult().DebugString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + NYdb::NPersQueue::TPersQueueClient pqClient(*driver); - + { auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -570,105 +570,105 @@ Y_UNIT_TEST_SUITE(DataStreams) { YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); } - auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings() + auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings() .ConsumerName("user1") - .DisableClusterDiscovery(true) + .DisableClusterDiscovery(true) .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName))); - ui32 readCount = 0; - while (readCount < 31) { - auto event = session->GetEvent(true); + ui32 readCount = 0; + while (readCount < 31) { + auto event = session->GetEvent(true); - if (auto* dataReceivedEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&*event)) { - for (const auto& item : dataReceivedEvent->GetMessages()) { + if (auto* dataReceivedEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&*event)) { + for (const auto& item : dataReceivedEvent->GetMessages()) { Cerr << item.DebugString(true) << Endl; - UNIT_ASSERT_VALUES_EQUAL(item.GetData(), item.GetPartitionKey()); + UNIT_ASSERT_VALUES_EQUAL(item.GetData(), item.GetPartitionKey()); auto hashKey = item.GetExplicitHash().empty() ? HexBytesToDecimal(MD5::Calc(item.GetPartitionKey())) : BytesToDecimal(item.GetExplicitHash()); UNIT_ASSERT_VALUES_EQUAL(NKikimr::NDataStreams::V1::ShardFromDecimal(hashKey, 5), item.GetPartitionStream()->GetPartitionId()); UNIT_ASSERT(!item.GetIp().empty()); - if (item.GetData() == dataStr) { + if (item.GetData() == dataStr) { UNIT_ASSERT_VALUES_EQUAL(item.GetExplicitHash(), dataStr); - } - readCount++; - } - } else if (auto* createPartitionStreamEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event)) { - createPartitionStreamEvent->Confirm(); - } else if (auto* destroyPartitionStreamEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent>(&*event)) { - destroyPartitionStreamEvent->Confirm(); - } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&*event)) { - break; - } - } - UNIT_ASSERT_VALUES_EQUAL(readCount, 31); - } - + } + readCount++; + } + } else if (auto* createPartitionStreamEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event)) { + createPartitionStreamEvent->Confirm(); + } else if (auto* destroyPartitionStreamEvent = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent>(&*event)) { + destroyPartitionStreamEvent->Confirm(); + } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&*event)) { + break; + } + } + UNIT_ASSERT_VALUES_EQUAL(readCount, 31); + } + Y_UNIT_TEST(TestPutRecordsCornerCases) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; const TString streamPath = "/Root/" + streamName; SET_YDS_LOCALS; - { + { auto result = testServer.DataStreamsClient->CreateStream( streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5) ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); - - // Test for too long partition key - TString longKey = TString(257, '1'); - TString shortEnoughKey = TString(256, '1'); + + // Test for too long partition key + TString longKey = TString(257, '1'); + TString shortEnoughKey = TString(256, '1'); auto result = client.PutRecords(streamName, {{longKey, longKey, ""}, {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + result = client.PutRecords(streamName, {{shortEnoughKey, shortEnoughKey, ""}, {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - // Test for too long data - TString longData = TString(1048577, '1'); - TString shortEnoughData = TString(1048576, '1'); - + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + // Test for too long data + TString longData = TString(1048577, '1'); + TString shortEnoughData = TString(1048576, '1'); + result = client.PutRecords(streamName, {{longData, shortEnoughKey, ""}, {shortEnoughData, shortEnoughKey, ""}}).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + result = client.PutRecords(streamName, {{shortEnoughData, shortEnoughKey, ""}, {"", shortEnoughKey, ""}}).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - TString longExplicitHash = "340282366920938463463374607431768211456"; - TString shortEnoughExplicitHash = "340282366920938463463374607431768211455"; - TString badExplicitHash = "-439025493205215"; - + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + TString longExplicitHash = "340282366920938463463374607431768211456"; + TString shortEnoughExplicitHash = "340282366920938463463374607431768211455"; + TString badExplicitHash = "-439025493205215"; + result = client.PutRecords(streamName, {{"", shortEnoughKey, longExplicitHash}, {"", shortEnoughKey, longExplicitHash}}).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + result = client.PutRecords(streamName, {{"", shortEnoughKey, badExplicitHash}, {"", shortEnoughKey, badExplicitHash}}).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + result = client.PutRecords(streamName, {{"", shortEnoughKey, shortEnoughExplicitHash}, {"", shortEnoughKey, shortEnoughExplicitHash}}).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); result = client.PutRecords(streamName, {{"", shortEnoughKey, "0"}, @@ -754,63 +754,63 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } UNIT_ASSERT_VALUES_EQUAL(readCount, 14); - } - + } + Y_UNIT_TEST(TestPutRecords) { TSecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; const TString streamPath = "/Root/" + streamName; - { + { auto result = testServer.DataStreamsClient->CreateStream(streamPath, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - + } + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); NYdb::NScheme::TSchemeClient schemeClient(*driver); - { + { std::vector<NYDS_V1::TDataRecord> records; - for (ui32 i = 1; i <= 30; ++i) { - TString data = Sprintf("%04u", i); - records.push_back({data, data, ""}); - } + for (ui32 i = 1; i <= 30; ++i) { + TString data = Sprintf("%04u", i); + records.push_back({data, data, ""}); + } auto result = client.PutRecords(streamPath, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - if (result.GetStatus() != EStatus::SUCCESS) { - result.GetIssues().PrintTo(Cerr); - } - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + if (result.GetStatus() != EStatus::SUCCESS) { + result.GetIssues().PrintTo(Cerr); + } + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); + { NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read", "ydb.generic.write"}); auto result = schemeClient.ModifyPermissions(streamPath, NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); } - + result = client.PutRecords(streamPath, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl; - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records_size(), records.size()); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE); - - TString dataStr = "9876543210"; + + TString dataStr = "9876543210"; auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); - Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); + Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().shard_id(), "shard-000004"); - UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "7"); + UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "7"); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE); - } - } - + } + } + Y_UNIT_TEST(TestPutEmptyMessage) { TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; @@ -894,34 +894,34 @@ Y_UNIT_TEST_SUITE(DataStreams) { } // List stream consumers more than allowed -> get BAD_REQUEST - { + { auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, NYDS_V1::TListStreamConsumersSettings().MaxResults(10001)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - + } + // List stream consumers less than allowed -> get BAD_REQUEST - { + { auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, NYDS_V1::TListStreamConsumersSettings().MaxResults(0)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - + } + // List not created stream -> get SCHEME_ERROR - { + { auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName + "_XXX", NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - + } + // Deregister unregistered consumer -> get NOT_FOUND - { + { auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1", NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::NOT_FOUND); } @@ -1079,9 +1079,9 @@ Y_UNIT_TEST_SUITE(DataStreams) { ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + } + + { std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { TString data = Sprintf("%04u", i); @@ -1089,7 +1089,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); } @@ -1302,8 +1302,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - + } + std::string data; data.resize(1 << 10); std::iota(data.begin(), data.end(), 1); @@ -1319,7 +1319,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { NYdb::NScheme::TSchemeClient schemeClient(*driver); TString shardIterator; - { + { auto result = client.GetShardIterator( streamName, "shard-000000", YDS_V1::ShardIteratorType::TRIM_HORIZON @@ -1485,7 +1485,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -1574,9 +1574,9 @@ Y_UNIT_TEST_SUITE(DataStreams) { NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { + } + + { auto result = testServer.DataStreamsClient->ListShards(streamName, {}, NYDS_V1::TListShardsSettings()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -1630,70 +1630,70 @@ Y_UNIT_TEST_SUITE(DataStreams) { TInsecureDatastreamsTestServer testServer; { auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->DescribeStreamConsumer().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->AddTagsToStream().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->DisableEnhancedMonitoring().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->EnableEnhancedMonitoring().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->ListTagsForStream().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->MergeShards().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->RemoveTagsFromStream().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->SplitShard().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->StartStreamEncryption().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + { auto result = testServer.DataStreamsClient->StopStreamEncryption().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - } - - } - -} + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + } + + } + +} diff --git a/ydb/services/datastreams/events.h b/ydb/services/datastreams/events.h index a10d4024c9..828f777287 100644 --- a/ydb/services/datastreams/events.h +++ b/ydb/services/datastreams/events.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/actors/core/event_local.h> - -namespace NKikimr::NDataStreams::V1 { - - struct TEvDataStreams { - enum EEv { - EvPartitionActorResult - }; - - struct TEvPartitionActorResult : public NActors::TEventLocal<TEvPartitionActorResult, EvPartitionActorResult> { - ui32 PartitionId; - ui64 CurrentOffset; - TMaybe<TString> ErrorText; - TMaybe<NPersQueue::NErrorCode::EErrorCode> ErrorCode; - }; - }; - -} - + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/event_local.h> + +namespace NKikimr::NDataStreams::V1 { + + struct TEvDataStreams { + enum EEv { + EvPartitionActorResult + }; + + struct TEvPartitionActorResult : public NActors::TEventLocal<TEvPartitionActorResult, EvPartitionActorResult> { + ui32 PartitionId; + ui64 CurrentOffset; + TMaybe<TString> ErrorText; + TMaybe<NPersQueue::NErrorCode::EErrorCode> ErrorCode; + }; + }; + +} + diff --git a/ydb/services/datastreams/grpc_service.cpp b/ydb/services/datastreams/grpc_service.cpp index c42c78cd09..7a18cd4dc6 100644 --- a/ydb/services/datastreams/grpc_service.cpp +++ b/ydb/services/datastreams/grpc_service.cpp @@ -1,164 +1,164 @@ -#include "grpc_service.h" - -#include "datastreams_proxy.h" - +#include "grpc_service.h" + +#include "datastreams_proxy.h" + #include <ydb/core/base/appdata.h> - + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/rpc_calls.h> - + #include <ydb/core/tx/scheme_board/cache.h> - -namespace NKikimr::NGRpcService { - -TGRpcDataStreamsService::TGRpcDataStreamsService(NActors::TActorSystem *system, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ -} - + +namespace NKikimr::NGRpcService { + +TGRpcDataStreamsService::TGRpcDataStreamsService(NActors::TActorSystem *system, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId id) + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ +} + void TGRpcDataStreamsService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) -{ - CQ_ = cq; - - InitNewSchemeCache(); - IActor *proxyService = NDataStreams::V1::CreateDataStreamsService(Counters_, NewSchemeCache); - auto actorId = ActorSystem_->Register(proxyService, TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); - ActorSystem_->RegisterLocalService(NDataStreams::V1::GetDataStreamsServiceActorID(), actorId); - - SetupIncomingRequests(logger); -} - +{ + CQ_ = cq; + + InitNewSchemeCache(); + IActor *proxyService = NDataStreams::V1::CreateDataStreamsService(Counters_, NewSchemeCache); + auto actorId = ActorSystem_->Register(proxyService, TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); + ActorSystem_->RegisterLocalService(NDataStreams::V1::GetDataStreamsServiceActorID(), actorId); + + SetupIncomingRequests(logger); +} + void TGRpcDataStreamsService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter *limiter) { - Limiter_ = limiter; -} - -bool TGRpcDataStreamsService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcDataStreamsService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - -void TGRpcDataStreamsService::InitNewSchemeCache() { - auto appData = ActorSystem_->AppData<TAppData>(); + Limiter_ = limiter; +} + +bool TGRpcDataStreamsService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcDataStreamsService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + +void TGRpcDataStreamsService::InitNewSchemeCache() { + auto appData = ActorSystem_->AppData<TAppData>(); auto cacheCounters = GetServiceCounters(Counters_, "pqproxy|schemecache"); auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); NewSchemeCache = ActorSystem_->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); -} - +} + void TGRpcDataStreamsService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) -{ - auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::DataStreams::V1::IN, Ydb::DataStreams::V1::OUT, TGRpcDataStreamsService>>(this, &Service_, CQ_, \ +{ + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + MakeIntrusive<TGRpcRequest<Ydb::DataStreams::V1::IN, Ydb::DataStreams::V1::OUT, TGRpcDataStreamsService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ - ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ACTION; \ - }, &Ydb::DataStreams::V1::DataStreamsService::AsyncService::Request ## NAME, \ + ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ACTION; \ + }, &Ydb::DataStreams::V1::DataStreamsService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("data_streams", #NAME))->Run(); - - ADD_REQUEST(DescribeStream, DescribeStreamRequest, DescribeStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamRequest(ctx)); - }) - ADD_REQUEST(CreateStream, CreateStreamRequest, CreateStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsCreateStreamRequest(ctx)); - }) - ADD_REQUEST(ListStreams, ListStreamsRequest, ListStreamsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListStreamsRequest(ctx)); - }) - ADD_REQUEST(DeleteStream, DeleteStreamRequest, DeleteStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDeleteStreamRequest(ctx)); - }) - ADD_REQUEST(ListShards, ListShardsRequest, ListShardsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListShardsRequest(ctx)); - }) - ADD_REQUEST(PutRecord, PutRecordRequest, PutRecordResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsPutRecordRequest(ctx)); - }) - ADD_REQUEST(PutRecords, PutRecordsRequest, PutRecordsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsPutRecordsRequest(ctx)); - }) - ADD_REQUEST(GetRecords, GetRecordsRequest, GetRecordsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsGetRecordsRequest(ctx)); - }) - ADD_REQUEST(GetShardIterator, GetShardIteratorRequest, GetShardIteratorResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsGetShardIteratorRequest(ctx)); - }) - ADD_REQUEST(SubscribeToShard, SubscribeToShardRequest, SubscribeToShardResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSubscribeToShardRequest(ctx)); - }) - ADD_REQUEST(DescribeLimits, DescribeLimitsRequest, DescribeLimitsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeLimitsRequest(ctx)); - }) - ADD_REQUEST(DescribeStreamSummary, DescribeStreamSummaryRequest, DescribeStreamSummaryResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamSummaryRequest(ctx)); - }) - ADD_REQUEST(DecreaseStreamRetentionPeriod, DecreaseStreamRetentionPeriodRequest, DecreaseStreamRetentionPeriodResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDecreaseStreamRetentionPeriodRequest(ctx)); - }) - ADD_REQUEST(IncreaseStreamRetentionPeriod, IncreaseStreamRetentionPeriodRequest, IncreaseStreamRetentionPeriodResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsIncreaseStreamRetentionPeriodRequest(ctx)); - }) - ADD_REQUEST(UpdateShardCount, UpdateShardCountRequest, UpdateShardCountResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsUpdateShardCountRequest(ctx)); - }) - ADD_REQUEST(RegisterStreamConsumer, RegisterStreamConsumerRequest, RegisterStreamConsumerResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsRegisterStreamConsumerRequest(ctx)); - }) - ADD_REQUEST(DeregisterStreamConsumer, DeregisterStreamConsumerRequest, DeregisterStreamConsumerResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDeregisterStreamConsumerRequest(ctx)); - }) - ADD_REQUEST(DescribeStreamConsumer, DescribeStreamConsumerRequest, DescribeStreamConsumerResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamConsumerRequest(ctx)); - }) - ADD_REQUEST(ListStreamConsumers, ListStreamConsumersRequest, ListStreamConsumersResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListStreamConsumersRequest(ctx)); - }) - ADD_REQUEST(AddTagsToStream, AddTagsToStreamRequest, AddTagsToStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsAddTagsToStreamRequest(ctx)); - }) - ADD_REQUEST(DisableEnhancedMonitoring, DisableEnhancedMonitoringRequest, DisableEnhancedMonitoringResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDisableEnhancedMonitoringRequest(ctx)); - }) - ADD_REQUEST(EnableEnhancedMonitoring, EnableEnhancedMonitoringRequest, EnableEnhancedMonitoringResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsEnableEnhancedMonitoringRequest(ctx)); - }) - ADD_REQUEST(ListTagsForStream, ListTagsForStreamRequest, ListTagsForStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListTagsForStreamRequest(ctx)); - }) - ADD_REQUEST(MergeShards, MergeShardsRequest, MergeShardsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsMergeShardsRequest(ctx)); - }) - ADD_REQUEST(RemoveTagsFromStream, RemoveTagsFromStreamRequest, RemoveTagsFromStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsRemoveTagsFromStreamRequest(ctx)); - }) - ADD_REQUEST(SplitShard, SplitShardRequest, SplitShardResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSplitShardRequest(ctx)); - }) - ADD_REQUEST(StartStreamEncryption, StartStreamEncryptionRequest, StartStreamEncryptionResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsStartStreamEncryptionRequest(ctx)); - }) - ADD_REQUEST(StopStreamEncryption, StopStreamEncryptionRequest, StopStreamEncryptionResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsStopStreamEncryptionRequest(ctx)); - }) - ADD_REQUEST(UpdateStream, UpdateStreamRequest, UpdateStreamResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsUpdateStreamRequest(ctx)); - }) - ADD_REQUEST(SetWriteQuota, SetWriteQuotaRequest, SetWriteQuotaResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSetWriteQuotaRequest(ctx)); - }) - -#undef ADD_REQUEST -} - -} + + ADD_REQUEST(DescribeStream, DescribeStreamRequest, DescribeStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamRequest(ctx)); + }) + ADD_REQUEST(CreateStream, CreateStreamRequest, CreateStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsCreateStreamRequest(ctx)); + }) + ADD_REQUEST(ListStreams, ListStreamsRequest, ListStreamsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListStreamsRequest(ctx)); + }) + ADD_REQUEST(DeleteStream, DeleteStreamRequest, DeleteStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDeleteStreamRequest(ctx)); + }) + ADD_REQUEST(ListShards, ListShardsRequest, ListShardsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListShardsRequest(ctx)); + }) + ADD_REQUEST(PutRecord, PutRecordRequest, PutRecordResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsPutRecordRequest(ctx)); + }) + ADD_REQUEST(PutRecords, PutRecordsRequest, PutRecordsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsPutRecordsRequest(ctx)); + }) + ADD_REQUEST(GetRecords, GetRecordsRequest, GetRecordsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsGetRecordsRequest(ctx)); + }) + ADD_REQUEST(GetShardIterator, GetShardIteratorRequest, GetShardIteratorResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsGetShardIteratorRequest(ctx)); + }) + ADD_REQUEST(SubscribeToShard, SubscribeToShardRequest, SubscribeToShardResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSubscribeToShardRequest(ctx)); + }) + ADD_REQUEST(DescribeLimits, DescribeLimitsRequest, DescribeLimitsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeLimitsRequest(ctx)); + }) + ADD_REQUEST(DescribeStreamSummary, DescribeStreamSummaryRequest, DescribeStreamSummaryResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamSummaryRequest(ctx)); + }) + ADD_REQUEST(DecreaseStreamRetentionPeriod, DecreaseStreamRetentionPeriodRequest, DecreaseStreamRetentionPeriodResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDecreaseStreamRetentionPeriodRequest(ctx)); + }) + ADD_REQUEST(IncreaseStreamRetentionPeriod, IncreaseStreamRetentionPeriodRequest, IncreaseStreamRetentionPeriodResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsIncreaseStreamRetentionPeriodRequest(ctx)); + }) + ADD_REQUEST(UpdateShardCount, UpdateShardCountRequest, UpdateShardCountResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsUpdateShardCountRequest(ctx)); + }) + ADD_REQUEST(RegisterStreamConsumer, RegisterStreamConsumerRequest, RegisterStreamConsumerResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsRegisterStreamConsumerRequest(ctx)); + }) + ADD_REQUEST(DeregisterStreamConsumer, DeregisterStreamConsumerRequest, DeregisterStreamConsumerResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDeregisterStreamConsumerRequest(ctx)); + }) + ADD_REQUEST(DescribeStreamConsumer, DescribeStreamConsumerRequest, DescribeStreamConsumerResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamConsumerRequest(ctx)); + }) + ADD_REQUEST(ListStreamConsumers, ListStreamConsumersRequest, ListStreamConsumersResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListStreamConsumersRequest(ctx)); + }) + ADD_REQUEST(AddTagsToStream, AddTagsToStreamRequest, AddTagsToStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsAddTagsToStreamRequest(ctx)); + }) + ADD_REQUEST(DisableEnhancedMonitoring, DisableEnhancedMonitoringRequest, DisableEnhancedMonitoringResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDisableEnhancedMonitoringRequest(ctx)); + }) + ADD_REQUEST(EnableEnhancedMonitoring, EnableEnhancedMonitoringRequest, EnableEnhancedMonitoringResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsEnableEnhancedMonitoringRequest(ctx)); + }) + ADD_REQUEST(ListTagsForStream, ListTagsForStreamRequest, ListTagsForStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsListTagsForStreamRequest(ctx)); + }) + ADD_REQUEST(MergeShards, MergeShardsRequest, MergeShardsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsMergeShardsRequest(ctx)); + }) + ADD_REQUEST(RemoveTagsFromStream, RemoveTagsFromStreamRequest, RemoveTagsFromStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsRemoveTagsFromStreamRequest(ctx)); + }) + ADD_REQUEST(SplitShard, SplitShardRequest, SplitShardResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSplitShardRequest(ctx)); + }) + ADD_REQUEST(StartStreamEncryption, StartStreamEncryptionRequest, StartStreamEncryptionResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsStartStreamEncryptionRequest(ctx)); + }) + ADD_REQUEST(StopStreamEncryption, StopStreamEncryptionRequest, StopStreamEncryptionResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsStopStreamEncryptionRequest(ctx)); + }) + ADD_REQUEST(UpdateStream, UpdateStreamRequest, UpdateStreamResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsUpdateStreamRequest(ctx)); + }) + ADD_REQUEST(SetWriteQuota, SetWriteQuotaRequest, SetWriteQuotaResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsSetWriteQuotaRequest(ctx)); + }) + +#undef ADD_REQUEST +} + +} diff --git a/ydb/services/datastreams/grpc_service.h b/ydb/services/datastreams/grpc_service.h index 37121bd222..d8ed51df52 100644 --- a/ydb/services/datastreams/grpc_service.h +++ b/ydb/services/datastreams/grpc_service.h @@ -1,35 +1,35 @@ -#pragma once - -#include <library/cpp/actors/core/actorsystem.h> +#pragma once + +#include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h> - -namespace NKikimr::NGRpcService { - + +namespace NKikimr::NGRpcService { + class TGRpcDataStreamsService : public NGrpc::TGrpcServiceBase<Ydb::DataStreams::V1::DataStreamsService> - { - public: - TGRpcDataStreamsService(NActors::TActorSystem* system, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId id); - + { + public: + TGRpcDataStreamsService(NActors::TActorSystem* system, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId id); + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); - - private: + + bool IncRequest(); + void DecRequest(); + + private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - void InitNewSchemeCache(); - - NActors::TActorSystem* ActorSystem_; + void InitNewSchemeCache(); + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; - NActors::TActorId GRpcRequestProxyId_; - NActors::TActorId NewSchemeCache; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + NActors::TActorId GRpcRequestProxyId_; + NActors::TActorId NewSchemeCache; NGrpc::TGlobalLimiter* Limiter_ = nullptr; - }; - -} + }; + +} diff --git a/ydb/services/datastreams/put_records_actor.cpp b/ydb/services/datastreams/put_records_actor.cpp index 284a6cf504..fe0f712e5a 100644 --- a/ydb/services/datastreams/put_records_actor.cpp +++ b/ydb/services/datastreams/put_records_actor.cpp @@ -1,2 +1,2 @@ -#include "put_records_actor.h" - +#include "put_records_actor.h" + diff --git a/ydb/services/datastreams/put_records_actor.h b/ydb/services/datastreams/put_records_actor.h index b97157ea7c..8f06a99157 100644 --- a/ydb/services/datastreams/put_records_actor.h +++ b/ydb/services/datastreams/put_records_actor.h @@ -1,31 +1,31 @@ -#pragma once - -#include "events.h" - +#pragma once + +#include "events.h" + #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/persqueue/events/global.h> #include <ydb/core/persqueue/write_meta.h> #include <ydb/core/protos/msgbus_pq.pb.h> #include <ydb/core/protos/grpc_pq_old.pb.h> - + #include <ydb/services/lib/actors/pq_schema_actor.h> #include <ydb/services/lib/sharding/sharding.h> - -#include <library/cpp/digest/md5/md5.h> - + +#include <library/cpp/digest/md5/md5.h> + #define PUT_UNIT_SIZE 40960u // 40Kb - -namespace NKikimr::NDataStreams::V1 { - + +namespace NKikimr::NDataStreams::V1 { - struct TPutRecordsItem { - TString Data; - TString Key; - TString ExplicitHash; + + struct TPutRecordsItem { + TString Data; + TString Key; + TString ExplicitHash; TString Ip; - }; - + }; + TString GetSerializedData(const TPutRecordsItem& item) { NKikimrPQClient::TDataChunk proto; @@ -40,21 +40,21 @@ namespace NKikimr::NDataStreams::V1 { } - class TDatastreamsPartitionActor : public TActorBootstrapped<TDatastreamsPartitionActor> { - public: - using TBase = TActorBootstrapped<TDatastreamsPartitionActor>; - - TDatastreamsPartitionActor(NActors::TActorId parentId, ui64 tabletId, ui32 partition, const TString& topic, TVector<TPutRecordsItem> dataToWrite) - : ParentId(std::move(parentId)) - , TabletId(tabletId) - , Partition(partition) - , Topic(topic) - , DataToWrite(std::move(dataToWrite)) - { - } - - void Bootstrap(const NActors::TActorContext& ctx) { - NTabletPipe::TClientConfig clientConfig; + class TDatastreamsPartitionActor : public TActorBootstrapped<TDatastreamsPartitionActor> { + public: + using TBase = TActorBootstrapped<TDatastreamsPartitionActor>; + + TDatastreamsPartitionActor(NActors::TActorId parentId, ui64 tabletId, ui32 partition, const TString& topic, TVector<TPutRecordsItem> dataToWrite) + : ParentId(std::move(parentId)) + , TabletId(tabletId) + , Partition(partition) + , Topic(topic) + , DataToWrite(std::move(dataToWrite)) + { + } + + void Bootstrap(const NActors::TActorContext& ctx) { + NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = { .RetryLimitCount = 6, .MinRetryTime = TDuration::MilliSeconds(10), @@ -62,276 +62,276 @@ namespace NKikimr::NDataStreams::V1 { .BackoffMultiplier = 2, .DoFirstRetryInstantly = true }; - PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig)); - - SendWriteRequest(ctx); - Become(&TDatastreamsPartitionActor::PartitionWriteFunc); - } - - private: - STFUNC(PartitionWriteFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvPersQueue::TEvResponse, HandlePartitionWriteResult); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - }; - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - - if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) { - ReplyWithError(ctx, TStringBuilder() << "Cannot connect to tablet " << TabletId, - NPersQueue::NErrorCode::ERROR); - } - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - - ReplyWithError(ctx, TStringBuilder() << "Connection to tablet " << TabletId << " is dead", NPersQueue::NErrorCode::TABLET_IS_DROPPED); - } - - void SendWriteRequest(const TActorContext& ctx) { - NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Topic); - request.MutablePartitionRequest()->SetPartition(Partition); - request.MutablePartitionRequest()->SetIsDirectWrite(true); - ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); + PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig)); + + SendWriteRequest(ctx); + Become(&TDatastreamsPartitionActor::PartitionWriteFunc); + } + + private: + STFUNC(PartitionWriteFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvPersQueue::TEvResponse, HandlePartitionWriteResult); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + }; + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + + if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) { + ReplyWithError(ctx, TStringBuilder() << "Cannot connect to tablet " << TabletId, + NPersQueue::NErrorCode::ERROR); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + + ReplyWithError(ctx, TStringBuilder() << "Connection to tablet " << TabletId << " is dead", NPersQueue::NErrorCode::TABLET_IS_DROPPED); + } + + void SendWriteRequest(const TActorContext& ctx) { + NKikimrClient::TPersQueueRequest request; + request.MutablePartitionRequest()->SetTopic(Topic); + request.MutablePartitionRequest()->SetPartition(Partition); + request.MutablePartitionRequest()->SetIsDirectWrite(true); + ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); ui64 totalSize = 0; - for (const auto& item : DataToWrite) { - auto w = request.MutablePartitionRequest()->AddCmdWrite(); + for (const auto& item : DataToWrite) { + auto w = request.MutablePartitionRequest()->AddCmdWrite(); w->SetData(GetSerializedData(item)); - w->SetPartitionKey(item.Key); + w->SetPartitionKey(item.Key); w->SetExplicitHash(item.ExplicitHash); - w->SetDisableDeduplication(true); - w->SetCreateTimeMS(TInstant::Now().MilliSeconds()); - w->SetUncompressedSize(item.Data.size()); + w->SetDisableDeduplication(true); + w->SetCreateTimeMS(TInstant::Now().MilliSeconds()); + w->SetUncompressedSize(item.Data.size()); w->SetExternalOperation(true); totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size()); - } + } ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE; if (totalSize % PUT_UNIT_SIZE != 0) putUnitsCount++; request.MutablePartitionRequest()->SetPutUnitsSize(putUnitsCount); - - TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); - req->Record.Swap(&request); - - NTabletPipe::SendData(ctx, PipeClient, req.Release()); - } - - void HandlePartitionWriteResult(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { - if (CheckForError(ev, ctx)) { - return; - } - - Y_VERIFY(ev->Get()->Record.HasPartitionResponse()); - Y_ENSURE(ev->Get()->Record.GetPartitionResponse().GetCmdWriteResult().size() > 0, "Wrong number of cmd write commands"); - auto offset = ev->Get()->Record.GetPartitionResponse().GetCmdWriteResult(0).GetOffset(); - ReplySuccessAndDie(ctx, offset); - } - - void ReplyWithError(const NActors::TActorContext& ctx, const TString& errorText, NPersQueue::NErrorCode::EErrorCode errorCode) { - auto result = MakeHolder<NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult>(); - result->PartitionId = Partition; - result->ErrorText = errorText; - result->ErrorCode = errorCode; - ctx.Send(ParentId, result.Release()); - Die(ctx); - } - - bool CheckForError(TEvPersQueue::TEvResponse::TPtr& ev, const NActors::TActorContext& ctx) { - const auto& record = ev->Get()->Record; - if (record.HasErrorCode() && record.GetErrorCode() != NPersQueue::NErrorCode::OK) { - ReplyWithError(ctx, record.GetErrorReason(), record.GetErrorCode()); - return true; - } - return false; - } - - void ReplySuccessAndDie(const NActors::TActorContext& ctx, ui64 offset) { - auto result = MakeHolder<NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult>(); - result->PartitionId = Partition; - result->CurrentOffset = offset; - ctx.Send(ParentId, result.Release()); - Die(ctx); - } - + + TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); + req->Record.Swap(&request); + + NTabletPipe::SendData(ctx, PipeClient, req.Release()); + } + + void HandlePartitionWriteResult(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { + if (CheckForError(ev, ctx)) { + return; + } + + Y_VERIFY(ev->Get()->Record.HasPartitionResponse()); + Y_ENSURE(ev->Get()->Record.GetPartitionResponse().GetCmdWriteResult().size() > 0, "Wrong number of cmd write commands"); + auto offset = ev->Get()->Record.GetPartitionResponse().GetCmdWriteResult(0).GetOffset(); + ReplySuccessAndDie(ctx, offset); + } + + void ReplyWithError(const NActors::TActorContext& ctx, const TString& errorText, NPersQueue::NErrorCode::EErrorCode errorCode) { + auto result = MakeHolder<NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult>(); + result->PartitionId = Partition; + result->ErrorText = errorText; + result->ErrorCode = errorCode; + ctx.Send(ParentId, result.Release()); + Die(ctx); + } + + bool CheckForError(TEvPersQueue::TEvResponse::TPtr& ev, const NActors::TActorContext& ctx) { + const auto& record = ev->Get()->Record; + if (record.HasErrorCode() && record.GetErrorCode() != NPersQueue::NErrorCode::OK) { + ReplyWithError(ctx, record.GetErrorReason(), record.GetErrorCode()); + return true; + } + return false; + } + + void ReplySuccessAndDie(const NActors::TActorContext& ctx, ui64 offset) { + auto result = MakeHolder<NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult>(); + result->PartitionId = Partition; + result->CurrentOffset = offset; + ctx.Send(ParentId, result.Release()); + Die(ctx); + } + void Die(const TActorContext& ctx) override { if (PipeClient) NTabletPipe::CloseClient(ctx, PipeClient); TBase::Die(ctx); } - private: - NActors::TActorId ParentId; - ui64 TabletId = 0; - ui32 Partition = 0; - TString Topic; - TVector<TPutRecordsItem> DataToWrite; - NActors::TActorId PipeClient; - }; - - //------------------------------------------------------------------------------------ - - namespace { - TString CheckRequestIsValid(const Ydb::DataStreams::V1::PutRecordsRequest& putRecordsRequest) { - if (putRecordsRequest.stream_name().empty()) { - return "Stream name is empty"; - } - if (putRecordsRequest.records_size() > 500) { - return TStringBuilder() << "Too many records in a single PutRecords request: " << putRecordsRequest.records_size() << " > 500"; - } + private: + NActors::TActorId ParentId; + ui64 TabletId = 0; + ui32 Partition = 0; + TString Topic; + TVector<TPutRecordsItem> DataToWrite; + NActors::TActorId PipeClient; + }; + + //------------------------------------------------------------------------------------ + + namespace { + TString CheckRequestIsValid(const Ydb::DataStreams::V1::PutRecordsRequest& putRecordsRequest) { + if (putRecordsRequest.stream_name().empty()) { + return "Stream name is empty"; + } + if (putRecordsRequest.records_size() > 500) { + return TStringBuilder() << "Too many records in a single PutRecords request: " << putRecordsRequest.records_size() << " > 500"; + } ui64 totalSize = 0; - for (const auto& record : putRecordsRequest.records()) { + for (const auto& record : putRecordsRequest.records()) { totalSize += record.partition_key().size() + record.data().size(); - if (record.partition_key().empty()) { - return "Empty partition key"; - } - if (record.data().size() > 1048576) { - return TStringBuilder() << "Data of size of " << record.data().size() << " bytes exceed limit of " << 1048576 << " bytes"; - } - if (record.partition_key().size() > 256) { - return TStringBuilder() << "Partition key is too long: " << record.partition_key(); - } - if (!record.explicit_hash_key().empty() && !IsValidDecimal(record.explicit_hash_key())) { - return TStringBuilder() << record.explicit_hash_key() << " is not a valid 128 bit decimal"; - } - } + if (record.partition_key().empty()) { + return "Empty partition key"; + } + if (record.data().size() > 1048576) { + return TStringBuilder() << "Data of size of " << record.data().size() << " bytes exceed limit of " << 1048576 << " bytes"; + } + if (record.partition_key().size() > 256) { + return TStringBuilder() << "Partition key is too long: " << record.partition_key(); + } + if (!record.explicit_hash_key().empty() && !IsValidDecimal(record.explicit_hash_key())) { + return TStringBuilder() << record.explicit_hash_key() << " is not a valid 128 bit decimal"; + } + } if (totalSize > 5 * 1024 * 1024) { return TStringBuilder() << "Total size of PutRecords request of " << totalSize << " bytes exceed limit of " << (5*1024*1024) << " bytes"; } - return ""; - } - } - - template<class TDerived, class TProto> - class TPutRecordsActorBase : public NGRpcProxy::V1::TPQGrpcSchemaBase<TPutRecordsActorBase<TDerived, TProto>, TProto> { - using TBase = NGRpcProxy::V1::TPQGrpcSchemaBase<TPutRecordsActorBase<TDerived, TProto>, TProto>; - - public: - TPutRecordsActorBase(TProto* request, NActors::TActorId newSchemeCache); - ~TPutRecordsActorBase() = default; - - void Bootstrap(const NActors::TActorContext &ctx); - void PreparePartitionActors(const NActors::TActorContext& ctx); + return ""; + } + } + + template<class TDerived, class TProto> + class TPutRecordsActorBase : public NGRpcProxy::V1::TPQGrpcSchemaBase<TPutRecordsActorBase<TDerived, TProto>, TProto> { + using TBase = NGRpcProxy::V1::TPQGrpcSchemaBase<TPutRecordsActorBase<TDerived, TProto>, TProto>; + + public: + TPutRecordsActorBase(TProto* request, NActors::TActorId newSchemeCache); + ~TPutRecordsActorBase() = default; + + void Bootstrap(const NActors::TActorContext &ctx); + void PreparePartitionActors(const NActors::TActorContext& ctx); void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - - protected: - void AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index); - - private: - struct TPartitionTask { - NActors::TActorId ActorId; - std::vector<ui32> RecordIndexes; - }; - - THashMap<ui32, TPartitionTask> PartitionToActor; - NActors::TActorId NewSchemeCache; - Ydb::DataStreams::V1::PutRecordsResult PutRecordsResult; - + + protected: + void AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index); + + private: + struct TPartitionTask { + NActors::TActorId ActorId; + std::vector<ui32> RecordIndexes; + }; + + THashMap<ui32, TPartitionTask> PartitionToActor; + NActors::TActorId NewSchemeCache; + Ydb::DataStreams::V1::PutRecordsResult PutRecordsResult; + TString Ip; - void SendNavigateRequest(const TActorContext &ctx); - - void Handle(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); - - void CheckFinish(const TActorContext& ctx); - - STFUNC(StateFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult, Handle); + void SendNavigateRequest(const TActorContext &ctx); + + void Handle(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); + + void CheckFinish(const TActorContext& ctx); + + STFUNC(StateFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult, Handle); default: TBase::StateWork(ev, ctx); - }; - } - }; - - template<class TDerived, class TProto> - TPutRecordsActorBase<TDerived, TProto>::TPutRecordsActorBase(TProto* request, NActors::TActorId newSchemeCache) - : TBase(request, request->GetProtoRequest()->stream_name()) - , NewSchemeCache(std::move(newSchemeCache)) + }; + } + }; + + template<class TDerived, class TProto> + TPutRecordsActorBase<TDerived, TProto>::TPutRecordsActorBase(TProto* request, NActors::TActorId newSchemeCache) + : TBase(request, request->GetProtoRequest()->stream_name()) + , NewSchemeCache(std::move(newSchemeCache)) , Ip(request->GetPeerName()) - { - Y_ENSURE(request); - } - - template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::Bootstrap(const NActors::TActorContext& ctx) { - TString error = CheckRequestIsValid(static_cast<TDerived*>(this)->GetPutRecordsRequest()); - - if (!error.empty()) { + { + Y_ENSURE(request); + } + + template<class TDerived, class TProto> + void TPutRecordsActorBase<TDerived, TProto>::Bootstrap(const NActors::TActorContext& ctx) { + TString error = CheckRequestIsValid(static_cast<TDerived*>(this)->GetPutRecordsRequest()); + + if (!error.empty()) { return this->ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - - if (this->Request_->GetInternalToken().empty()) { - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - "Request without authentication are not allowed", ctx); - } - } - - SendNavigateRequest(ctx); - this->Become(&TPutRecordsActorBase<TDerived, TProto>::StateFunc); - } - - template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::SendNavigateRequest(const TActorContext& ctx) { - auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = NKikimr::SplitPath(this->GetTopicPath(ctx)); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; - entry.SyncVersion = true; - schemeCacheRequest->ResultSet.emplace_back(entry); - ctx.Send(NewSchemeCache, MakeHolder<TEvTxProxySchemeCache::TEvNavigateKeySet>(schemeCacheRequest.release())); - } - - template<class TDerived, class TProto> + } + + if (this->Request_->GetInternalToken().empty()) { + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + "Request without authentication are not allowed", ctx); + } + } + + SendNavigateRequest(ctx); + this->Become(&TPutRecordsActorBase<TDerived, TProto>::StateFunc); + } + + template<class TDerived, class TProto> + void TPutRecordsActorBase<TDerived, TProto>::SendNavigateRequest(const TActorContext& ctx) { + auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = NKikimr::SplitPath(this->GetTopicPath(ctx)); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; + entry.SyncVersion = true; + schemeCacheRequest->ResultSet.emplace_back(entry); + ctx.Send(NewSchemeCache, MakeHolder<TEvTxProxySchemeCache::TEvNavigateKeySet>(schemeCacheRequest.release())); + } + + template<class TDerived, class TProto> void TPutRecordsActorBase<TDerived, TProto>::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { if (TBase::ReplyIfNotTopic(ev, ctx)) { return; } - const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); - auto topicInfo = navigate->ResultSet.begin(); - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::UpdateRow, - this->Request_->GetInternalToken())) { - return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - TStringBuilder() << "Access for stream " + const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); + auto topicInfo = navigate->ResultSet.begin(); + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::UpdateRow, + this->Request_->GetInternalToken())) { + return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + TStringBuilder() << "Access for stream " << this->GetProtoRequest()->stream_name() << " is denied for subject " << this->Request_->GetInternalToken(), ctx); - } - } - - THashMap<ui32, TVector<TPutRecordsItem>> items; - const auto& pqDescription = topicInfo->PQGroupInfo->Description; - ui32 totalShardsCount = pqDescription.GetPartitions().size(); - for (int i = 0; i < static_cast<TDerived*>(this)->GetPutRecordsRequest().records_size(); ++i) { - PutRecordsResult.add_records(); - AddRecord(items, totalShardsCount, i); - } - + } + } + + THashMap<ui32, TVector<TPutRecordsItem>> items; + const auto& pqDescription = topicInfo->PQGroupInfo->Description; + ui32 totalShardsCount = pqDescription.GetPartitions().size(); + for (int i = 0; i < static_cast<TDerived*>(this)->GetPutRecordsRequest().records_size(); ++i) { + PutRecordsResult.add_records(); + AddRecord(items, totalShardsCount, i); + } + for (auto& partition : pqDescription.GetPartitions()) { auto part = partition.GetPartitionId(); if (items[part].empty()) continue; PartitionToActor[part].ActorId = ctx.Register( new TDatastreamsPartitionActor(ctx.SelfID, partition.GetTabletId(), part, this->GetTopicPath(ctx), std::move(items[part])) ); - } - this->CheckFinish(ctx); - } - - template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::CheckFinish(const TActorContext& ctx) { - if (PartitionToActor.size() == 0) { - static_cast<TDerived*>(this)->SendResult(PutRecordsResult, ctx); - } - } - + } + this->CheckFinish(ctx); + } + + template<class TDerived, class TProto> + void TPutRecordsActorBase<TDerived, TProto>::CheckFinish(const TActorContext& ctx) { + if (PartitionToActor.size() == 0) { + static_cast<TDerived*>(this)->SendResult(PutRecordsResult, ctx); + } + } + TString GetErrorText(const NPersQueue::NErrorCode::EErrorCode errorCode) { if (errorCode == NPersQueue::NErrorCode::OVERLOAD) return "ProvisionedThroughputExceededException"; @@ -339,109 +339,109 @@ namespace NKikimr::NDataStreams::V1 { //TODO: other codes https://docs.aws.amazon.com/kinesis/latest/APIReference/CommonErrors.html } - template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::Handle(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult::TPtr& ev, const TActorContext& ctx) { - auto it = PartitionToActor.find(ev->Get()->PartitionId); - Y_ENSURE(it != PartitionToActor.end()); - if (ev->Get()->ErrorText.Defined()) { - PutRecordsResult.set_failed_record_count( - PutRecordsResult.failed_record_count() + it->second.RecordIndexes.size()); - } + template<class TDerived, class TProto> + void TPutRecordsActorBase<TDerived, TProto>::Handle(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult::TPtr& ev, const TActorContext& ctx) { + auto it = PartitionToActor.find(ev->Get()->PartitionId); + Y_ENSURE(it != PartitionToActor.end()); + if (ev->Get()->ErrorText.Defined()) { + PutRecordsResult.set_failed_record_count( + PutRecordsResult.failed_record_count() + it->second.RecordIndexes.size()); + } PutRecordsResult.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE); - for (ui32 i = 0; i < it->second.RecordIndexes.size(); ++i) { - ui32 index = it->second.RecordIndexes[i]; - auto& record = *PutRecordsResult.mutable_records(index); - if (ev->Get()->ErrorText.Defined()) { - record.set_error_message(*ev->Get()->ErrorText); + for (ui32 i = 0; i < it->second.RecordIndexes.size(); ++i) { + ui32 index = it->second.RecordIndexes[i]; + auto& record = *PutRecordsResult.mutable_records(index); + if (ev->Get()->ErrorText.Defined()) { + record.set_error_message(*ev->Get()->ErrorText); record.set_error_code(GetErrorText(*(ev->Get()->ErrorCode))); //TODO: Throttling exception sometimes - } else { + } else { record.set_shard_id(GetShardName(it->first)); - record.set_sequence_number(Sprintf("%lu", ev->Get()->CurrentOffset + i)); - } - } - - PartitionToActor.erase(it); - CheckFinish(ctx); - } - - template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index) { - const auto& record = static_cast<TDerived*>(this)->GetPutRecordsRequest().records(index); - ui32 shard = 0; - if (record.explicit_hash_key().empty()) { + record.set_sequence_number(Sprintf("%lu", ev->Get()->CurrentOffset + i)); + } + } + + PartitionToActor.erase(it); + CheckFinish(ctx); + } + + template<class TDerived, class TProto> + void TPutRecordsActorBase<TDerived, TProto>::AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index) { + const auto& record = static_cast<TDerived*>(this)->GetPutRecordsRequest().records(index); + ui32 shard = 0; + if (record.explicit_hash_key().empty()) { auto hashKey = HexBytesToDecimal(MD5::Calc(record.partition_key())); - shard = ShardFromDecimal(hashKey, totalShardsCount); - } else { + shard = ShardFromDecimal(hashKey, totalShardsCount); + } else { auto hashKey = BytesToDecimal(record.explicit_hash_key()); shard = ShardFromDecimal(hashKey, totalShardsCount); - } + } items[shard].push_back(TPutRecordsItem{record.data(), record.partition_key(), record.explicit_hash_key(), Ip}); - PartitionToActor[shard].RecordIndexes.push_back(index); - } - - class TPutRecordsActor : public TPutRecordsActorBase<TPutRecordsActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest> { - public: - using TBase = TPutRecordsActorBase<TPutRecordsActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest>; - - TPutRecordsActor(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest* request, TActorId newSchemeCache) - : TBase(request, newSchemeCache) - {} - - const Ydb::DataStreams::V1::PutRecordsRequest& GetPutRecordsRequest() const; - void SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx); - }; - - const Ydb::DataStreams::V1::PutRecordsRequest& TPutRecordsActor::GetPutRecordsRequest() const { - return *GetProtoRequest(); - } - - void TPutRecordsActor::SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx) { - this->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); - } - - //------------------------------------------------------------------------------- - - class TPutRecordActor : public TPutRecordsActorBase<TPutRecordActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest> { - public: - using TBase = TPutRecordsActorBase<TPutRecordActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest>; - - TPutRecordActor(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest* request, TActorId newSchemeCache) - : TBase(request, newSchemeCache) - { - PutRecordsRequest.set_stream_name(request->GetProtoRequest()->stream_name()); - auto& record = *PutRecordsRequest.add_records(); - - record.set_data(request->GetProtoRequest()->data()); - record.set_explicit_hash_key(request->GetProtoRequest()->explicit_hash_key()); - record.set_partition_key(request->GetProtoRequest()->partition_key()); - } - - const Ydb::DataStreams::V1::PutRecordsRequest& GetPutRecordsRequest() const; - void SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx); - - private: - Ydb::DataStreams::V1::PutRecordsRequest PutRecordsRequest; - }; - - const Ydb::DataStreams::V1::PutRecordsRequest& TPutRecordActor::GetPutRecordsRequest() const { - return PutRecordsRequest; - } - - void TPutRecordActor::SendResult(const Ydb::DataStreams::V1::PutRecordsResult& putRecordsResult, const TActorContext& ctx) { - Ydb::DataStreams::V1::PutRecordResult result; - - if (putRecordsResult.failed_record_count() == 0) { - result.set_sequence_number(putRecordsResult.records(0).sequence_number()); + PartitionToActor[shard].RecordIndexes.push_back(index); + } + + class TPutRecordsActor : public TPutRecordsActorBase<TPutRecordsActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest> { + public: + using TBase = TPutRecordsActorBase<TPutRecordsActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest>; + + TPutRecordsActor(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest* request, TActorId newSchemeCache) + : TBase(request, newSchemeCache) + {} + + const Ydb::DataStreams::V1::PutRecordsRequest& GetPutRecordsRequest() const; + void SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx); + }; + + const Ydb::DataStreams::V1::PutRecordsRequest& TPutRecordsActor::GetPutRecordsRequest() const { + return *GetProtoRequest(); + } + + void TPutRecordsActor::SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx) { + this->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); + } + + //------------------------------------------------------------------------------- + + class TPutRecordActor : public TPutRecordsActorBase<TPutRecordActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest> { + public: + using TBase = TPutRecordsActorBase<TPutRecordActor, NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest>; + + TPutRecordActor(NKikimr::NGRpcService::TEvDataStreamsPutRecordRequest* request, TActorId newSchemeCache) + : TBase(request, newSchemeCache) + { + PutRecordsRequest.set_stream_name(request->GetProtoRequest()->stream_name()); + auto& record = *PutRecordsRequest.add_records(); + + record.set_data(request->GetProtoRequest()->data()); + record.set_explicit_hash_key(request->GetProtoRequest()->explicit_hash_key()); + record.set_partition_key(request->GetProtoRequest()->partition_key()); + } + + const Ydb::DataStreams::V1::PutRecordsRequest& GetPutRecordsRequest() const; + void SendResult(const Ydb::DataStreams::V1::PutRecordsResult& result, const TActorContext& ctx); + + private: + Ydb::DataStreams::V1::PutRecordsRequest PutRecordsRequest; + }; + + const Ydb::DataStreams::V1::PutRecordsRequest& TPutRecordActor::GetPutRecordsRequest() const { + return PutRecordsRequest; + } + + void TPutRecordActor::SendResult(const Ydb::DataStreams::V1::PutRecordsResult& putRecordsResult, const TActorContext& ctx) { + Ydb::DataStreams::V1::PutRecordResult result; + + if (putRecordsResult.failed_record_count() == 0) { + result.set_sequence_number(putRecordsResult.records(0).sequence_number()); result.set_shard_id(putRecordsResult.records(0).shard_id()); result.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE); return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); - } else { + } else { if (putRecordsResult.records(0).error_code() == "ProvisionedThroughputExceededException") { return ReplyWithResult(Ydb::StatusIds::OVERLOADED, ctx); } //TODO: other codes - access denied and so on return ReplyWithResult(Ydb::StatusIds::INTERNAL_ERROR, ctx); - } - } - + } + } + } diff --git a/ydb/services/datastreams/ut/ya.make b/ydb/services/datastreams/ut/ya.make index 9f84efa007..4767eeefe7 100644 --- a/ydb/services/datastreams/ut/ya.make +++ b/ydb/services/datastreams/ut/ya.make @@ -1,32 +1,32 @@ UNITTEST_FOR(ydb/services/datastreams) - + OWNER( g:kikimr g:yql ) -FORK_SUBTESTS() - -ENV(PERSQUEUE_NEW_SCHEMECACHE="true") - -SIZE(MEDIUM) - -TIMEOUT(600) - -SRCS( - datastreams_ut.cpp -) - -PEERDIR( - library/cpp/getopt +FORK_SUBTESTS() + +ENV(PERSQUEUE_NEW_SCHEMECACHE="true") + +SIZE(MEDIUM) + +TIMEOUT(600) + +SRCS( + datastreams_ut.cpp +) + +PEERDIR( + library/cpp/getopt library/cpp/grpc/client - library/cpp/svnversion + library/cpp/svnversion ydb/core/testlib ydb/services/datastreams -) - -YQL_LAST_ABI_VERSION() - +) + +YQL_LAST_ABI_VERSION() + REQUIREMENTS(ram:11) -END() +END() diff --git a/ydb/services/datastreams/ya.make b/ydb/services/datastreams/ya.make index 7ccc01eb0c..0a26e96de9 100644 --- a/ydb/services/datastreams/ya.make +++ b/ydb/services/datastreams/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - datastreams_proxy.cpp - grpc_service.cpp +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + datastreams_proxy.cpp + grpc_service.cpp next_token.cpp - put_records_actor.cpp + put_records_actor.cpp shard_iterator.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/grpc/server ydb/core/base ydb/core/client/server @@ -24,9 +24,9 @@ PEERDIR( ydb/services/lib/actors ydb/services/lib/sharding ydb/services/ydb -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp index 63988e5269..f48e398488 100644 --- a/ydb/services/lib/actors/pq_schema_actor.cpp +++ b/ydb/services/lib/actors/pq_schema_actor.cpp @@ -1,25 +1,25 @@ -#include "pq_schema_actor.h" - +#include "pq_schema_actor.h" + #include <ydb/library/persqueue/obfuscate/obfuscate.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> - + #include <ydb/public/lib/jwt/jwt.h> - + #include <ydb/library/yql/public/decimal/yql_decimal.h> - -#include <util/string/vector.h> - -namespace NKikimr::NGRpcProxy::V1 { - + +#include <util/string/vector.h> + +namespace NKikimr::NGRpcProxy::V1 { + #define DEFAULT_PARTITION_SPEED 1048576 // 1Mb - constexpr i32 MAX_READ_RULES_COUNT = 3000; - - constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; - static const i64 DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD = 16*24*60*60*1000; - - constexpr i32 MAX_SUPPORTED_CODECS_COUNT = 100; - + constexpr i32 MAX_READ_RULES_COUNT = 3000; + + constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; + static const i64 DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD = 16*24*60*60*1000; + + constexpr i32 MAX_SUPPORTED_CODECS_COUNT = 100; + TClientServiceTypes GetSupportedClientServiceTypes(const TActorContext& ctx) { TClientServiceTypes serviceTypes; const auto& pqConfig = AppData(ctx)->PQConfig; @@ -35,7 +35,7 @@ namespace NKikimr::NGRpcProxy::V1 { } return serviceTypes; } - + TString ReadRuleServiceTypeMigration(NKikimrPQ::TPQTabletConfig *config, const TActorContext& ctx) { auto rrServiceTypes = config->MutableReadRuleServiceTypes(); if (config->ReadRuleServiceTypesSize() > config->ReadRulesSize()) { @@ -60,7 +60,7 @@ namespace NKikimr::NGRpcProxy::V1 { const TClientServiceTypes& supportedClientServiceTypes, const TActorContext& ctx ) { - auto consumerName = NPersQueue::ConvertNewConsumerName(rr.consumer_name(), ctx); + auto consumerName = NPersQueue::ConvertNewConsumerName(rr.consumer_name(), ctx); if(consumerName.find("/") != TString::npos || consumerName.find("|") != TString::npos) { return TStringBuilder() << "consumer '" << rr.consumer_name() << "' has illegal symbols"; } @@ -71,37 +71,37 @@ namespace NKikimr::NGRpcProxy::V1 { } } - config->AddReadRules(consumerName); - - if (rr.starting_message_timestamp_ms() < 0) { - return TStringBuilder() << "starting_message_timestamp_ms in read_rule can't be negative, provided " << rr.starting_message_timestamp_ms(); - } - config->AddReadFromTimestampsMs(rr.starting_message_timestamp_ms()); - - if (!Ydb::PersQueue::V1::TopicSettings::Format_IsValid((int)rr.supported_format()) || rr.supported_format() == 0) { - return TStringBuilder() << "Unknown format version with value " << (int)rr.supported_format() << " for " << rr.consumer_name(); - } - config->AddConsumerFormatVersions(rr.supported_format() - 1); - - if (rr.version() < 0) { - return TStringBuilder() << "version in read_rule can't be negative, provided " << rr.version(); - } - config->AddReadRuleVersions(rr.version()); - auto ct = config->AddConsumerCodecs(); - if (rr.supported_codecs().size() > MAX_SUPPORTED_CODECS_COUNT) { - return TStringBuilder() << "supported_codecs count cannot be more than " - << MAX_SUPPORTED_CODECS_COUNT << ", provided " << rr.supported_codecs().size(); - } - for (const auto& codec : rr.supported_codecs()) { - if (!Ydb::PersQueue::V1::Codec_IsValid(codec) || codec == 0) - return TStringBuilder() << "Unknown codec with value " << codec << " for " << rr.consumer_name(); - ct->AddIds(codec - 1); - ct->AddCodecs(to_lower(Ydb::PersQueue::V1::Codec_Name((Ydb::PersQueue::V1::Codec)codec)).substr(6)); - } - - if (rr.important()) - config->MutablePartitionConfig()->AddImportantClientId(consumerName); - + config->AddReadRules(consumerName); + + if (rr.starting_message_timestamp_ms() < 0) { + return TStringBuilder() << "starting_message_timestamp_ms in read_rule can't be negative, provided " << rr.starting_message_timestamp_ms(); + } + config->AddReadFromTimestampsMs(rr.starting_message_timestamp_ms()); + + if (!Ydb::PersQueue::V1::TopicSettings::Format_IsValid((int)rr.supported_format()) || rr.supported_format() == 0) { + return TStringBuilder() << "Unknown format version with value " << (int)rr.supported_format() << " for " << rr.consumer_name(); + } + config->AddConsumerFormatVersions(rr.supported_format() - 1); + + if (rr.version() < 0) { + return TStringBuilder() << "version in read_rule can't be negative, provided " << rr.version(); + } + config->AddReadRuleVersions(rr.version()); + auto ct = config->AddConsumerCodecs(); + if (rr.supported_codecs().size() > MAX_SUPPORTED_CODECS_COUNT) { + return TStringBuilder() << "supported_codecs count cannot be more than " + << MAX_SUPPORTED_CODECS_COUNT << ", provided " << rr.supported_codecs().size(); + } + for (const auto& codec : rr.supported_codecs()) { + if (!Ydb::PersQueue::V1::Codec_IsValid(codec) || codec == 0) + return TStringBuilder() << "Unknown codec with value " << codec << " for " << rr.consumer_name(); + ct->AddIds(codec - 1); + ct->AddCodecs(to_lower(Ydb::PersQueue::V1::Codec_Name((Ydb::PersQueue::V1::Codec)codec)).substr(6)); + } + + if (rr.important()) + config->MutablePartitionConfig()->AddImportantClientId(consumerName); + if (!rr.service_type().empty()) { if (!supportedClientServiceTypes.contains(rr.service_type())) { return TStringBuilder() << "Unknown read rule service type '" << rr.service_type() @@ -116,9 +116,9 @@ namespace NKikimr::NGRpcProxy::V1 { const auto& defaultCientServiceType = pqConfig.GetDefaultClientServiceType().GetName(); config->AddReadRuleServiceTypes(defaultCientServiceType); } - return ""; - } - + return ""; + } + TString RemoveReadRuleFromConfig( NKikimrPQ::TPQTabletConfig* config, const NKikimrPQ::TPQTabletConfig& originalConfig, @@ -185,14 +185,14 @@ namespace NKikimr::NGRpcProxy::V1 { return false; } - THashSet<TString> readRuleConsumers; - for (auto consumerName : config.GetReadRules()) { - if (readRuleConsumers.find(consumerName) != readRuleConsumers.end()) { - error = TStringBuilder() << "Duplicate consumer name " << consumerName; - return true; - } - readRuleConsumers.insert(consumerName); - } + THashSet<TString> readRuleConsumers; + for (auto consumerName : config.GetReadRules()) { + if (readRuleConsumers.find(consumerName) != readRuleConsumers.end()) { + error = TStringBuilder() << "Duplicate consumer name " << consumerName; + return true; + } + readRuleConsumers.insert(consumerName); + } for (const auto& t : supportedClientServiceTypes) { @@ -208,64 +208,64 @@ namespace NKikimr::NGRpcProxy::V1 { } } - return false; - } - - NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode) { - NYql::TIssue res(NYql::TPosition(), errorReason); - res.SetCode(errorCode, NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR); - - return res; - } - - Ydb::StatusIds::StatusCode FillProposeRequestImpl(const TString& name, const Ydb::PersQueue::V1::TopicSettings& settings, + return false; + } + + NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode) { + NYql::TIssue res(NYql::TPosition(), errorReason); + res.SetCode(errorCode, NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR); + + return res; + } + + Ydb::StatusIds::StatusCode FillProposeRequestImpl(const TString& name, const Ydb::PersQueue::V1::TopicSettings& settings, NKikimrSchemeOp::TModifyScheme& modifyScheme, const TActorContext& ctx, bool alter, TString& error) - { + { modifyScheme.SetOperationType(alter ? NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup : NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup); - - auto pqDescr = alter ? modifyScheme.MutableAlterPersQueueGroup() : modifyScheme.MutableCreatePersQueueGroup(); - pqDescr->SetName(name); - if (settings.partitions_count() <= 0) { - error = TStringBuilder() << "Partitions count must be positive, provided " << settings.partitions_count(); - return Ydb::StatusIds::BAD_REQUEST; - } - - pqDescr->SetTotalGroupCount(settings.partitions_count()); - - auto config = pqDescr->MutablePQTabletConfig(); - - config->SetRequireAuthWrite(true); - config->SetRequireAuthRead(true); - if (!alter) - pqDescr->SetPartitionPerTablet(2); - - for (auto& pair : settings.attributes()) { - if (pair.first == "_partitions_per_tablet") { - try { - if (!alter) - pqDescr->SetPartitionPerTablet(FromString<ui32>(pair.second)); - if (pqDescr->GetPartitionPerTablet() > 20) { - error = TStringBuilder() << "Attirbute partitons_per_tablet is " << pair.second << ", which is greater than 20"; - return Ydb::StatusIds::BAD_REQUEST; - } - } catch(...) { - error = TStringBuilder() << "Attirbute partitons_per_tablet is " << pair.second << ", which is not ui32"; - return Ydb::StatusIds::BAD_REQUEST; - } - } else if (pair.first == "_allow_unauthenticated_read") { - try { - config->SetRequireAuthRead(!FromString<bool>(pair.second)); - } catch(...) { - error = TStringBuilder() << "Attirbute allow_unauthenticated_read is " << pair.second << ", which is not bool"; - return Ydb::StatusIds::BAD_REQUEST; - } - } else if (pair.first == "_allow_unauthenticated_write") { - try { - config->SetRequireAuthWrite(!FromString<bool>(pair.second)); - } catch(...) { - error = TStringBuilder() << "Attirbute allow_unauthenticated_write is " << pair.second << ", which is not bool"; - return Ydb::StatusIds::BAD_REQUEST; - } + + auto pqDescr = alter ? modifyScheme.MutableAlterPersQueueGroup() : modifyScheme.MutableCreatePersQueueGroup(); + pqDescr->SetName(name); + if (settings.partitions_count() <= 0) { + error = TStringBuilder() << "Partitions count must be positive, provided " << settings.partitions_count(); + return Ydb::StatusIds::BAD_REQUEST; + } + + pqDescr->SetTotalGroupCount(settings.partitions_count()); + + auto config = pqDescr->MutablePQTabletConfig(); + + config->SetRequireAuthWrite(true); + config->SetRequireAuthRead(true); + if (!alter) + pqDescr->SetPartitionPerTablet(2); + + for (auto& pair : settings.attributes()) { + if (pair.first == "_partitions_per_tablet") { + try { + if (!alter) + pqDescr->SetPartitionPerTablet(FromString<ui32>(pair.second)); + if (pqDescr->GetPartitionPerTablet() > 20) { + error = TStringBuilder() << "Attirbute partitons_per_tablet is " << pair.second << ", which is greater than 20"; + return Ydb::StatusIds::BAD_REQUEST; + } + } catch(...) { + error = TStringBuilder() << "Attirbute partitons_per_tablet is " << pair.second << ", which is not ui32"; + return Ydb::StatusIds::BAD_REQUEST; + } + } else if (pair.first == "_allow_unauthenticated_read") { + try { + config->SetRequireAuthRead(!FromString<bool>(pair.second)); + } catch(...) { + error = TStringBuilder() << "Attirbute allow_unauthenticated_read is " << pair.second << ", which is not bool"; + return Ydb::StatusIds::BAD_REQUEST; + } + } else if (pair.first == "_allow_unauthenticated_write") { + try { + config->SetRequireAuthWrite(!FromString<bool>(pair.second)); + } catch(...) { + error = TStringBuilder() << "Attirbute allow_unauthenticated_write is " << pair.second << ", which is not bool"; + return Ydb::StatusIds::BAD_REQUEST; + } } else if (pair.first == "_abc_slug") { config->SetAbcSlug(pair.second); } else if (pair.first == "_abc_id") { @@ -275,118 +275,118 @@ namespace NKikimr::NGRpcProxy::V1 { error = TStringBuilder() << "Attirbute abc_id is " << pair.second << ", which is not integer"; return Ydb::StatusIds::BAD_REQUEST; } - } else { - error = TStringBuilder() << "Attirbute " << pair.first << " is not supported"; - return Ydb::StatusIds::BAD_REQUEST; - } - } - bool local = !settings.client_write_disabled(); - config->SetLocalDC(local); - config->SetDC(NPersQueue::GetDC(name)); - config->SetProducer(NPersQueue::GetProducer(name)); + } else { + error = TStringBuilder() << "Attirbute " << pair.first << " is not supported"; + return Ydb::StatusIds::BAD_REQUEST; + } + } + bool local = !settings.client_write_disabled(); + config->SetLocalDC(local); + config->SetDC(NPersQueue::GetDC(name)); + config->SetProducer(NPersQueue::GetProducer(name)); config->SetTopic(LegacySubstr(NPersQueue::GetRealTopic(name), config->GetProducer().size() + 2)); - config->SetIdent(config->GetProducer()); - auto partConfig = config->MutablePartitionConfig(); - - const auto& channelProfiles = AppData(ctx)->PQConfig.GetChannelProfiles(); - if (channelProfiles.size() > 2) { - partConfig->SetNumChannels(channelProfiles.size() - 2); // channels 0,1 are reserved in tablet - partConfig->MutableExplicitChannelProfiles()->CopyFrom(channelProfiles); - } - if (settings.max_partition_storage_size() < 0) { - error = TStringBuilder() << "Max_partiton_strorage_size must can't be negative, provided " << settings.max_partition_storage_size(); - return Ydb::StatusIds::BAD_REQUEST; - } - partConfig->SetMaxSizeInPartition(settings.max_partition_storage_size() ? settings.max_partition_storage_size() : Max<i64>()); - partConfig->SetMaxCountInPartition(Max<i32>()); - - if (settings.retention_period_ms() <= 0) { - error = TStringBuilder() << "retention_period_ms must be positive, provided " << settings.retention_period_ms(); - return Ydb::StatusIds::BAD_REQUEST; - } - partConfig->SetLifetimeSeconds(settings.retention_period_ms() > 999 ? settings.retention_period_ms() / 1000 : 1); - - if (settings.message_group_seqno_retention_period_ms() > 0 && settings.message_group_seqno_retention_period_ms() < settings.retention_period_ms()) { - error = TStringBuilder() << "message_group_seqno_retention_period_ms (provided " << settings.message_group_seqno_retention_period_ms() << ") must be more then retention_period_ms (provided " << settings.retention_period_ms() << ")"; - return Ydb::StatusIds::BAD_REQUEST; - } - if (settings.message_group_seqno_retention_period_ms() > DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD) { - error = TStringBuilder() << "message_group_seqno_retention_period_ms (provided " << settings.message_group_seqno_retention_period_ms() << ") must be less then default limit for database " << DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD; - return Ydb::StatusIds::BAD_REQUEST; - } - if (settings.message_group_seqno_retention_period_ms() < 0) { - error = TStringBuilder() << "message_group_seqno_retention_period_ms can't be negative, provided " << settings.message_group_seqno_retention_period_ms(); - return Ydb::StatusIds::BAD_REQUEST; - } - if (settings.message_group_seqno_retention_period_ms() > 0) { - partConfig->SetSourceIdLifetimeSeconds(settings.message_group_seqno_retention_period_ms() > 999 ? settings.message_group_seqno_retention_period_ms() / 1000 :1); - } else { - // default value - partConfig->SetSourceIdLifetimeSeconds(NKikimrPQ::TPartitionConfig().GetSourceIdLifetimeSeconds()); - } - - if (settings.max_partition_message_groups_seqno_stored() < 0) { - error = TStringBuilder() << "max_partition_message_groups_seqno_stored can't be negative, provided " << settings.max_partition_message_groups_seqno_stored(); - return Ydb::StatusIds::BAD_REQUEST; - } - if (settings.max_partition_message_groups_seqno_stored() > 0) { - partConfig->SetSourceIdMaxCounts(settings.max_partition_message_groups_seqno_stored()); - } else { - // default value - partConfig->SetSourceIdMaxCounts(NKikimrPQ::TPartitionConfig().GetSourceIdMaxCounts()); - } - - if (local) { + config->SetIdent(config->GetProducer()); + auto partConfig = config->MutablePartitionConfig(); + + const auto& channelProfiles = AppData(ctx)->PQConfig.GetChannelProfiles(); + if (channelProfiles.size() > 2) { + partConfig->SetNumChannels(channelProfiles.size() - 2); // channels 0,1 are reserved in tablet + partConfig->MutableExplicitChannelProfiles()->CopyFrom(channelProfiles); + } + if (settings.max_partition_storage_size() < 0) { + error = TStringBuilder() << "Max_partiton_strorage_size must can't be negative, provided " << settings.max_partition_storage_size(); + return Ydb::StatusIds::BAD_REQUEST; + } + partConfig->SetMaxSizeInPartition(settings.max_partition_storage_size() ? settings.max_partition_storage_size() : Max<i64>()); + partConfig->SetMaxCountInPartition(Max<i32>()); + + if (settings.retention_period_ms() <= 0) { + error = TStringBuilder() << "retention_period_ms must be positive, provided " << settings.retention_period_ms(); + return Ydb::StatusIds::BAD_REQUEST; + } + partConfig->SetLifetimeSeconds(settings.retention_period_ms() > 999 ? settings.retention_period_ms() / 1000 : 1); + + if (settings.message_group_seqno_retention_period_ms() > 0 && settings.message_group_seqno_retention_period_ms() < settings.retention_period_ms()) { + error = TStringBuilder() << "message_group_seqno_retention_period_ms (provided " << settings.message_group_seqno_retention_period_ms() << ") must be more then retention_period_ms (provided " << settings.retention_period_ms() << ")"; + return Ydb::StatusIds::BAD_REQUEST; + } + if (settings.message_group_seqno_retention_period_ms() > DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD) { + error = TStringBuilder() << "message_group_seqno_retention_period_ms (provided " << settings.message_group_seqno_retention_period_ms() << ") must be less then default limit for database " << DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD; + return Ydb::StatusIds::BAD_REQUEST; + } + if (settings.message_group_seqno_retention_period_ms() < 0) { + error = TStringBuilder() << "message_group_seqno_retention_period_ms can't be negative, provided " << settings.message_group_seqno_retention_period_ms(); + return Ydb::StatusIds::BAD_REQUEST; + } + if (settings.message_group_seqno_retention_period_ms() > 0) { + partConfig->SetSourceIdLifetimeSeconds(settings.message_group_seqno_retention_period_ms() > 999 ? settings.message_group_seqno_retention_period_ms() / 1000 :1); + } else { + // default value + partConfig->SetSourceIdLifetimeSeconds(NKikimrPQ::TPartitionConfig().GetSourceIdLifetimeSeconds()); + } + + if (settings.max_partition_message_groups_seqno_stored() < 0) { + error = TStringBuilder() << "max_partition_message_groups_seqno_stored can't be negative, provided " << settings.max_partition_message_groups_seqno_stored(); + return Ydb::StatusIds::BAD_REQUEST; + } + if (settings.max_partition_message_groups_seqno_stored() > 0) { + partConfig->SetSourceIdMaxCounts(settings.max_partition_message_groups_seqno_stored()); + } else { + // default value + partConfig->SetSourceIdMaxCounts(NKikimrPQ::TPartitionConfig().GetSourceIdMaxCounts()); + } + + if (local) { auto partSpeed = settings.max_partition_write_speed(); if (partSpeed < 0) { error = TStringBuilder() << "max_partition_write_speed can't be negative, provided " << partSpeed; - return Ydb::StatusIds::BAD_REQUEST; + return Ydb::StatusIds::BAD_REQUEST; } else if (partSpeed == 0) { partSpeed = DEFAULT_PARTITION_SPEED; - } + } partConfig->SetWriteSpeedInBytesPerSecond(partSpeed); - + const auto& burstSpeed = settings.max_partition_write_burst(); if (burstSpeed < 0) { error = TStringBuilder() << "max_partition_write_burst can't be negative, provided " << burstSpeed; - return Ydb::StatusIds::BAD_REQUEST; + return Ydb::StatusIds::BAD_REQUEST; } else if (burstSpeed == 0) { partConfig->SetBurstSize(partSpeed); } else { partConfig->SetBurstSize(burstSpeed); - } - } - - if (!Ydb::PersQueue::V1::TopicSettings::Format_IsValid((int)settings.supported_format()) || settings.supported_format() == 0) { - error = TStringBuilder() << "Unknown format version with value " << (int)settings.supported_format(); - return Ydb::StatusIds::BAD_REQUEST; - } - config->SetFormatVersion(settings.supported_format() - 1); - - auto ct = config->MutableCodecs(); - if (settings.supported_codecs().size() > MAX_SUPPORTED_CODECS_COUNT) { - error = TStringBuilder() << "supported_codecs count cannot be more than " - << MAX_SUPPORTED_CODECS_COUNT << ", provided " << settings.supported_codecs().size(); - return Ydb::StatusIds::BAD_REQUEST; - - } - for(const auto& codec : settings.supported_codecs()) { - if (!Ydb::PersQueue::V1::Codec_IsValid(codec) || codec == 0) { - error = TStringBuilder() << "Unknown codec with value " << codec; - return Ydb::StatusIds::BAD_REQUEST; - } - ct->AddIds(codec - 1); + } + } + + if (!Ydb::PersQueue::V1::TopicSettings::Format_IsValid((int)settings.supported_format()) || settings.supported_format() == 0) { + error = TStringBuilder() << "Unknown format version with value " << (int)settings.supported_format(); + return Ydb::StatusIds::BAD_REQUEST; + } + config->SetFormatVersion(settings.supported_format() - 1); + + auto ct = config->MutableCodecs(); + if (settings.supported_codecs().size() > MAX_SUPPORTED_CODECS_COUNT) { + error = TStringBuilder() << "supported_codecs count cannot be more than " + << MAX_SUPPORTED_CODECS_COUNT << ", provided " << settings.supported_codecs().size(); + return Ydb::StatusIds::BAD_REQUEST; + + } + for(const auto& codec : settings.supported_codecs()) { + if (!Ydb::PersQueue::V1::Codec_IsValid(codec) || codec == 0) { + error = TStringBuilder() << "Unknown codec with value " << codec; + return Ydb::StatusIds::BAD_REQUEST; + } + ct->AddIds(codec - 1); ct->AddCodecs(LegacySubstr(to_lower(Ydb::PersQueue::V1::Codec_Name((Ydb::PersQueue::V1::Codec)codec)), 6)); - } - - //TODO: check all values with defaults - - if (settings.read_rules().size() > MAX_READ_RULES_COUNT) { - error = TStringBuilder() << "read rules count cannot be more than " - << MAX_SUPPORTED_CODECS_COUNT << ", provided " << settings.read_rules().size(); - return Ydb::StatusIds::BAD_REQUEST; - } - + } + + //TODO: check all values with defaults + + if (settings.read_rules().size() > MAX_READ_RULES_COUNT) { + error = TStringBuilder() << "read rules count cannot be more than " + << MAX_SUPPORTED_CODECS_COUNT << ", provided " << settings.read_rules().size(); + return Ydb::StatusIds::BAD_REQUEST; + } + { error = ReadRuleServiceTypeMigration(config, ctx); if (error) { @@ -394,95 +394,95 @@ namespace NKikimr::NGRpcProxy::V1 { } } const auto& supportedClientServiceTypes = GetSupportedClientServiceTypes(ctx); - for (const auto& rr : settings.read_rules()) { + for (const auto& rr : settings.read_rules()) { error = AddReadRuleToConfig(config, rr, supportedClientServiceTypes, ctx); - if (!error.Empty()) { - return Ydb::StatusIds::BAD_REQUEST; - } - } - if (settings.has_remote_mirror_rule()) { - auto mirrorFrom = partConfig->MutableMirrorFrom(); - { - TString endpoint = settings.remote_mirror_rule().endpoint(); - if (endpoint.StartsWith(GRPCS_ENDPOINT_PREFIX)) { - mirrorFrom->SetUseSecureConnection(true); - endpoint = TString(endpoint.begin() + GRPCS_ENDPOINT_PREFIX.size(), endpoint.end()); - } - auto parts = SplitString(endpoint, ":"); - if (parts.size() != 2) { - error = TStringBuilder() << "endpoint in remote mirror rule must be in format [grpcs://]server:port, but got '" - << settings.remote_mirror_rule().endpoint() << "'"; - return Ydb::StatusIds::BAD_REQUEST; - } - ui16 port; - if (!TryFromString(parts[1], port)) { - error = TStringBuilder() << "cannot parse port from endpoint ('" << settings.remote_mirror_rule().endpoint() << "') for remote mirror rule"; - return Ydb::StatusIds::BAD_REQUEST; - } - mirrorFrom->SetEndpoint(parts[0]); - mirrorFrom->SetEndpointPort(port); - } - mirrorFrom->SetTopic(settings.remote_mirror_rule().topic_path()); - mirrorFrom->SetConsumer(settings.remote_mirror_rule().consumer_name()); - if (settings.remote_mirror_rule().starting_message_timestamp_ms() < 0) { - error = TStringBuilder() << "starting_message_timestamp_ms in remote_mirror_rule can't be negative, provided " - << settings.remote_mirror_rule().starting_message_timestamp_ms(); - return Ydb::StatusIds::BAD_REQUEST; - } - mirrorFrom->SetReadFromTimestampsMs(settings.remote_mirror_rule().starting_message_timestamp_ms()); - if (!settings.remote_mirror_rule().has_credentials()) { - error = "credentials for remote mirror rule must be set"; - return Ydb::StatusIds::BAD_REQUEST; - } - const auto& credentials = settings.remote_mirror_rule().credentials(); - switch (credentials.credentials_case()) { - case Ydb::PersQueue::V1::Credentials::kOauthToken: { - mirrorFrom->MutableCredentials()->SetOauthToken(credentials.oauth_token()); - break; - } - case Ydb::PersQueue::V1::Credentials::kJwtParams: { - try { - auto res = NYdb::ParseJwtParams(credentials.jwt_params()); - NYdb::MakeSignedJwt(res); - } catch (...) { - error = TStringBuilder() << "incorrect jwt params in remote mirror rule: " << CurrentExceptionMessage(); - return Ydb::StatusIds::BAD_REQUEST; - } - mirrorFrom->MutableCredentials()->SetJwtParams(credentials.jwt_params()); - break; - } - case Ydb::PersQueue::V1::Credentials::kIam: { - try { - auto res = NYdb::ParseJwtParams(credentials.iam().service_account_key()); - NYdb::MakeSignedJwt(res); - } catch (...) { - error = TStringBuilder() << "incorrect service account key for iam in remote mirror rule: " << CurrentExceptionMessage(); - return Ydb::StatusIds::BAD_REQUEST; - } - if (credentials.iam().endpoint().empty()) { - error = "iam endpoint must be set in remote mirror rule"; - return Ydb::StatusIds::BAD_REQUEST; - } - mirrorFrom->MutableCredentials()->MutableIam()->SetEndpoint(credentials.iam().endpoint()); - mirrorFrom->MutableCredentials()->MutableIam()->SetServiceAccountKey(credentials.iam().service_account_key()); - break; - } - case Ydb::PersQueue::V1::Credentials::CREDENTIALS_NOT_SET: { - error = "one of the credential fields must be filled for remote mirror rule"; - return Ydb::StatusIds::BAD_REQUEST; - } - default: { - error = TStringBuilder() << "unsupported credentials type " << ::NPersQueue::ObfuscateString(ToString(credentials)); - return Ydb::StatusIds::BAD_REQUEST; - } - } - if (settings.remote_mirror_rule().database()) { - mirrorFrom->SetDatabase(settings.remote_mirror_rule().database()); - } - } - + if (!error.Empty()) { + return Ydb::StatusIds::BAD_REQUEST; + } + } + if (settings.has_remote_mirror_rule()) { + auto mirrorFrom = partConfig->MutableMirrorFrom(); + { + TString endpoint = settings.remote_mirror_rule().endpoint(); + if (endpoint.StartsWith(GRPCS_ENDPOINT_PREFIX)) { + mirrorFrom->SetUseSecureConnection(true); + endpoint = TString(endpoint.begin() + GRPCS_ENDPOINT_PREFIX.size(), endpoint.end()); + } + auto parts = SplitString(endpoint, ":"); + if (parts.size() != 2) { + error = TStringBuilder() << "endpoint in remote mirror rule must be in format [grpcs://]server:port, but got '" + << settings.remote_mirror_rule().endpoint() << "'"; + return Ydb::StatusIds::BAD_REQUEST; + } + ui16 port; + if (!TryFromString(parts[1], port)) { + error = TStringBuilder() << "cannot parse port from endpoint ('" << settings.remote_mirror_rule().endpoint() << "') for remote mirror rule"; + return Ydb::StatusIds::BAD_REQUEST; + } + mirrorFrom->SetEndpoint(parts[0]); + mirrorFrom->SetEndpointPort(port); + } + mirrorFrom->SetTopic(settings.remote_mirror_rule().topic_path()); + mirrorFrom->SetConsumer(settings.remote_mirror_rule().consumer_name()); + if (settings.remote_mirror_rule().starting_message_timestamp_ms() < 0) { + error = TStringBuilder() << "starting_message_timestamp_ms in remote_mirror_rule can't be negative, provided " + << settings.remote_mirror_rule().starting_message_timestamp_ms(); + return Ydb::StatusIds::BAD_REQUEST; + } + mirrorFrom->SetReadFromTimestampsMs(settings.remote_mirror_rule().starting_message_timestamp_ms()); + if (!settings.remote_mirror_rule().has_credentials()) { + error = "credentials for remote mirror rule must be set"; + return Ydb::StatusIds::BAD_REQUEST; + } + const auto& credentials = settings.remote_mirror_rule().credentials(); + switch (credentials.credentials_case()) { + case Ydb::PersQueue::V1::Credentials::kOauthToken: { + mirrorFrom->MutableCredentials()->SetOauthToken(credentials.oauth_token()); + break; + } + case Ydb::PersQueue::V1::Credentials::kJwtParams: { + try { + auto res = NYdb::ParseJwtParams(credentials.jwt_params()); + NYdb::MakeSignedJwt(res); + } catch (...) { + error = TStringBuilder() << "incorrect jwt params in remote mirror rule: " << CurrentExceptionMessage(); + return Ydb::StatusIds::BAD_REQUEST; + } + mirrorFrom->MutableCredentials()->SetJwtParams(credentials.jwt_params()); + break; + } + case Ydb::PersQueue::V1::Credentials::kIam: { + try { + auto res = NYdb::ParseJwtParams(credentials.iam().service_account_key()); + NYdb::MakeSignedJwt(res); + } catch (...) { + error = TStringBuilder() << "incorrect service account key for iam in remote mirror rule: " << CurrentExceptionMessage(); + return Ydb::StatusIds::BAD_REQUEST; + } + if (credentials.iam().endpoint().empty()) { + error = "iam endpoint must be set in remote mirror rule"; + return Ydb::StatusIds::BAD_REQUEST; + } + mirrorFrom->MutableCredentials()->MutableIam()->SetEndpoint(credentials.iam().endpoint()); + mirrorFrom->MutableCredentials()->MutableIam()->SetServiceAccountKey(credentials.iam().service_account_key()); + break; + } + case Ydb::PersQueue::V1::Credentials::CREDENTIALS_NOT_SET: { + error = "one of the credential fields must be filled for remote mirror rule"; + return Ydb::StatusIds::BAD_REQUEST; + } + default: { + error = TStringBuilder() << "unsupported credentials type " << ::NPersQueue::ObfuscateString(ToString(credentials)); + return Ydb::StatusIds::BAD_REQUEST; + } + } + if (settings.remote_mirror_rule().database()) { + mirrorFrom->SetDatabase(settings.remote_mirror_rule().database()); + } + } + CheckReadRulesConfig(*config, supportedClientServiceTypes, error); return error.empty() ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::BAD_REQUEST; - } - + } + } diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h index 7ccd181d7a..1be2e7fa24 100644 --- a/ydb/services/lib/actors/pq_schema_actor.h +++ b/ydb/services/lib/actors/pq_schema_actor.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <ydb/core/grpc_services/rpc_scheme_base.h> - + #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr::NGRpcProxy::V1 { - + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/hfunc.h> + +namespace NKikimr::NGRpcProxy::V1 { + Ydb::StatusIds::StatusCode FillProposeRequestImpl( const TString& name, const Ydb::PersQueue::V1::TopicSettings& settings, @@ -43,68 +43,68 @@ namespace NKikimr::NGRpcProxy::V1 { const TString& consumerName, const TActorContext& ctx ); - NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode); - - - template<class TDerived, class TRequest> - class TPQGrpcSchemaBase : public NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest> { + NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode); + + + template<class TDerived, class TRequest> + class TPQGrpcSchemaBase : public NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest> { protected: - using TBase = NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest>; - - public: - TPQGrpcSchemaBase(TRequest *request, const TString& topicPath) - : TBase(request) - , TopicPath(topicPath) - { - } - + using TBase = NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest>; + + public: + TPQGrpcSchemaBase(TRequest *request, const TString& topicPath) + : TBase(request) + , TopicPath(topicPath) + { + } + void PrepareTopicPath(const NActors::TActorContext &ctx) { // ToDo !! - TopicPath = NPersQueue::GetFullTopicPath(ctx, this->Request_->GetDatabaseName(), TopicPath); - } - - TString GetTopicPath(const NActors::TActorContext& ctx) { - PrepareTopicPath(ctx); - return TopicPath; - } - - protected: - // TDerived must implement FillProposeRequest(TEvProposeTransaction&, const TActorContext& ctx, TString workingDir, TString name); - void SendProposeRequest(const NActors::TActorContext &ctx) { - PrepareTopicPath(ctx); - std::pair <TString, TString> pathPair; - try { - pathPair = NKikimr::NGRpcService::SplitPath(TopicPath); - } catch (const std::exception &ex) { - this->Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); - return this->ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx); - } - - const auto &workingDir = pathPair.first; - const auto &name = pathPair.second; - - std::unique_ptr <TEvTxUserProxy::TEvProposeTransaction> proposal( - new TEvTxUserProxy::TEvProposeTransaction()); - - SetDatabase(proposal.get(), *this->Request_); - - if (this->Request_->GetInternalToken().empty()) { - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - "Unauthenticated access is forbidden, please provide credentials", ctx); - } - } else { - proposal->Record.SetUserToken(this->Request_->GetInternalToken()); - } - - static_cast<TDerived*>(this)->FillProposeRequest(*proposal, ctx, workingDir, name); - - if (!IsDead) { - ctx.Send(MakeTxProxyID(), proposal.release()); - } - } - - void SendDescribeProposeRequest(const NActors::TActorContext& ctx) { - PrepareTopicPath(ctx); + TopicPath = NPersQueue::GetFullTopicPath(ctx, this->Request_->GetDatabaseName(), TopicPath); + } + + TString GetTopicPath(const NActors::TActorContext& ctx) { + PrepareTopicPath(ctx); + return TopicPath; + } + + protected: + // TDerived must implement FillProposeRequest(TEvProposeTransaction&, const TActorContext& ctx, TString workingDir, TString name); + void SendProposeRequest(const NActors::TActorContext &ctx) { + PrepareTopicPath(ctx); + std::pair <TString, TString> pathPair; + try { + pathPair = NKikimr::NGRpcService::SplitPath(TopicPath); + } catch (const std::exception &ex) { + this->Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); + return this->ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx); + } + + const auto &workingDir = pathPair.first; + const auto &name = pathPair.second; + + std::unique_ptr <TEvTxUserProxy::TEvProposeTransaction> proposal( + new TEvTxUserProxy::TEvProposeTransaction()); + + SetDatabase(proposal.get(), *this->Request_); + + if (this->Request_->GetInternalToken().empty()) { + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + "Unauthenticated access is forbidden, please provide credentials", ctx); + } + } else { + proposal->Record.SetUserToken(this->Request_->GetInternalToken()); + } + + static_cast<TDerived*>(this)->FillProposeRequest(*proposal, ctx, workingDir, name); + + if (!IsDead) { + ctx.Send(MakeTxProxyID(), proposal.release()); + } + } + + void SendDescribeProposeRequest(const NActors::TActorContext& ctx) { + PrepareTopicPath(ctx); auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); navigateRequest->DatabaseName = CanonizePath(this->Request_->GetDatabaseName().GetOrElse("")); @@ -114,18 +114,18 @@ namespace NKikimr::NGRpcProxy::V1 { entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; navigateRequest->ResultSet.emplace_back(entry); - if (this->Request_->GetInternalToken().empty()) { - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - "Unauthenticated access is forbidden, please provide credentials", ctx); - } - } else { + if (this->Request_->GetInternalToken().empty()) { + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + "Unauthenticated access is forbidden, please provide credentials", ctx); + } + } else { navigateRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken()); - } - if (!IsDead) { + } + if (!IsDead) { ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release())); - } - } + } + } bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); @@ -169,7 +169,7 @@ namespace NKikimr::NGRpcProxy::V1 { ) ); return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } + } case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: { this->Request_->RaiseIssue( FillIssue( @@ -204,30 +204,30 @@ namespace NKikimr::NGRpcProxy::V1 { default: return TBase::Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); } - } - + } + void ReplyWithError(Ydb::StatusIds::StatusCode status, Ydb::PersQueue::ErrorCode::ErrorCode pqStatus, const TString& messageText, const NActors::TActorContext& ctx) { - this->Request_->RaiseIssue(FillIssue(messageText, pqStatus)); - this->Request_->ReplyWithYdbStatus(status); - this->Die(ctx); - IsDead = true; - } - - void ReplyWithResult(Ydb::StatusIds::StatusCode status, const NActors::TActorContext& ctx) { - this->Request_->ReplyWithYdbStatus(status); - this->Die(ctx); - IsDead = true; - } - - template<class TProtoResult> - void ReplyWithResult(Ydb::StatusIds::StatusCode status, const TProtoResult& result, const NActors::TActorContext& ctx) { - this->Request_->SendResult(result, status); - this->Die(ctx); - IsDead = true; - } - + this->Request_->RaiseIssue(FillIssue(messageText, pqStatus)); + this->Request_->ReplyWithYdbStatus(status); + this->Die(ctx); + IsDead = true; + } + + void ReplyWithResult(Ydb::StatusIds::StatusCode status, const NActors::TActorContext& ctx) { + this->Request_->ReplyWithYdbStatus(status); + this->Die(ctx); + IsDead = true; + } + + template<class TProtoResult> + void ReplyWithResult(Ydb::StatusIds::StatusCode status, const TProtoResult& result, const NActors::TActorContext& ctx) { + this->Request_->SendResult(result, status); + this->Die(ctx); + IsDead = true; + } + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); @@ -235,53 +235,53 @@ namespace NKikimr::NGRpcProxy::V1 { } } - private: - bool IsDead = false; - TString TopicPath; - }; - - //----------------------------------------------------------------------------------- - - template<class TDerived, class TRequest> - class TUpdateSchemeActor : public TPQGrpcSchemaBase<TDerived, TRequest> { - using TBase = TPQGrpcSchemaBase<TDerived, TRequest>; - - public: - TUpdateSchemeActor(TRequest* request, const TString& topicPath) - : TBase(request, topicPath) - {} - ~TUpdateSchemeActor() = default; - - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name) - { - Y_UNUSED(name); + private: + bool IsDead = false; + TString TopicPath; + }; + + //----------------------------------------------------------------------------------- + + template<class TDerived, class TRequest> + class TUpdateSchemeActor : public TPQGrpcSchemaBase<TDerived, TRequest> { + using TBase = TPQGrpcSchemaBase<TDerived, TRequest>; + + public: + TUpdateSchemeActor(TRequest* request, const TString& topicPath) + : TBase(request, topicPath) + {} + ~TUpdateSchemeActor() = default; + + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name) + { + Y_UNUSED(name); const auto& response = DescribeSchemeResult->Get()->Request.Get()->ResultSet.front(); NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup); - modifyScheme.SetWorkingDir(workingDir); - - auto* config = modifyScheme.MutableAlterPersQueueGroup(); + modifyScheme.SetWorkingDir(workingDir); + + auto* config = modifyScheme.MutableAlterPersQueueGroup(); Y_VERIFY(response.Self); Y_VERIFY(response.PQGroupInfo); config->CopyFrom(response.PQGroupInfo->Description); - - { - auto applyIf = modifyScheme.AddApplyIf(); + + { + auto applyIf = modifyScheme.AddApplyIf(); applyIf->SetPathId(response.Self->Info.GetPathId()); applyIf->SetPathVersion(response.Self->Info.GetPathVersion()); - } - + } + static_cast<TDerived*>(this)->ModifyPersqueueConfig( ctx, *config, response.PQGroupInfo->Description, response.Self->Info ); - - this->DescribeSchemeResult.Reset(); - } - + + this->DescribeSchemeResult.Reset(); + } + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); Y_VERIFY(result->ResultSet.size() == 1); @@ -289,14 +289,14 @@ namespace NKikimr::NGRpcProxy::V1 { return this->SendProposeRequest(ctx); } - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { default: TBase::StateWork(ev, ctx); - } - } - - private: + } + } + + private: THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult; - }; - + }; + } diff --git a/ydb/services/lib/actors/ya.make b/ydb/services/lib/actors/ya.make index c70e88208c..3947cc2ade 100644 --- a/ydb/services/lib/actors/ya.make +++ b/ydb/services/lib/actors/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - pq_schema_actor.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + pq_schema_actor.cpp +) + +PEERDIR( library/cpp/grpc/server ydb/core/grpc_services ydb/core/mind @@ -16,6 +16,6 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/lib/jwt ydb/public/lib/operation_id -) - -END() +) + +END() diff --git a/ydb/services/lib/sharding/sharding.cpp b/ydb/services/lib/sharding/sharding.cpp index 4bbc183a60..ce0e5a1c0a 100644 --- a/ydb/services/lib/sharding/sharding.cpp +++ b/ydb/services/lib/sharding/sharding.cpp @@ -1,14 +1,14 @@ -#include "sharding.h" - +#include "sharding.h" + #include <vector> -#include <util/generic/maybe.h> -#include <util/generic/yexception.h> -#include <util/string/printf.h> - -namespace NKikimr::NDataStreams::V1 { - - namespace { - +#include <util/generic/maybe.h> +#include <util/generic/yexception.h> +#include <util/string/printf.h> + +namespace NKikimr::NDataStreams::V1 { + + namespace { + NYql::NDecimal::TUint128 Uint128FromString(const TString& bytes, ui32 base = 10) { Y_VERIFY(base == 10 || base == 16); NYql::NDecimal::TUint128 x = 1; @@ -21,20 +21,20 @@ namespace NKikimr::NDataStreams::V1 { ythrow yexception() << "string is not valid Uint128"; res += x * v; x = base == 16 ? x << 4 : x * 10; - } + } return res; - } - + } + ui32 ShardFromUint128(NYql::NDecimal::TUint128 value, NYql::NDecimal::TUint128 totalShardsCount) { - NYql::NDecimal::TUint128 max = -1; - NYql::NDecimal::TUint128 sliceSize = max / totalShardsCount; - NYql::NDecimal::TUint128 shard = value / sliceSize; - - return shard >= totalShardsCount ? (ui32)(totalShardsCount - 1) : (ui32)shard; - } - - } - + NYql::NDecimal::TUint128 max = -1; + NYql::NDecimal::TUint128 sliceSize = max / totalShardsCount; + NYql::NDecimal::TUint128 shard = value / sliceSize; + + return shard >= totalShardsCount ? (ui32)(totalShardsCount - 1) : (ui32)shard; + } + + } + TString Uint128ToDecimalString(NYql::NDecimal::TUint128 value, const NYql::NDecimal::TUint128& base) { std::vector<char> result; while (value != 0) { @@ -45,51 +45,51 @@ namespace NKikimr::NDataStreams::V1 { return result.size() > 0 ? TString(result.begin(), result.end()) : "0"; } - TString GetShardName(ui32 index) { - return Sprintf("shard-%06d", index); - } - + TString GetShardName(ui32 index) { + return Sprintf("shard-%06d", index); + } + NYql::NDecimal::TUint128 BytesToDecimal(const TString& bytes) { return Uint128FromString(bytes, 10); - } - + } + bool IsValidDecimal(const TString& bytes) { if (bytes.empty()) return false; if (bytes.size() > 1 && (bytes[0] < '1' || bytes[0] > '9')) return false; - - static const TString UI128_MAX = "340282366920938463463374607431768211455"; + + static const TString UI128_MAX = "340282366920938463463374607431768211455"; if (bytes.size() > UI128_MAX.size() || (bytes.size() == UI128_MAX.size() && bytes > UI128_MAX)) { - return false; - } + return false; + } for (auto& c : bytes) { if (c < '0' || c > '9') return false; } return true; - } - + } + NYql::NDecimal::TUint128 HexBytesToDecimal(const TString& hex) { return Uint128FromString(hex, 16); - } - + } + ui32 ShardFromDecimal(const NYql::NDecimal::TUint128& decimal, ui32 totalShardsCount) { return ShardFromUint128(decimal, totalShardsCount); } - THashKeyRange RangeFromShardNumber(ui32 shardNumber, ui32 shardCount) { - Y_ENSURE(shardNumber < shardCount); - - NYql::NDecimal::TUint128 max = -1; - if (shardCount == 1) { + THashKeyRange RangeFromShardNumber(ui32 shardNumber, ui32 shardCount) { + Y_ENSURE(shardNumber < shardCount); + + NYql::NDecimal::TUint128 max = -1; + if (shardCount == 1) { return {0, max}; - } - NYql::NDecimal::TUint128 slice = max / shardCount; - NYql::NDecimal::TUint128 left = NYql::NDecimal::TUint128(shardNumber) * slice; - NYql::NDecimal::TUint128 right = - shardNumber + 1 == shardCount ? max : NYql::NDecimal::TUint128(shardNumber + 1) * slice - - NYql::NDecimal::TUint128(1); + } + NYql::NDecimal::TUint128 slice = max / shardCount; + NYql::NDecimal::TUint128 left = NYql::NDecimal::TUint128(shardNumber) * slice; + NYql::NDecimal::TUint128 right = + shardNumber + 1 == shardCount ? max : NYql::NDecimal::TUint128(shardNumber + 1) * slice - + NYql::NDecimal::TUint128(1); return {left, right}; - } -} + } +} diff --git a/ydb/services/lib/sharding/sharding.h b/ydb/services/lib/sharding/sharding.h index ac36a30b05..257ee8d2be 100644 --- a/ydb/services/lib/sharding/sharding.h +++ b/ydb/services/lib/sharding/sharding.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/decimal/yql_decimal.h> -#include <util/generic/string.h> - -namespace NKikimr::NDataStreams::V1 { - - struct THashKeyRange { +#include <util/generic/string.h> + +namespace NKikimr::NDataStreams::V1 { + + struct THashKeyRange { NYql::NDecimal::TUint128 Start; NYql::NDecimal::TUint128 End; - }; - - THashKeyRange RangeFromShardNumber(ui32 shardNumber, ui32 shardCount); - TString GetShardName(ui32 index); + }; + + THashKeyRange RangeFromShardNumber(ui32 shardNumber, ui32 shardCount); + TString GetShardName(ui32 index); NYql::NDecimal::TUint128 HexBytesToDecimal(const TString &hex); ui32 ShardFromDecimal(const NYql::NDecimal::TUint128 &decimal, ui32 totalShardsCount); NYql::NDecimal::TUint128 BytesToDecimal(const TString &bytes); diff --git a/ydb/services/lib/sharding/ya.make b/ydb/services/lib/sharding/ya.make index 1b4dad41ca..97df64d0e1 100644 --- a/ydb/services/lib/sharding/ya.make +++ b/ydb/services/lib/sharding/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - sharding.cpp -) - -END() +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + sharding.cpp +) + +END() diff --git a/ydb/services/lib/ya.make b/ydb/services/lib/ya.make index 4c898bfdc4..cb3aa46026 100644 --- a/ydb/services/lib/ya.make +++ b/ydb/services/lib/ya.make @@ -1,6 +1,6 @@ -OWNER(g:kikimr) - -RECURSE( - actors - sharding -) +OWNER(g:kikimr) + +RECURSE( + actors + sharding +) diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h index 25399613f5..65c8aca1db 100644 --- a/ydb/services/persqueue_v1/grpc_pq_actor.h +++ b/ydb/services/persqueue_v1/grpc_pq_actor.h @@ -1,7 +1,7 @@ #pragma once - -#include "persqueue_utils.h" - + +#include "persqueue_utils.h" + #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_deferrable.h> #include <ydb/core/grpc_services/rpc_calls.h> @@ -36,7 +36,7 @@ #include <util/generic/guid.h> #include <util/system/compiler.h> -namespace NKikimr::NGRpcProxy::V1 { +namespace NKikimr::NGRpcProxy::V1 { using namespace Ydb; @@ -482,7 +482,7 @@ private: HFunc(TEvPQProxy::TEvDieCommand, HandlePoison) HFunc(TEvPQProxy::TEvWriteInit, Handle) HFunc(TEvPQProxy::TEvWrite, Handle) - HFunc(TEvPQProxy::TEvUpdateToken, Handle) + HFunc(TEvPQProxy::TEvUpdateToken, Handle) HFunc(TEvPQProxy::TEvDone, Handle) HFunc(TEvPersQueue::TEvGetPartitionIdForWriteResponse, Handle) @@ -516,12 +516,12 @@ private: void CheckACL(const TActorContext& ctx); // Requests fresh ACL from 'SchemeCache' - void InitCheckSchema(const TActorContext& ctx, bool needWaitSchema = false); + void InitCheckSchema(const TActorContext& ctx, bool needWaitSchema = false); void Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvWrite::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvUpdateToken::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvDone::TPtr& ev, const NActors::TActorContext& ctx); - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx); void LogSession(const TActorContext& ctx); @@ -531,7 +531,7 @@ private: void ProceedPartition(const ui32 partition, const NActors::TActorContext& ctx); THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx); - void InitCheckACL(const TActorContext& ctx); + void InitCheckACL(const TActorContext& ctx); void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx); void Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx); @@ -571,7 +571,7 @@ private: EState State; TActorId SchemeCache; - TActorId NewSchemeCache; + TActorId NewSchemeCache; TActorId Writer; TString PeerName; @@ -591,7 +591,7 @@ private: ui32 NumReserveBytesRequests; - THolder<TAclWrapper> ACL; + THolder<TAclWrapper> ACL; struct TWriteRequestBatchInfo: public TSimpleRefCount<TWriteRequestBatchInfo> { using TPtr = TIntrusivePtr<TWriteRequestBatchInfo>; @@ -707,12 +707,12 @@ private: bool ProcessTopicSchemeCacheResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx); void HandleClientSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void SendCacheNavigateRequest(const TActorContext& ctx, const TString& path); + void SendCacheNavigateRequest(const TActorContext& ctx, const TString& path); void HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx); void FinishInitialization(const NActors::TActorContext& ctx); bool CheckTopicACL(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx); - void CheckClientACL(const TActorContext& ctx); + void CheckClientACL(const TActorContext& ctx); bool CheckACLPermissionsForNavigate(const TIntrusivePtr<TSecurityObject>& secObject, const TString& path, NACLib::EAccessRights rights, @@ -1126,23 +1126,23 @@ private: }; -class TDropTopicActor : public TPQGrpcSchemaBase<TDropTopicActor, NKikimr::NGRpcService::TEvPQDropTopicRequest> { -using TBase = TPQGrpcSchemaBase<TDropTopicActor, TEvPQDropTopicRequest>; - -public: +class TDropTopicActor : public TPQGrpcSchemaBase<TDropTopicActor, NKikimr::NGRpcService::TEvPQDropTopicRequest> { +using TBase = TPQGrpcSchemaBase<TDropTopicActor, TEvPQDropTopicRequest>; + +public: TDropTopicActor(NKikimr::NGRpcService::TEvPQDropTopicRequest* request); ~TDropTopicActor() = default; - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name); + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name); void Bootstrap(const NActors::TActorContext& ctx); void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } }; -class TDescribeTopicActor : public TPQGrpcSchemaBase<TDescribeTopicActor, NKikimr::NGRpcService::TEvPQDescribeTopicRequest> { -using TBase = TPQGrpcSchemaBase<TDescribeTopicActor, TEvPQDescribeTopicRequest>; +class TDescribeTopicActor : public TPQGrpcSchemaBase<TDescribeTopicActor, NKikimr::NGRpcService::TEvPQDescribeTopicRequest> { +using TBase = TPQGrpcSchemaBase<TDescribeTopicActor, TEvPQDescribeTopicRequest>; public: TDescribeTopicActor(NKikimr::NGRpcService::TEvPQDescribeTopicRequest* request); @@ -1156,42 +1156,42 @@ public: }; -class TAddReadRuleActor : public TUpdateSchemeActor<TAddReadRuleActor, TEvPQAddReadRuleRequest> { - using TBase = TUpdateSchemeActor<TAddReadRuleActor, TEvPQAddReadRuleRequest>; +class TAddReadRuleActor : public TUpdateSchemeActor<TAddReadRuleActor, TEvPQAddReadRuleRequest> { + using TBase = TUpdateSchemeActor<TAddReadRuleActor, TEvPQAddReadRuleRequest>; -public: - TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest *request); +public: + TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest *request); - void Bootstrap(const NActors::TActorContext &ctx); + void Bootstrap(const NActors::TActorContext &ctx); void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); -}; - -class TRemoveReadRuleActor : public TUpdateSchemeActor<TRemoveReadRuleActor, TEvPQRemoveReadRuleRequest> { - using TBase = TUpdateSchemeActor<TRemoveReadRuleActor, TEvPQRemoveReadRuleRequest>; - +}; + +class TRemoveReadRuleActor : public TUpdateSchemeActor<TRemoveReadRuleActor, TEvPQRemoveReadRuleRequest> { + using TBase = TUpdateSchemeActor<TRemoveReadRuleActor, TEvPQRemoveReadRuleRequest>; + public: - TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request); - - void Bootstrap(const NActors::TActorContext &ctx); + TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request); + + void Bootstrap(const NActors::TActorContext &ctx); void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); -}; - - -class TCreateTopicActor : public TPQGrpcSchemaBase<TCreateTopicActor, NKikimr::NGRpcService::TEvPQCreateTopicRequest> { -using TBase = TPQGrpcSchemaBase<TCreateTopicActor, TEvPQCreateTopicRequest>; - -public: +}; + + +class TCreateTopicActor : public TPQGrpcSchemaBase<TCreateTopicActor, NKikimr::NGRpcService::TEvPQCreateTopicRequest> { +using TBase = TPQGrpcSchemaBase<TCreateTopicActor, TEvPQCreateTopicRequest>; + +public: TCreateTopicActor(NKikimr::NGRpcService::TEvPQCreateTopicRequest* request, const TString& localCluster, const TVector<TString>& clusters); ~TCreateTopicActor() = default; - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name); + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name); void Bootstrap(const NActors::TActorContext& ctx); @@ -1202,15 +1202,15 @@ private: TVector<TString> Clusters; }; -class TAlterTopicActor : public TPQGrpcSchemaBase<TAlterTopicActor, NKikimr::NGRpcService::TEvPQAlterTopicRequest> { -using TBase = TPQGrpcSchemaBase<TAlterTopicActor, TEvPQAlterTopicRequest>; +class TAlterTopicActor : public TPQGrpcSchemaBase<TAlterTopicActor, NKikimr::NGRpcService::TEvPQAlterTopicRequest> { +using TBase = TPQGrpcSchemaBase<TAlterTopicActor, TEvPQAlterTopicRequest>; public: TAlterTopicActor(NKikimr::NGRpcService::TEvPQAlterTopicRequest* request); ~TAlterTopicActor() = default; - void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name); + void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name); void Bootstrap(const NActors::TActorContext& ctx); diff --git a/ydb/services/persqueue_v1/grpc_pq_read.cpp b/ydb/services/persqueue_v1/grpc_pq_read.cpp index dd7aab9893..6b7e72fdd0 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read.cpp @@ -21,17 +21,17 @@ using namespace PersQueue::V1; -IActor* CreatePQReadService(const TActorId& schemeCache, const TActorId& newSchemeCache, +IActor* CreatePQReadService(const TActorId& schemeCache, const TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions) { - return new TPQReadService(schemeCache, newSchemeCache, counters, maxSessions); + return new TPQReadService(schemeCache, newSchemeCache, counters, maxSessions); } -TPQReadService::TPQReadService(const TActorId& schemeCache, const TActorId& newSchemeCache, +TPQReadService::TPQReadService(const TActorId& schemeCache, const TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions) : SchemeCache(schemeCache) - , NewSchemeCache(newSchemeCache) + , NewSchemeCache(newSchemeCache) , Counters(counters) , MaxSessions(maxSessions) , LocalCluster("") @@ -42,9 +42,9 @@ TPQReadService::TPQReadService(const TActorId& schemeCache, const TActorId& newS void TPQReadService::Bootstrap(const TActorContext& ctx) { HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - proper condition if (HaveClusters) { - ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), - new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); - } + ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), + new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); + } ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() diff --git a/ydb/services/persqueue_v1/grpc_pq_read.h b/ydb/services/persqueue_v1/grpc_pq_read.h index 186d3d00bf..558063ed13 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.h +++ b/ydb/services/persqueue_v1/grpc_pq_read.h @@ -23,13 +23,13 @@ inline TActorId GetPQReadServiceActorID() { return TActorId(0, "PQReadSvc"); } -IActor* CreatePQReadService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); +IActor* CreatePQReadService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); class TPQReadService : public NActors::TActorBootstrapped<TPQReadService> { public: - TPQReadService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); + TPQReadService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); ~TPQReadService() {} diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index ee2e536524..654edfcfcd 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -365,7 +365,7 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor } case MigrationStreamingReadClientMessage::kStartRead: { const auto& req = request.start_read(); - + const ui64 readOffset = req.read_offset(); const ui64 commitOffset = req.commit_offset(); const bool verifyReadOffset = req.verify_read_offset(); @@ -703,7 +703,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo return; } - ClientId = NPersQueue::ConvertNewConsumerName(init.consumer(), ctx); + ClientId = NPersQueue::ConvertNewConsumerName(init.consumer(), ctx); ClientPath = init.consumer(); TStringBuilder session; @@ -1001,7 +1001,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T return; } - if (NumPartitionsFromTopic[topic]++ == 0) { + if (NumPartitionsFromTopic[topic]++ == 0) { if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { SetupTopicCounters(topic, jt->second.CloudId, jt->second.DbId, jt->second.FolderId); } else { @@ -1009,7 +1009,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T } } - auto it = TopicCounters.find(topic); + auto it = TopicCounters.find(topic); Y_VERIFY(it != TopicCounters.end()); ui64 assignId = NextAssignId++; @@ -1926,12 +1926,12 @@ bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, cons continue; //TODO - no such chunks must be on prod } - TString sourceId; - if (!r.GetSourceId().empty()) { + TString sourceId; + if (!r.GetSourceId().empty()) { if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_PROXY, "read bad sourceId from " << Partition - << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "'"); - } + LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_PROXY, "read bad sourceId from " << Partition + << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "'"); + } sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); } @@ -1984,9 +1984,9 @@ bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, cons message->set_create_timestamp_ms(r.GetCreateTimestampMS()); message->set_offset(r.GetOffset()); - message->set_explicit_hash(r.GetExplicitHash()); - message->set_partition_key(r.GetPartitionKey()); - + message->set_explicit_hash(r.GetExplicitHash()); + message->set_partition_key(r.GetPartitionKey()); + if (proto.HasCodec()) { message->set_codec(NPQ::ToV1Codec((NPersQueueCommon::ECodec)proto.GetCodec())); } @@ -2577,7 +2577,7 @@ TReadInitAndAuthActor::TReadInitAndAuthActor( , MetaCacheId(metaCache) , NewSchemeCache(newSchemeCache) , ClientId(clientId) - , ClientPath(NPersQueue::ConvertOldConsumerName(ClientId, ctx)) + , ClientPath(NPersQueue::ConvertOldConsumerName(ClientId, ctx)) , Token(token) , Counters(counters) { @@ -2596,7 +2596,7 @@ void TReadInitAndAuthActor::Bootstrap(const TActorContext &ctx) { TVector<TString> topicNames; for (const auto& topic : Topics) { topicNames.emplace_back(topic.second.TopicNameConverter->GetPrimaryPath()); - } + } DoCheckACL = AppData(ctx)->PQConfig.GetCheckACL() && Token; ctx.Send(MetaCacheId, new TEvDescribeTopicsRequest(topicNames)); } @@ -2618,18 +2618,18 @@ void TReadInitAndAuthActor::CloseSession(const TString& errorReason, const PersQ Die(ctx); } -void TReadInitAndAuthActor::SendCacheNavigateRequest(const TActorContext& ctx, const TString& path) { - auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = NKikimr::SplitPath(path); +void TReadInitAndAuthActor::SendCacheNavigateRequest(const TActorContext& ctx, const TString& path) { + auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = NKikimr::SplitPath(path); entry.SyncVersion = true; - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - schemeCacheRequest->ResultSet.emplace_back(entry); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + schemeCacheRequest->ResultSet.emplace_back(entry); LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Send client acl request"); - ctx.Send(NewSchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); -} + ctx.Send(NewSchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); +} + - bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse( const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx @@ -2638,9 +2638,9 @@ bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse( Y_VERIFY(entry.PQGroupInfo); topicsIter->second.TabletID = pqDescr.GetBalancerTabletID(); return CheckTopicACL(entry, topicsIter->first, ctx); -} +} + - void TReadInitAndAuthActor::HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Handle describe topics response"); for (const auto& entry : ev->Get()->Result->ResultSet) { @@ -2803,7 +2803,7 @@ void TReadInfoActor::Bootstrap(const TActorContext& ctx) { auto request = dynamic_cast<const ReadInfoRequest*>(GetProtoRequest()); Y_VERIFY(request); - ClientId = NPersQueue::ConvertNewConsumerName(request->consumer().path(), ctx); + ClientId = NPersQueue::ConvertNewConsumerName(request->consumer().path(), ctx); bool readOnlyLocal = request->get_only_original(); diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp index ea62323a04..7756adc797 100644 --- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp @@ -15,7 +15,7 @@ using namespace NKikimrClient; using grpc::Status; -namespace NKikimr::NGRpcProxy::V1 { +namespace NKikimr::NGRpcProxy::V1 { constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; @@ -24,7 +24,7 @@ constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; using namespace PersQueue::V1; -IActor* CreatePQSchemaService(const TActorId& schemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { +IActor* CreatePQSchemaService(const TActorId& schemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { return new TPQSchemaService(schemeCache, counters); } @@ -41,8 +41,8 @@ TPQSchemaService::TPQSchemaService(const TActorId& schemeCache, void TPQSchemaService::Bootstrap(const TActorContext& ctx) { if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration]: switch to haveClusters - ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), - new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); + ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), + new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); } Become(&TThis::StateFunc); @@ -87,7 +87,7 @@ void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQDropTopicRequest::TPtr TDropTopicActor::TDropTopicActor(NKikimr::NGRpcService::TEvPQDropTopicRequest* request) - : TBase(request, request->GetProtoRequest()->path()) + : TBase(request, request->GetProtoRequest()->path()) { } @@ -99,10 +99,10 @@ void TDropTopicActor::Bootstrap(const NActors::TActorContext& ctx) } -void TDropTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name) -{ - Y_UNUSED(ctx); +void TDropTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name) +{ + Y_UNUSED(ctx); NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetWorkingDir(workingDir); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropPersQueueGroup); @@ -110,7 +110,7 @@ void TDropTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& } TDescribeTopicActor::TDescribeTopicActor(NKikimr::NGRpcService::TEvPQDescribeTopicRequest* request) - : TBase(request, request->GetProtoRequest()->path()) + : TBase(request, request->GetProtoRequest()->path()) { } @@ -262,21 +262,21 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv void TDescribeTopicActor::Bootstrap(const NActors::TActorContext& ctx) { TBase::Bootstrap(ctx); - - SendDescribeProposeRequest(ctx); + + SendDescribeProposeRequest(ctx); Become(&TDescribeTopicActor::StateWork); } -TAddReadRuleActor::TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest* request) - : TBase(request, request->GetProtoRequest()->path()) -{ -} +TAddReadRuleActor::TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest* request) + : TBase(request, request->GetProtoRequest()->path()) +{ +} -void TAddReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TBase::StateWork); +void TAddReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TBase::StateWork); } void TAddReadRuleActor::ModifyPersqueueConfig( @@ -287,8 +287,8 @@ void TAddReadRuleActor::ModifyPersqueueConfig( ) { Y_UNUSED(pqGroupDescription); - auto* pqConfig = groupConfig.MutablePQTabletConfig(); - auto rule = GetProtoRequest()->read_rule(); + auto* pqConfig = groupConfig.MutablePQTabletConfig(); + auto rule = GetProtoRequest()->read_rule(); if (rule.version() == 0) { rule.set_version(selfInfo.GetVersion().GetPQVersion()); @@ -296,28 +296,28 @@ void TAddReadRuleActor::ModifyPersqueueConfig( auto serviceTypes = GetSupportedClientServiceTypes(ctx); TString error = AddReadRuleToConfig(pqConfig, rule, serviceTypes, ctx); bool hasDuplicates = false; - if (error.Empty()) { + if (error.Empty()) { hasDuplicates = CheckReadRulesConfig(*pqConfig, serviceTypes, error); - } - - if (!error.Empty()) { - return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST, + } + + if (!error.Empty()) { + return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST, hasDuplicates ? Ydb::PersQueue::ErrorCode::OK : Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } -} - -TRemoveReadRuleActor::TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request) - : TBase(request, request->GetProtoRequest()->path()) -{ - Y_ASSERT(request); -} - -void TRemoveReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TBase::StateWork); -} - + } +} + +TRemoveReadRuleActor::TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request) + : TBase(request, request->GetProtoRequest()->path()) +{ + Y_ASSERT(request); +} + +void TRemoveReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TBase::StateWork); +} + void TRemoveReadRuleActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -334,11 +334,11 @@ void TRemoveReadRuleActor::ModifyPersqueueConfig( ); if (!error.Empty()) { return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } -} - + } +} + TCreateTopicActor::TCreateTopicActor(NKikimr::NGRpcService::TEvPQCreateTopicRequest* request, const TString& localCluster, const TVector<TString>& clusters) - : TBase(request, request->GetProtoRequest()->path()) + : TBase(request, request->GetProtoRequest()->path()) , LocalCluster(localCluster) , Clusters(clusters) { @@ -354,7 +354,7 @@ void TCreateTopicActor::Bootstrap(const NActors::TActorContext& ctx) TAlterTopicActor::TAlterTopicActor(NKikimr::NGRpcService::TEvPQAlterTopicRequest* request) - : TBase(request, request->GetProtoRequest()->path()) + : TBase(request, request->GetProtoRequest()->path()) { Y_ASSERT(request); } @@ -366,9 +366,9 @@ void TAlterTopicActor::Bootstrap(const NActors::TActorContext& ctx) Become(&TAlterTopicActor::StateWork); } -void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name) -{ +void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name) +{ NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetWorkingDir(workingDir); @@ -396,16 +396,16 @@ void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction } -void TAlterTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, - const TString& workingDir, const TString& name) { +void TAlterTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, + const TString& workingDir, const TString& name) { NKikimrSchemeOp::TModifyScheme &modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetWorkingDir(workingDir); - TString error; - auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, true, error); - if (!error.empty()) { - Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST)); - - return ReplyWithResult(status, ctx); + TString error; + auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, true, error); + if (!error.empty()) { + Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST)); + + return ReplyWithResult(status, ctx); } } @@ -417,21 +417,21 @@ void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQAlterTopicRequest::TPt ctx.Register(new TAlterTopicActor(ev->Release().Release())); } -void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new Add read rules request"); - ctx.Register(new TAddReadRuleActor(ev->Release().Release())); -} - -void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new Remove read rules request"); - ctx.Register(new TRemoveReadRuleActor(ev->Release().Release())); -} - -void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQCreateTopicRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new create topic request"); - ctx.Register(new TCreateTopicActor(ev->Release().Release(), LocalCluster, Clusters)); -} - +void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new Add read rules request"); + ctx.Register(new TAddReadRuleActor(ev->Release().Release())); +} + +void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new Remove read rules request"); + ctx.Register(new TRemoveReadRuleActor(ev->Release().Release())); +} + +void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQCreateTopicRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new create topic request"); + ctx.Register(new TCreateTopicActor(ev->Release().Release(), LocalCluster, Clusters)); +} + void TPQSchemaService::Handle(NKikimr::NGRpcService::TEvPQDescribeTopicRequest::TPtr& ev, const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "new Describe topic request"); ctx.Register(new TDescribeTopicActor(ev->Release().Release())); @@ -456,11 +456,11 @@ void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEv void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvPQDescribeTopicRequest::TPtr& ev, const TActorContext& ctx) { ctx.Send(NKikimr::NGRpcProxy::V1::GetPQSchemaServiceActorID(), ev->Release().Release()); } - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NGRpcProxy::V1::GetPQSchemaServiceActorID(), ev->Release().Release()); -} - -void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(NKikimr::NGRpcProxy::V1::GetPQSchemaServiceActorID(), ev->Release().Release()); -} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NGRpcProxy::V1::GetPQSchemaServiceActorID(), ev->Release().Release()); +} + +void NKikimr::NGRpcService::TGRpcRequestProxy::Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(NKikimr::NGRpcProxy::V1::GetPQSchemaServiceActorID(), ev->Release().Release()); +} diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.h b/ydb/services/persqueue_v1/grpc_pq_schema.h index 89b5b16b9c..2ef8c503e4 100644 --- a/ydb/services/persqueue_v1/grpc_pq_schema.h +++ b/ydb/services/persqueue_v1/grpc_pq_schema.h @@ -12,7 +12,7 @@ #include <util/system/mutex.h> -namespace NKikimr::NGRpcProxy::V1 { +namespace NKikimr::NGRpcProxy::V1 { static const i64 DEFAULT_MAX_DATABASE_MESSAGEGROUP_SEQNO_RETENTION_PERIOD = 16*24*60*60*1000; @@ -36,8 +36,8 @@ private: HFunc(NKikimr::NGRpcService::TEvPQDropTopicRequest, Handle); HFunc(NKikimr::NGRpcService::TEvPQCreateTopicRequest, Handle); HFunc(NKikimr::NGRpcService::TEvPQAlterTopicRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvPQAddReadRuleRequest, Handle); - HFunc(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvPQAddReadRuleRequest, Handle); + HFunc(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest, Handle); HFunc(NKikimr::NGRpcService::TEvPQDescribeTopicRequest, Handle); hFunc(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate, Handle); @@ -48,8 +48,8 @@ private: void Handle(NKikimr::NGRpcService::TEvPQDropTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(NKikimr::NGRpcService::TEvPQCreateTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(NKikimr::NGRpcService::TEvPQAlterTopicRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx); void Handle(NKikimr::NGRpcService::TEvPQDescribeTopicRequest::TPtr& ev, const TActorContext& ctx); void Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev); diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp index 12b680a1ad..5f17a2f3ab 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp @@ -18,17 +18,17 @@ using namespace PersQueue::V1; /////////////////////////////////////////////////////////////////////////////// -IActor* CreatePQWriteService(const TActorId& schemeCache, const TActorId& newSchemeCache, +IActor* CreatePQWriteService(const TActorId& schemeCache, const TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions) { - return new TPQWriteService(schemeCache, newSchemeCache, counters, maxSessions); + return new TPQWriteService(schemeCache, newSchemeCache, counters, maxSessions); } -TPQWriteService::TPQWriteService(const TActorId& schemeCache, const TActorId& newSchemeCache, +TPQWriteService::TPQWriteService(const TActorId& schemeCache, const TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions) : SchemeCache(schemeCache) - , NewSchemeCache(newSchemeCache) + , NewSchemeCache(newSchemeCache) , Counters(counters) , MaxSessions(maxSessions) , Enabled(false) @@ -39,9 +39,9 @@ TPQWriteService::TPQWriteService(const TActorId& schemeCache, const TActorId& ne void TPQWriteService::Bootstrap(const TActorContext& ctx) { HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration]: switch to proper option if (HaveClusters) { - ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), - new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); - } + ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), + new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); + } ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); ConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() @@ -176,8 +176,8 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt return; } - - TString localCluster = AvailableLocalCluster(ctx); + + TString localCluster = AvailableLocalCluster(ctx); if (HaveClusters && localCluster.empty()) { ev->Get()->GetStreamCtx()->Attach(ctx.SelfID); if (LocalCluster) { diff --git a/ydb/services/persqueue_v1/grpc_pq_write.h b/ydb/services/persqueue_v1/grpc_pq_write.h index d55d55c93b..73148388b2 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.h +++ b/ydb/services/persqueue_v1/grpc_pq_write.h @@ -21,13 +21,13 @@ inline TActorId GetPQWriteServiceActorID() { return TActorId(0, "PQWriteSvc"); } -IActor* CreatePQWriteService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); +IActor* CreatePQWriteService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); class TPQWriteService : public NActors::TActorBootstrapped<TPQWriteService> { public: - TPQWriteService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); + TPQWriteService(const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions); ~TPQWriteService() {} @@ -38,7 +38,7 @@ private: ui64 NextCookie(); bool TooMuchSessions(); - TString AvailableLocalCluster(const TActorContext& ctx) const; + TString AvailableLocalCluster(const TActorContext& ctx) const; STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { @@ -60,7 +60,7 @@ private: void Handle(TEvPQProxy::TEvSessionDead::TPtr& ev, const TActorContext& ctx); NActors::TActorId SchemeCache; - NActors::TActorId NewSchemeCache; + NActors::TActorId NewSchemeCache; TAtomic LastCookie = 0; diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index 96034c7fe4..3038cc82a6 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -90,7 +90,7 @@ static const TString SELECT_SOURCEID_QUERY1 = "DECLARE $Topic AS Utf8; " "DECLARE $SourceId AS Utf8; " "SELECT Partition, CreateTime FROM `"; -static const TString SELECT_SOURCEID_QUERY2 = "` " +static const TString SELECT_SOURCEID_QUERY2 = "` " "WHERE Hash == $Hash AND Topic == $Topic AND SourceId == $SourceId; "; static const TString UPDATE_SOURCEID_QUERY1 = @@ -102,7 +102,7 @@ static const TString UPDATE_SOURCEID_QUERY1 = "DECLARE $CreateTime AS Uint64; " "DECLARE $AccessTime AS Uint64; " "UPSERT INTO `"; -static const TString UPDATE_SOURCEID_QUERY2 = "` (Hash, Topic, SourceId, CreateTime, AccessTime, Partition) VALUES " +static const TString UPDATE_SOURCEID_QUERY2 = "` (Hash, Topic, SourceId, CreateTime, AccessTime, Partition) VALUES " "($Hash, $Topic, $SourceId, $CreateTime, $AccessTime, $Partition); "; //TODO: add here tracking of bytes in/out @@ -117,7 +117,7 @@ TWriteSessionActor::TWriteSessionActor( : Request(request) , State(ES_CREATED) , SchemeCache(schemeCache) - , NewSchemeCache(newSchemeCache) + , NewSchemeCache(newSchemeCache) , PeerName("") , Cookie(cookie) , TopicsController(topicsController) @@ -154,8 +154,8 @@ TWriteSessionActor::~TWriteSessionActor() = default; void TWriteSessionActor::Bootstrap(const TActorContext& ctx) { Y_VERIFY(Request); - SelectSourceIdQuery = SELECT_SOURCEID_QUERY1 + AppData(ctx)->PQConfig.GetSourceIdTablePath() + SELECT_SOURCEID_QUERY2; - UpdateSourceIdQuery = UPDATE_SOURCEID_QUERY1 + AppData(ctx)->PQConfig.GetSourceIdTablePath() + UPDATE_SOURCEID_QUERY2; + SelectSourceIdQuery = SELECT_SOURCEID_QUERY1 + AppData(ctx)->PQConfig.GetSourceIdTablePath() + SELECT_SOURCEID_QUERY2; + UpdateSourceIdQuery = UPDATE_SOURCEID_QUERY1 + AppData(ctx)->PQConfig.GetSourceIdTablePath() + UPDATE_SOURCEID_QUERY2; Request->GetStreamCtx()->Attach(ctx.SelfID); if (!Request->GetStreamCtx()->Read()) { @@ -263,9 +263,9 @@ void TWriteSessionActor::CheckACL(const TActorContext& ctx) { //Y_VERIFY(ACLCheckInProgress); NACLib::EAccessRights rights = NACLib::EAccessRights::UpdateRow; - - Y_VERIFY(ACL); - if (ACL->CheckAccess(rights, *Token)) { + + Y_VERIFY(ACL); + if (ACL->CheckAccess(rights, *Token)) { ACLCheckInProgress = false; if (FirstACLCheck) { FirstACLCheck = false; @@ -311,7 +311,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor PersQueue::ErrorCode::BAD_REQUEST, ctx ); return; - } + } PeerName = event->PeerName; @@ -333,16 +333,16 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor UserAgent = "pqv1 server"; LogSession(ctx); - if (Request->GetInternalToken().empty()) { // session without auth - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - Request->ReplyUnauthenticated("Unauthenticated access is forbidden, please provide credentials"); - Die(ctx); - return; - } - } + if (Request->GetInternalToken().empty()) { // session without auth + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + Request->ReplyUnauthenticated("Unauthenticated access is forbidden, please provide credentials"); + Die(ctx); + return; + } + } + + InitCheckSchema(ctx, true); - InitCheckSchema(ctx, true); - PreferedPartition = init.partition_group_id() > 0 ? init.partition_group_id() - 1 : Max<ui32>(); InitMeta = GetInitialDataChunk(init, TopicConverter->GetFullLegacyName(), PeerName); // ToDo[migration] - check? @@ -393,18 +393,18 @@ void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& db SessionsActive.Inc(); } -void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema"); +void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) { + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema"); if (!needWaitSchema) { ACLCheckInProgress = true; - } + } ctx.Send(SchemeCache, new TEvDescribeTopicsRequest({TopicConverter->GetPrimaryPath()})); - if (needWaitSchema) { - State = ES_WAIT_SCHEME_2; - } -} - + if (needWaitSchema) { + State = ES_WAIT_SCHEME_2; + } +} + void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { auto* res = ev->Get()->Result.Get(); Y_VERIFY(res->ResultSet.size() == 1); @@ -452,44 +452,44 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo } else { Y_VERIFY(Request->GetYdbToken()); Auth = *Request->GetYdbToken(); - + Token = new NACLib::TUserToken(Request->GetInternalToken()); CheckACL(ctx); } } -void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); - const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); - Y_VERIFY(navigate->ResultSet.size() == 1); - if (navigate->ErrorCount > 0) { - const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; +void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); + const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); + Y_VERIFY(navigate->ResultSet.size() == 1); + if (navigate->ErrorCount > 0) { + const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; return CloseSession( TStringBuilder() << "Failed to read ACL for '" << TopicConverter->GetClientsideName() << "' Scheme cache error : " << status, PersQueue::ErrorCode::ERROR, ctx ); - } - + } + const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description; - + Y_VERIFY(pqDescription.PartitionsSize() > 0); Y_VERIFY(pqDescription.HasPQTabletConfig()); InitialPQTabletConfig = pqDescription.GetPQTabletConfig(); - + if (!pqDescription.HasBalancerTabletID()) { TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str()); - CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); - return; - } - + CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); + return; + } + BalancerTabletId = pqDescription.GetBalancerTabletID(); - + for (ui32 i = 0; i < pqDescription.PartitionsSize(); ++i) { const auto& pi = pqDescription.GetPartitions(i); - PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId(); - } - + PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId(); + } + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { const auto& tabletConfig = pqDescription.GetPQTabletConfig(); SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(), @@ -497,22 +497,22 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult:: } else { SetupCounters(); } - - Y_VERIFY(!navigate->ResultSet.empty()); - ACL.Reset(new TAclWrapper(navigate->ResultSet.front().SecurityObject)); - - if (Request->GetInternalToken().empty()) { // session without auth - // We've already checked authentication flag in init request. Here we should finish it - FirstACLCheck = false; - DiscoverPartition(ctx); - } else { - Y_VERIFY(Request->GetYdbToken()); - Auth = *Request->GetYdbToken(); - Token = new NACLib::TUserToken(Request->GetInternalToken()); - CheckACL(ctx); - } -} - + + Y_VERIFY(!navigate->ResultSet.empty()); + ACL.Reset(new TAclWrapper(navigate->ResultSet.front().SecurityObject)); + + if (Request->GetInternalToken().empty()) { // session without auth + // We've already checked authentication flag in init request. Here we should finish it + FirstACLCheck = false; + DiscoverPartition(ctx); + } else { + Y_VERIFY(Request->GetYdbToken()); + Auth = *Request->GetYdbToken(); + Token = new NACLib::TUserToken(Request->GetInternalToken()); + CheckACL(ctx); + } +} + void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) { if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration] - separate flag for having config tables @@ -1026,14 +1026,14 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvUpdateToken::TPtr& ev, const TAct void TWriteSessionActor::Handle(NGRpcService::TGRpcRequestProxy::TEvRefreshTokenResponse::TPtr &ev , const TActorContext& ctx) { Y_UNUSED(ctx); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "updating token"); - + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "updating token"); + if (ev->Get()->Authenticated && !ev->Get()->InternalToken.empty()) { Token = new NACLib::TUserToken(ev->Get()->InternalToken); - Request->SetInternalToken(ev->Get()->InternalToken); + Request->SetInternalToken(ev->Get()->InternalToken); UpdateTokenAuthenticated = true; if (!ACLCheckInProgress) { - InitCheckSchema(ctx); + InitCheckSchema(ctx); } } else { Request->ReplyUnauthenticated("refreshed token is invalid"); @@ -1156,7 +1156,7 @@ void TWriteSessionActor::HandleWakeup(const TActorContext& ctx) { ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup()); if (Token && !ACLCheckInProgress && RequestNotChecked && (ctx.Now() - LastACLCheckTimestamp > TDuration::Seconds(AppData(ctx)->PQConfig.GetACLRetryTimeoutSec()))) { RequestNotChecked = false; - InitCheckSchema(ctx); + InitCheckSchema(ctx); } // ToDo[migration] - separate flag for having config tables if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen() && !SourceIdUpdateInfly && ctx.Now() - LastSourceIdUpdate > SOURCEID_UPDATE_PERIOD) { diff --git a/ydb/services/persqueue_v1/persqueue.cpp b/ydb/services/persqueue_v1/persqueue.cpp index e79bacf346..ccacc4d43a 100644 --- a/ydb/services/persqueue_v1/persqueue.cpp +++ b/ydb/services/persqueue_v1/persqueue.cpp @@ -17,7 +17,7 @@ namespace V1 { static const ui32 PersQueueWriteSessionsMaxCount = 1000000; static const ui32 PersQueueReadSessionsMaxCount = 100000; -TGRpcPersQueueService::TGRpcPersQueueService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const NActors::TActorId& schemeCache,const NActors::TActorId& grpcRequestProxy) +TGRpcPersQueueService::TGRpcPersQueueService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const NActors::TActorId& schemeCache,const NActors::TActorId& grpcRequestProxy) : ActorSystem(system) , Counters(counters) , SchemeCache(schemeCache) @@ -26,15 +26,15 @@ TGRpcPersQueueService::TGRpcPersQueueService(NActors::TActorSystem *system, TInt void TGRpcPersQueueService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { CQ = cq; - InitNewSchemeCacheActor(); - + InitNewSchemeCacheActor(); + if (ActorSystem->AppData<TAppData>()->PQConfig.GetEnabled()) { - IActor* writeSvc = NGRpcProxy::V1::CreatePQWriteService(SchemeCache, NewSchemeCache,Counters, PersQueueWriteSessionsMaxCount); + IActor* writeSvc = NGRpcProxy::V1::CreatePQWriteService(SchemeCache, NewSchemeCache,Counters, PersQueueWriteSessionsMaxCount); TActorId actorId = ActorSystem->Register(writeSvc, TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); ActorSystem->RegisterLocalService(NGRpcProxy::V1::GetPQWriteServiceActorID(), actorId); - IActor* readSvc = NGRpcProxy::V1::CreatePQReadService(SchemeCache, NewSchemeCache, Counters, PersQueueReadSessionsMaxCount); + IActor* readSvc = NGRpcProxy::V1::CreatePQReadService(SchemeCache, NewSchemeCache, Counters, PersQueueReadSessionsMaxCount); actorId = ActorSystem->Register(readSvc, TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); ActorSystem->RegisterLocalService(NGRpcProxy::V1::GetPQReadServiceActorID(), actorId); @@ -58,14 +58,14 @@ void TGRpcPersQueueService::DecRequest() { Limiter->Dec(); } -void TGRpcPersQueueService::InitNewSchemeCacheActor() { - auto appData = ActorSystem->AppData<TAppData>(); +void TGRpcPersQueueService::InitNewSchemeCacheActor() { + auto appData = ActorSystem->AppData<TAppData>(); auto cacheCounters = GetServiceCounters(Counters, "pqproxy|schemecache"); auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); NewSchemeCache = ActorSystem->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); -} - +} + void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { auto getCounterBlock = NKikimr::NGRpcService::CreateCounterCb(Counters, ActorSystem); @@ -138,12 +138,12 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { ADD_REQUEST(DescribeTopic, PersQueueService, DescribeTopicRequest, DescribeTopicResponse, { ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQDescribeTopicRequest(ctx)); }) - ADD_REQUEST(AddReadRule, PersQueueService, AddReadRuleRequest, AddReadRuleResponse, { - ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQAddReadRuleRequest(ctx)); - }) - ADD_REQUEST(RemoveReadRule, PersQueueService, RemoveReadRuleRequest, RemoveReadRuleResponse, { - ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQRemoveReadRuleRequest(ctx)); - }) + ADD_REQUEST(AddReadRule, PersQueueService, AddReadRuleRequest, AddReadRuleResponse, { + ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQAddReadRuleRequest(ctx)); + }) + ADD_REQUEST(RemoveReadRule, PersQueueService, RemoveReadRuleRequest, RemoveReadRuleResponse, { + ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQRemoveReadRuleRequest(ctx)); + }) #undef ADD_REQUEST diff --git a/ydb/services/persqueue_v1/persqueue.h b/ydb/services/persqueue_v1/persqueue.h index 4284323182..eefe1ee4c2 100644 --- a/ydb/services/persqueue_v1/persqueue.h +++ b/ydb/services/persqueue_v1/persqueue.h @@ -30,15 +30,15 @@ public: private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - void InitNewSchemeCacheActor(); - + void InitNewSchemeCacheActor(); + NActors::TActorSystem* ActorSystem; grpc::ServerCompletionQueue* CQ = nullptr; TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; NGrpc::TGlobalLimiter* Limiter = nullptr; NActors::TActorId SchemeCache; - NActors::TActorId NewSchemeCache; + NActors::TActorId NewSchemeCache; NActors::TActorId GRpcRequestProxy; }; diff --git a/ydb/services/persqueue_v1/persqueue_common_ut.cpp b/ydb/services/persqueue_v1/persqueue_common_ut.cpp index 918c3b0ed2..9d706d0656 100644 --- a/ydb/services/persqueue_v1/persqueue_common_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_common_ut.cpp @@ -1,41 +1,41 @@ -#include "grpc_pq_actor.h" +#include "grpc_pq_actor.h" #include <ydb/services/persqueue_v1/ut/pq_data_writer.h> #include <ydb/services/persqueue_v1/ut/api_test_setup.h> #include <ydb/services/persqueue_v1/ut/test_utils.h> #include <ydb/services/persqueue_v1/ut/persqueue_test_fixture.h> - + #include <ydb/core/base/appdata.h> #include <ydb/core/testlib/test_pq_client.h> #include <ydb/core/persqueue/cluster_tracker.h> - + #include <ydb/core/tablet/tablet_counters_aggregator.h> - + #include <ydb/library/aclib/aclib.h> #include <ydb/library/persqueue/obfuscate/obfuscate.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> - -#include <library/cpp/testing/unittest/tests_data.h> -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/json/json_reader.h> - -#include <util/string/join.h> - -#include <grpc++/client_context.h> - + +#include <library/cpp/testing/unittest/tests_data.h> +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/json/json_reader.h> + +#include <util/string/join.h> + +#include <grpc++/client_context.h> + #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> - - -namespace NKikimr::NPersQueueTests { - -using namespace Tests; -using namespace NKikimrClient; -using namespace Ydb::PersQueue; -using namespace Ydb::PersQueue::V1; -using namespace NThreading; -using namespace NNetClassifier; - + + +namespace NKikimr::NPersQueueTests { + +using namespace Tests; +using namespace NKikimrClient; +using namespace Ydb::PersQueue; +using namespace Ydb::PersQueue::V1; +using namespace NThreading; +using namespace NNetClassifier; + #define MAKE_WRITE_STREAM(TOKEN) \ grpc::ClientContext context; \ @@ -43,274 +43,274 @@ using namespace NNetClassifier; auto stream = server.ServiceStub->StreamingWrite(&context); \ -Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { - // Auth* tests are for both authentication and authorization +Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { + // Auth* tests are for both authentication and authorization Y_UNIT_TEST(Auth_CreateGrpcStreamWithInvalidTokenInInitialMetadata_SessionClosedWithUnauthenticatedError) { TPersQueueV1TestServer server; SET_LOCALS; runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); runtime->GetAppData().EnforceUserTokenRequirement = true; - TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; - - for (const auto &invalidToken : invalidTokens) { - Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; + TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; + + for (const auto &invalidToken : invalidTokens) { + Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; MAKE_WRITE_STREAM(invalidToken); - - // TODO: Message should be written to gRPC in order to get error. Fix gRPC data plane API code if our expectations are different. - // Note that I check that initial metadata is sent during gRPC stream constructor. - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + // TODO: Message should be written to gRPC in order to get error. Fix gRPC data plane API code if our expectations are different. + // Note that I check that initial metadata is sent during gRPC stream constructor. + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - - UNIT_ASSERT_C(!stream->Read(&serverMessage), - "Expect session close with no messages from the server but got message: " - << serverMessage.DebugString()); - UNIT_ASSERT_EQUAL(grpc::StatusCode::UNAUTHENTICATED, stream->Finish().error_code()); - } - } - - TString GenerateValidToken(int i = 0) { - return "test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN; - } - + clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + + UNIT_ASSERT_C(!stream->Read(&serverMessage), + "Expect session close with no messages from the server but got message: " + << serverMessage.DebugString()); + UNIT_ASSERT_EQUAL(grpc::StatusCode::UNAUTHENTICATED, stream->Finish().error_code()); + } + } + + TString GenerateValidToken(int i = 0) { + return "test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN; + } + Y_UNIT_TEST(Auth_MultipleUpdateTokenRequestIterationsWithValidToken_GotUpdateTokenResponseForEachRequest) { TPersQueueV1TestServer server; - const int iterations = 10; - NACLib::TDiffACL acl; - for (int i = 0; i != iterations; ++i) { - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); - } - + const int iterations = 10; + NACLib::TDiffACL acl; + for (int i = 0; i != iterations; ++i) { + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); + } + server.ModifyTopicACL(server.GetTopic(), acl); - + MAKE_WRITE_STREAM(GenerateValidToken(0)); - - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, - serverMessage); - - - for (int i = 1; i != iterations; ++i) { - clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(i)); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C( - serverMessage.server_message_case() == StreamingWriteServerMessage::kUpdateTokenResponse, - serverMessage); - } - } - + clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, + serverMessage); + + + for (int i = 1; i != iterations; ++i) { + clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(i)); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C( + serverMessage.server_message_case() == StreamingWriteServerMessage::kUpdateTokenResponse, + serverMessage); + } + } + Y_UNIT_TEST( Auth_WriteSessionWithValidTokenAndACEAndThenRemoveACEAndSendWriteRequest_SessionClosedWithUnauthorizedErrorAfterSuccessfullWriteResponse ) { TPersQueueV1TestServer server; - //setup.GetPQConfig().SetACLRetryTimeoutSec(0); - NACLib::TDiffACL acl; - const auto token = GenerateValidToken(); - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, token); + //setup.GetPQConfig().SetACLRetryTimeoutSec(0); + NACLib::TDiffACL acl; + const auto token = GenerateValidToken(); + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, token); server.ModifyTopicACL(server.GetTopic(), acl); - + MAKE_WRITE_STREAM(token); - - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, - serverMessage); - - acl.ClearAccess(); + clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, + serverMessage); + + acl.ClearAccess(); server.ModifyTopicACL(server.GetTopic(), acl); - - clientMessage = StreamingWriteClientMessage(); - auto *writeRequest = clientMessage.mutable_write_request(); - TString message = "x"; - writeRequest->add_sequence_numbers(1); - writeRequest->add_message_sizes(message.size()); - writeRequest->add_created_at_ms(TInstant::Now().MilliSeconds()); - writeRequest->add_sent_at_ms(TInstant::Now().MilliSeconds()); - writeRequest->add_blocks_offsets(0); - writeRequest->add_blocks_part_numbers(0); - writeRequest->add_blocks_message_counts(1); - writeRequest->add_blocks_uncompressed_sizes(message.size()); - writeRequest->add_blocks_headers(TString(1, '\0')); - writeRequest->add_blocks_data(message); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kBatchWriteResponse, - serverMessage); - - Cerr << "Wait for session created with token with removed ACE to die"; - AssertStreamingSessionDead(stream, Ydb::StatusIds::UNAUTHORIZED, - Ydb::PersQueue::ErrorCode::ACCESS_DENIED); - } - - // TODO: Replace this test with a unit-test of TWriteSessionActor + + clientMessage = StreamingWriteClientMessage(); + auto *writeRequest = clientMessage.mutable_write_request(); + TString message = "x"; + writeRequest->add_sequence_numbers(1); + writeRequest->add_message_sizes(message.size()); + writeRequest->add_created_at_ms(TInstant::Now().MilliSeconds()); + writeRequest->add_sent_at_ms(TInstant::Now().MilliSeconds()); + writeRequest->add_blocks_offsets(0); + writeRequest->add_blocks_part_numbers(0); + writeRequest->add_blocks_message_counts(1); + writeRequest->add_blocks_uncompressed_sizes(message.size()); + writeRequest->add_blocks_headers(TString(1, '\0')); + writeRequest->add_blocks_data(message); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kBatchWriteResponse, + serverMessage); + + Cerr << "Wait for session created with token with removed ACE to die"; + AssertStreamingSessionDead(stream, Ydb::StatusIds::UNAUTHORIZED, + Ydb::PersQueue::ErrorCode::ACCESS_DENIED); + } + + // TODO: Replace this test with a unit-test of TWriteSessionActor Y_UNIT_TEST( Auth_MultipleInflightWriteUpdateTokenRequestWithDifferentValidToken_SessionClosedWithOverloadedError ) { TPersQueueV1TestServer server; SET_LOCALS; - const int iterations = 3; - NACLib::TDiffACL acl; - for (int i = 0; i != iterations; ++i) { - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); - } - + const int iterations = 3; + NACLib::TDiffACL acl; + for (int i = 0; i != iterations; ++i) { + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); + } + server.ModifyTopicACL(server.GetTopic(), acl); MAKE_WRITE_STREAM(GenerateValidToken(0)); - - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, - serverMessage); - - // TWriteSessionActor uses GRpcRequestProxy for authentication. This will make next update token procedure stuck indefinetely - auto noopActorID = TActorId(); + clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, + serverMessage); + + // TWriteSessionActor uses GRpcRequestProxy for authentication. This will make next update token procedure stuck indefinetely + auto noopActorID = TActorId(); for (size_t i = 0; i != runtime->GetNodeCount(); ++i) { - // Destroy GRpcRequestProxy + // Destroy GRpcRequestProxy runtime->RegisterService(NGRpcService::CreateGRpcRequestProxyId(), noopActorID); - } - - clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(1)); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - - - clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(2)); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - - - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_VALUES_EQUAL_C(Ydb::StatusIds::OVERLOADED, serverMessage.status(), serverMessage); - UNIT_ASSERT_C( - serverMessage.server_message_case() == StreamingWriteServerMessage::SERVER_MESSAGE_NOT_SET, - serverMessage); - } - + } + + clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(1)); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + + + clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(2)); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + + + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_VALUES_EQUAL_C(Ydb::StatusIds::OVERLOADED, serverMessage.status(), serverMessage); + UNIT_ASSERT_C( + serverMessage.server_message_case() == StreamingWriteServerMessage::SERVER_MESSAGE_NOT_SET, + serverMessage); + } + Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithInvalidToken_SessionClosedWithUnauthenticatedError) { TPersQueueV1TestServer server; SET_LOCALS; runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); - const TString validToken = "test_user@" BUILTIN_ACL_DOMAIN; - // TODO: Why test fails with 'BUILTIN_ACL_DOMAIN' as domain in invalid token? - TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; - NACLib::TDiffACL acl; - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); + const TString validToken = "test_user@" BUILTIN_ACL_DOMAIN; + // TODO: Why test fails with 'BUILTIN_ACL_DOMAIN' as domain in invalid token? + TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; + NACLib::TDiffACL acl; + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); server.ModifyTopicACL(server.GetTopic(), acl); - - for (const auto &invalidToken : invalidTokens) { - Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; + + for (const auto &invalidToken : invalidTokens) { + Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; MAKE_WRITE_STREAM(validToken); - - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, - serverMessage); - - clientMessage.mutable_update_token_request()->set_token(invalidToken); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - - - UNIT_ASSERT_C(!stream->Read(&serverMessage), - "Expect session close with no messages from the server but got message: " - << serverMessage.DebugString()); - UNIT_ASSERT_EQUAL(grpc::StatusCode::UNAUTHENTICATED, stream->Finish().error_code()); - } - } - + clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, + serverMessage); + + clientMessage.mutable_update_token_request()->set_token(invalidToken); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + + + UNIT_ASSERT_C(!stream->Read(&serverMessage), + "Expect session close with no messages from the server but got message: " + << serverMessage.DebugString()); + UNIT_ASSERT_EQUAL(grpc::StatusCode::UNAUTHENTICATED, stream->Finish().error_code()); + } + } + Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithValidTokenButWithoutACL_SessionClosedWithUnauthorizedError) { TPersQueueV1TestServer server; - const TString validToken = "test_user@" - BUILTIN_ACL_DOMAIN; - const TString invalidToken = "test_user_2@" - BUILTIN_ACL_DOMAIN; - NACLib::TDiffACL acl; - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); - + const TString validToken = "test_user@" + BUILTIN_ACL_DOMAIN; + const TString invalidToken = "test_user_2@" + BUILTIN_ACL_DOMAIN; + NACLib::TDiffACL acl; + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); + server.ModifyTopicACL(server.GetTopic(), acl); - + MAKE_WRITE_STREAM(validToken); - - StreamingWriteClientMessage clientMessage; - StreamingWriteServerMessage serverMessage; + + StreamingWriteClientMessage clientMessage; + StreamingWriteServerMessage serverMessage; clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); - clientMessage.mutable_init_request()->set_message_group_id("test-message-group"); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, - serverMessage); - - - clientMessage.mutable_update_token_request()->set_token(invalidToken); - AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); - - - AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - UNIT_ASSERT_VALUES_EQUAL(Ydb::StatusIds::UNAUTHORIZED, serverMessage.status()); - UNIT_ASSERT_EQUAL_C(StreamingWriteServerMessage::SERVER_MESSAGE_NOT_SET, - serverMessage.server_message_case(), serverMessage); - } - + clientMessage.mutable_init_request()->set_message_group_id("test-message-group"); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_C(serverMessage.server_message_case() == StreamingWriteServerMessage::kInitResponse, + serverMessage); + + + clientMessage.mutable_update_token_request()->set_token(invalidToken); + AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); + + + AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); + UNIT_ASSERT_VALUES_EQUAL(Ydb::StatusIds::UNAUTHORIZED, serverMessage.status()); + UNIT_ASSERT_EQUAL_C(StreamingWriteServerMessage::SERVER_MESSAGE_NOT_SET, + serverMessage.server_message_case(), serverMessage); + } + void TestWriteWithRateLimiter(TPersQueueV1TestServerWithRateLimiter& server) { - const std::vector<TString> differentTopicPathsTypes = { - "account1/topic", // without folder - "account2/folder/topic", // with folder - "account3/folder1/folder2/topic", // complex - }; - const TString data = TString("12345") * 100; - for (const TString &topicPath : differentTopicPathsTypes) { + const std::vector<TString> differentTopicPathsTypes = { + "account1/topic", // without folder + "account2/folder/topic", // with folder + "account3/folder1/folder2/topic", // complex + }; + const TString data = TString("12345") * 100; + for (const TString &topicPath : differentTopicPathsTypes) { server.CreateTopicWithQuota(topicPath); - + auto driver = server.Server->AnnoyingClient->GetDriver(); - + { auto writer = CreateSimpleWriter(*driver, server.TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath, "123"); writer->Write(data); bool res = writer->Close(TDuration::Seconds(10)); UNIT_ASSERT(res); } - } - } - + } + } + Y_UNIT_TEST(TestWriteWithRateLimiterWithBlobsRateLimit) { TPersQueueV1TestServerWithRateLimiter server; server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::WRITTEN_BLOB_SIZE); TestWriteWithRateLimiter(server); - } - + } + Y_UNIT_TEST(TestWriteWithRateLimiterWithUserPayloadRateLimit) { TPersQueueV1TestServerWithRateLimiter server; server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE); TestWriteWithRateLimiter(server); - } - + } + void TestRateLimiterLimitsWrite(TPersQueueV1TestServerWithRateLimiter& server) { - const TString topicPath = "account/topic"; - + const TString topicPath = "account/topic"; + server.CreateTopicWithQuota(topicPath, true, 100.0); - const TString data = TString("123") * 100; // 300 bytes // 3 seconds - + const TString data = TString("123") * 100; // 300 bytes // 3 seconds + auto driver = server.Server->AnnoyingClient->GetDriver(); // Warm up write @@ -322,10 +322,10 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { UNIT_ASSERT(res); } - // will be removed - - const TInstant startWrite = TInstant::Now(); - + // will be removed + + const TInstant startWrite = TInstant::Now(); + { auto writer = CreateSimpleWriter(*driver, server.TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath, "123"); @@ -344,17 +344,17 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { } - const TInstant endWrite = TInstant::Now(); - // Check write time with quota - const TDuration writeTime = endWrite - startWrite; - // in new scheme cache rate limiting is turned off + const TInstant endWrite = TInstant::Now(); + // Check write time with quota + const TDuration writeTime = endWrite - startWrite; + // in new scheme cache rate limiting is turned off if (server.TenantModeEnabled()) { - UNIT_ASSERT_GE_C(TDuration::Seconds(3), writeTime, "Write time: " << writeTime); - } else { - UNIT_ASSERT_GE_C(writeTime, TDuration::Seconds(3), "Write time: " << writeTime); - } - - } + UNIT_ASSERT_GE_C(TDuration::Seconds(3), writeTime, "Write time: " << writeTime); + } else { + UNIT_ASSERT_GE_C(writeTime, TDuration::Seconds(3), "Write time: " << writeTime); + } + + } +} + } - -} diff --git a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp index c33a7ad4f9..8e209dbd9d 100644 --- a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp @@ -1,168 +1,168 @@ -#include "grpc_pq_actor.h" +#include "grpc_pq_actor.h" #include <ydb/services/persqueue_v1/ut/pq_data_writer.h> #include <ydb/services/persqueue_v1/ut/test_utils.h> #include <ydb/services/persqueue_v1/ut/persqueue_test_fixture.h> - + #include <ydb/core/testlib/test_pq_client.h> #include <ydb/core/persqueue/cluster_tracker.h> - + #include <ydb/core/tablet/tablet_counters_aggregator.h> - + #include <ydb/library/aclib/aclib.h> #include <ydb/library/persqueue/obfuscate/obfuscate.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> - -#include <library/cpp/testing/unittest/tests_data.h> -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/json/json_reader.h> - -#include <util/string/join.h> - -#include <grpc++/client_context.h> - + +#include <library/cpp/testing/unittest/tests_data.h> +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/json/json_reader.h> + +#include <util/string/join.h> + +#include <grpc++/client_context.h> + #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> - -namespace { + +namespace { const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; const static TString DEFAULT_TOPIC_PATH = "/Root/PQ/rt3.dc1--topic1"; const static TString SHORT_TOPIC_NAME = "topic1"; -} - - -namespace NKikimr::NPersQueueTests { - - using namespace Tests; - using namespace NKikimrClient; - using namespace Ydb::PersQueue; - using namespace Ydb::PersQueue::V1; - using namespace NThreading; - using namespace NNetClassifier; +} + + +namespace NKikimr::NPersQueueTests { + + using namespace Tests; + using namespace NKikimrClient; + using namespace Ydb::PersQueue; + using namespace Ydb::PersQueue::V1; + using namespace NThreading; + using namespace NNetClassifier; using namespace NYdb::NPersQueue; using namespace NPersQueue; - - NJson::TJsonValue GetCountersNewSchemeCache(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath) { - TString escapedPath = "%2F" + JoinStrings(SplitString(topicPath, "/"), "%2F"); - TString query = TStringBuilder() << "/counters/counters=" << counters - << "/subsystem=" << subsystem - << "/Topic=" << escapedPath << "/json"; - - Cerr << "Will execute query " << query << Endl; - TNetworkAddress addr("localhost", port); - TSocket s(addr); - - SendMinimalHttpRequest(s, "localhost", query); - TSocketInput si(s); - THttpInput input(&si); - Cerr << input.ReadAll() << Endl; - unsigned httpCode = ParseHttpRetCode(input.FirstLine()); - UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); - - NJson::TJsonValue value; - UNIT_ASSERT(NJson::ReadJsonTree(&input, &value)); - - Cerr << "counters: " << value.GetStringRobust() << "\n"; - return value; - } - - Y_UNIT_TEST_SUITE(TPersQueueNewSchemeCacheTest) { - - void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) { + + NJson::TJsonValue GetCountersNewSchemeCache(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath) { + TString escapedPath = "%2F" + JoinStrings(SplitString(topicPath, "/"), "%2F"); + TString query = TStringBuilder() << "/counters/counters=" << counters + << "/subsystem=" << subsystem + << "/Topic=" << escapedPath << "/json"; + + Cerr << "Will execute query " << query << Endl; + TNetworkAddress addr("localhost", port); + TSocket s(addr); + + SendMinimalHttpRequest(s, "localhost", query); + TSocketInput si(s); + THttpInput input(&si); + Cerr << input.ReadAll() << Endl; + unsigned httpCode = ParseHttpRetCode(input.FirstLine()); + UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); + + NJson::TJsonValue value; + UNIT_ASSERT(NJson::ReadJsonTree(&input, &value)); + + Cerr << "counters: " << value.GetStringRobust() << "\n"; + return value; + } + + Y_UNIT_TEST_SUITE(TPersQueueNewSchemeCacheTest) { + + void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) { annoyingClient.SetNoConfigMode(); annoyingClient.FullInit(); - annoyingClient.InitUserRegistry(); - annoyingClient.MkDir("/Root", "account1"); - annoyingClient.MkDir("/Root/PQ", "account1"); + annoyingClient.InitUserRegistry(); + annoyingClient.MkDir("/Root", "account1"); + annoyingClient.MkDir("/Root/PQ", "account1"); annoyingClient.CreateTopicNoLegacy(DEFAULT_TOPIC_PATH, 5, false); annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false); annoyingClient.CreateTopicNoLegacy("/Root/account2/topic2", 5); - } - - Y_UNIT_TEST(CheckGrpcWriteNoDC) { + } + + Y_UNIT_TEST(CheckGrpcWriteNoDC) { TTestServer server(false); server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); - + server.StartServer(); server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE, NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE} ); PrepareForGrpcNoDC(*server.AnnoyingClient); - + TPQDataWriter writer("source1", server, DEFAULT_TOPIC_PATH); - + writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); - writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); - - NACLib::TDiffACL acl; - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN); + writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); + + NACLib::TDiffACL acl; + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString()); server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString()); - - WaitACLModification(); + + WaitACLModification(); writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); - - writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); - writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN); - - } - - Y_UNIT_TEST(CheckGrpcReadNoDC) { + + writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); + writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN); + + } + + Y_UNIT_TEST(CheckGrpcReadNoDC) { TTestServer server(false); server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); server.StartServer(); server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE}); PrepareForGrpcNoDC(*server.AnnoyingClient); NYdb::TDriverConfig driverCfg; - + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg); auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver); - + { auto res = persQueueClient->AddReadRule("/Root/account2/topic2", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user1"))); res.Wait(); UNIT_ASSERT(res.GetValue().IsSuccess()); } - - { - NACLib::TDiffACL acl; - acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user1@" BUILTIN_ACL_DOMAIN); + + { + NACLib::TDiffACL acl; + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user1@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString()); - } - - WaitACLModification(); - - { + } + + WaitACLModification(); + + { auto writer = CreateSimpleWriter(*ydbDriver, "/Root/account2/topic2", "123", 1); - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { bool res = writer->Write(TString(10, 'a')); UNIT_ASSERT(res); - } + } bool res = writer->Close(TDuration::Seconds(10)); UNIT_ASSERT(res); - } - - auto testReadFromTopic = [&](const TString& topicPath) { + } + + auto testReadFromTopic = [&](const TString& topicPath) { NYdb::NPersQueue::TReadSessionSettings settings; settings.ConsumerName("user1").AppendTopics(topicPath); auto reader = CreateReader(*ydbDriver, settings); - + for (int i = 0; i < 4; ++i) { auto msg = GetNextMessageSkipAssignment(reader); UNIT_ASSERT(msg); Cerr << "GOT MESSAGE: " << DebugString(*msg) << "\n"; - } - }; - + } + }; + testReadFromTopic("/Root/account2/topic2"); testReadFromTopic("account2/topic2"); - } - - } - + } + + } + Y_UNIT_TEST_SUITE(TPersqueueDataPlaneTestSuite) { Y_UNIT_TEST(WriteSession) { @@ -241,143 +241,143 @@ namespace NKikimr::NPersQueueTests { auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); - grpc::ClientContext grpcContext; - grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); - auto readStream = stub->MigrationStreamingRead(&grpcContext); - UNIT_ASSERT(readStream); - - // init read session - { - MigrationStreamingReadClientMessage req; - MigrationStreamingReadServerMessage resp; - - req.mutable_init_request()->add_topics_read_settings()->set_topic("topic1"); - - req.mutable_init_request()->set_consumer("user"); - req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(3); - - if (!readStream->Write(req)) { - ythrow yexception() << "write fail"; - } - - UNIT_ASSERT(readStream->Read(&resp)); - UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse); - } - } - + grpc::ClientContext grpcContext; + grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); + auto readStream = stub->MigrationStreamingRead(&grpcContext); + UNIT_ASSERT(readStream); + + // init read session + { + MigrationStreamingReadClientMessage req; + MigrationStreamingReadServerMessage resp; + + req.mutable_init_request()->add_topics_read_settings()->set_topic("topic1"); + + req.mutable_init_request()->set_consumer("user"); + req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(3); + + if (!readStream->Write(req)) { + ythrow yexception() << "write fail"; + } + + UNIT_ASSERT(readStream->Read(&resp)); + UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse); + } + } + Y_UNIT_TEST(SetupWriteLockSessionWithDatabase) { TPersQueueV1TestServer server; auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); - grpc::ClientContext grpcContext; - grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); - - auto writeStream = stub->StreamingWrite(&grpcContext); - UNIT_ASSERT(writeStream); - - { - StreamingWriteClientMessage req; - StreamingWriteServerMessage resp; - - req.mutable_init_request()->set_topic("topic1"); - req.mutable_init_request()->set_message_group_id("12345678"); - if (!writeStream->Write(req)) { - ythrow yexception() << "write fail"; - } - - UNIT_ASSERT(writeStream->Read(&resp)); - UNIT_ASSERT(resp.has_init_response()); - } - } - + grpc::ClientContext grpcContext; + grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); + + auto writeStream = stub->StreamingWrite(&grpcContext); + UNIT_ASSERT(writeStream); + + { + StreamingWriteClientMessage req; + StreamingWriteServerMessage resp; + + req.mutable_init_request()->set_topic("topic1"); + req.mutable_init_request()->set_message_group_id("12345678"); + if (!writeStream->Write(req)) { + ythrow yexception() << "write fail"; + } + + UNIT_ASSERT(writeStream->Read(&resp)); + UNIT_ASSERT(resp.has_init_response()); + } + } + Y_UNIT_TEST(TestAddRemoveReadRule) { TPersQueueV1TestServer server; SET_LOCALS; - + pqClient->CreateConsumer("goodUser"); - + TString path = server.GetTopicPath(); - Ydb::PersQueue::V1::AddReadRuleRequest addRuleRequest; - Ydb::PersQueue::V1::AddReadRuleResponse addRuleResponse; - addRuleRequest.set_path(path); - + Ydb::PersQueue::V1::AddReadRuleRequest addRuleRequest; + Ydb::PersQueue::V1::AddReadRuleResponse addRuleResponse; + addRuleRequest.set_path(path); + auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); - { - grpc::ClientContext grpcContext; - grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); - addRuleRequest.set_path("topic1"); - auto* rr = addRuleRequest.mutable_read_rule(); - rr->set_consumer_name("goodUser"); + { + grpc::ClientContext grpcContext; + grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); + addRuleRequest.set_path("topic1"); + auto* rr = addRuleRequest.mutable_read_rule(); + rr->set_consumer_name("goodUser"); rr->set_version(0); - rr->set_important(true); - rr->set_supported_format(TopicSettings::FORMAT_BASE); - rr->add_supported_codecs(CODEC_ZSTD); - auto status = stub->AddReadRule(&grpcContext, addRuleRequest, &addRuleResponse); + rr->set_important(true); + rr->set_supported_format(TopicSettings::FORMAT_BASE); + rr->add_supported_codecs(CODEC_ZSTD); + auto status = stub->AddReadRule(&grpcContext, addRuleRequest, &addRuleResponse); Cerr << "ADD RR RESPONSE " << addRuleResponse << "\n"; - UNIT_ASSERT(status.ok() && addRuleResponse.operation().status() == Ydb::StatusIds::SUCCESS); - addRuleRequest.set_path(path); - } - - // don't allow add the same read rule twice - { - grpc::ClientContext grpcContext; - auto status = stub->AddReadRule(&grpcContext, addRuleRequest, &addRuleResponse); - UNIT_ASSERT(status.ok() && addRuleResponse.operation().status() == Ydb::StatusIds::ALREADY_EXISTS); - } - - Ydb::PersQueue::V1::RemoveReadRuleRequest removeRuleRequest; - Ydb::PersQueue::V1::RemoveReadRuleResponse removeRuleResponse; - removeRuleRequest.set_path(path); - removeRuleRequest.set_consumer_name("badUser"); - - // trying to remove user that not exist - { - grpc::ClientContext grpcContext; - auto status = stub->RemoveReadRule(&grpcContext, removeRuleRequest, &removeRuleResponse); - UNIT_ASSERT(status.ok() && removeRuleResponse.operation().status() == Ydb::StatusIds::NOT_FOUND); - } + UNIT_ASSERT(status.ok() && addRuleResponse.operation().status() == Ydb::StatusIds::SUCCESS); + addRuleRequest.set_path(path); + } + + // don't allow add the same read rule twice + { + grpc::ClientContext grpcContext; + auto status = stub->AddReadRule(&grpcContext, addRuleRequest, &addRuleResponse); + UNIT_ASSERT(status.ok() && addRuleResponse.operation().status() == Ydb::StatusIds::ALREADY_EXISTS); + } + + Ydb::PersQueue::V1::RemoveReadRuleRequest removeRuleRequest; + Ydb::PersQueue::V1::RemoveReadRuleResponse removeRuleResponse; + removeRuleRequest.set_path(path); + removeRuleRequest.set_consumer_name("badUser"); + + // trying to remove user that not exist + { + grpc::ClientContext grpcContext; + auto status = stub->RemoveReadRule(&grpcContext, removeRuleRequest, &removeRuleResponse); + UNIT_ASSERT(status.ok() && removeRuleResponse.operation().status() == Ydb::StatusIds::NOT_FOUND); + } auto findReadRule = [&](const TString& consumerName, const TMaybe<i64> version, const TopicSettings& settings) { - for (const auto& rr : settings.read_rules()) { - if (rr.consumer_name() == consumerName) { + for (const auto& rr : settings.read_rules()) { + if (rr.consumer_name() == consumerName) { Cerr << rr << "\n"; return !version || rr.version() == *version; - } - } - return false; - }; - - Ydb::PersQueue::V1::DescribeTopicRequest describeTopicRequest; - Ydb::PersQueue::V1::DescribeTopicResponse describeTopicResponse; - describeTopicRequest.set_path(path); - { - grpc::ClientContext grpcContext; - grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); - addRuleRequest.set_path("topic1"); - auto status = stub->DescribeTopic(&grpcContext, describeTopicRequest, &describeTopicResponse); - Ydb::PersQueue::V1::DescribeTopicResult res; - UNIT_ASSERT(status.ok() && describeTopicResponse.operation().status() == Ydb::StatusIds::SUCCESS); - describeTopicResponse.operation().result().UnpackTo(&res); + } + } + return false; + }; + + Ydb::PersQueue::V1::DescribeTopicRequest describeTopicRequest; + Ydb::PersQueue::V1::DescribeTopicResponse describeTopicResponse; + describeTopicRequest.set_path(path); + { + grpc::ClientContext grpcContext; + grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); + addRuleRequest.set_path("topic1"); + auto status = stub->DescribeTopic(&grpcContext, describeTopicRequest, &describeTopicResponse); + Ydb::PersQueue::V1::DescribeTopicResult res; + UNIT_ASSERT(status.ok() && describeTopicResponse.operation().status() == Ydb::StatusIds::SUCCESS); + describeTopicResponse.operation().result().UnpackTo(&res); UNIT_ASSERT(findReadRule("goodUser", 1, res.settings())); - addRuleRequest.set_path(path); - } - - { - grpc::ClientContext grpcContext; - grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); - removeRuleRequest.set_consumer_name("goodUser"); - auto status = stub->RemoveReadRule(&grpcContext, removeRuleRequest, &removeRuleResponse); - Cerr << removeRuleResponse.ShortDebugString() << Endl; - UNIT_ASSERT(status.ok() && removeRuleResponse.operation().status() == Ydb::StatusIds::SUCCESS); - removeRuleRequest.set_path(path); - } - { - grpc::ClientContext grpcContext; - stub->DescribeTopic(&grpcContext, describeTopicRequest, &describeTopicResponse); - Ydb::PersQueue::V1::DescribeTopicResult res; - describeTopicResponse.operation().result().UnpackTo(&res); + addRuleRequest.set_path(path); + } + + { + grpc::ClientContext grpcContext; + grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); + removeRuleRequest.set_consumer_name("goodUser"); + auto status = stub->RemoveReadRule(&grpcContext, removeRuleRequest, &removeRuleResponse); + Cerr << removeRuleResponse.ShortDebugString() << Endl; + UNIT_ASSERT(status.ok() && removeRuleResponse.operation().status() == Ydb::StatusIds::SUCCESS); + removeRuleRequest.set_path(path); + } + { + grpc::ClientContext grpcContext; + stub->DescribeTopic(&grpcContext, describeTopicRequest, &describeTopicResponse); + Ydb::PersQueue::V1::DescribeTopicResult res; + describeTopicResponse.operation().result().UnpackTo(&res); UNIT_ASSERT(!findReadRule("goodUser", {}, res.settings())); - } + } { grpc::ClientContext grpcContext; @@ -401,6 +401,6 @@ namespace NKikimr::NPersQueueTests { UNIT_ASSERT(findReadRule("goodUser", 3, res.settings())); // version is 3 : add, remove and add } - } - } -} + } + } +} diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index 8db7b21357..3bc9b21728 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -38,7 +38,7 @@ #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> -namespace NKikimr::NPersQueueTests { +namespace NKikimr::NPersQueueTests { using namespace Tests; using namespace NKikimrClient; @@ -241,16 +241,16 @@ namespace { auto driver = server.Server->AnnoyingClient->GetDriver(); - { + { auto writer = CreateSimpleWriter(*driver, "acc/topic1", "source"); for (int i = 1; i < 17; ++i) { bool res = writer->Write("valuevaluevalue" + ToString(i), i); UNIT_ASSERT(res); - } + } bool res = writer->Close(TDuration::Seconds(10)); UNIT_ASSERT(res); - } - + } + //check read results MigrationStreamingReadServerMessage resp; for (ui32 i = 10; i < 16; ++i) { @@ -296,7 +296,7 @@ namespace { UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1); UNIT_ASSERT_VALUES_EQUAL(res.topics(0).partitions_size(), 10); } - + { ReadInfoRequest request; ReadInfoResponse response; @@ -2597,7 +2597,7 @@ namespace { DescribeTopicResult res; response.operation().result().UnpackTo(&res); Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); } { diff --git a/ydb/services/persqueue_v1/persqueue_utils.cpp b/ydb/services/persqueue_v1/persqueue_utils.cpp index 289b832c00..51d7f38045 100644 --- a/ydb/services/persqueue_v1/persqueue_utils.cpp +++ b/ydb/services/persqueue_v1/persqueue_utils.cpp @@ -1,15 +1,15 @@ -#include "persqueue_utils.h" - +#include "persqueue_utils.h" + #include <ydb/core/base/path.h> namespace NKikimr::NGRpcProxy::V1 { - + TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl) : AclOldSchemeCache(std::move(acl)) { Y_VERIFY(AclOldSchemeCache); } - + TAclWrapper::TAclWrapper(TIntrusivePtr<TSecurityObject> acl) : AclNewSchemeCache(std::move(acl)) { @@ -21,9 +21,9 @@ bool TAclWrapper::CheckAccess(NACLib::EAccessRights rights, const NACLib::TUserT return AclOldSchemeCache->CheckAccess(rights, userToken); } else { return AclNewSchemeCache->CheckAccess(rights, userToken); - } + } } - + using namespace NSchemeCache; TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEntry& entry) { @@ -37,7 +37,7 @@ TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEnt fullPath.c_str(), entry.Path[0].c_str()), true }; - } + } case TSchemeCacheNavigate::EStatus::PathErrorUnknown: { return TProcessingResult { Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, @@ -55,8 +55,8 @@ TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEnt true }; } - } - + } + if (entry.Kind != TSchemeCacheNavigate::KindTopic) { return TProcessingResult { Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, @@ -81,6 +81,6 @@ TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEnt }; } return {}; -} +} } // namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/persqueue_utils.h b/ydb/services/persqueue_v1/persqueue_utils.h index 1ef19eafcc..7860067982 100644 --- a/ydb/services/persqueue_v1/persqueue_utils.h +++ b/ydb/services/persqueue_v1/persqueue_utils.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h> #include <ydb/library/aclib/aclib.h> #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/base/counters.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> - + namespace NKikimr::NGRpcProxy::V1 { - + class TAclWrapper { public: TAclWrapper(THolder<NACLib::TSecurityObject>); TAclWrapper(TIntrusivePtr<TSecurityObject>); - + bool CheckAccess(NACLib::EAccessRights, const NACLib::TUserToken& userToken); - + private: THolder<NACLib::TSecurityObject> AclOldSchemeCache; TIntrusivePtr<TSecurityObject> AclNewSchemeCache; }; - + struct TProcessingResult { Ydb::PersQueue::ErrorCode::ErrorCode ErrorCode; TString Reason; diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make index ffc87f00f0..2a8fc402a9 100644 --- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make +++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make @@ -1,40 +1,40 @@ UNITTEST_FOR(ydb/services/persqueue_v1) -OWNER( - zhenyok +OWNER( + zhenyok g:kikimr g:logbroker -) - +) + CFLAGS( -DACTORLIB_HUGE_PB_SIZE ) - -FORK_SUBTESTS() -IF (WITH_VALGRIND) - TIMEOUT(1800) - SIZE(LARGE) - TAG(ya:fat) +FORK_SUBTESTS() + +IF (WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) REQUIREMENTS(ram:32) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -ENV(PERSQUEUE_NEW_SCHEMECACHE="true") - -SRCS( - persqueue_new_schemecache_ut.cpp - persqueue_common_ut.cpp - ut/api_test_setup.h - ut/test_utils.h - ut/pq_data_writer.h - ut/rate_limiter_test_setup.h - ut/rate_limiter_test_setup.cpp -) - -PEERDIR( +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +ENV(PERSQUEUE_NEW_SCHEMECACHE="true") + +SRCS( + persqueue_new_schemecache_ut.cpp + persqueue_common_ut.cpp + ut/api_test_setup.h + ut/test_utils.h + ut/pq_data_writer.h + ut/rate_limiter_test_setup.h + ut/rate_limiter_test_setup.cpp +) + +PEERDIR( library/cpp/getopt library/cpp/svnversion ydb/core/testlib @@ -43,8 +43,8 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils ydb/public/sdk/cpp/client/ydb_table ydb/services/persqueue_v1 -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h index 90ab7939a4..2933d1f3a1 100644 --- a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h +++ b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h @@ -1,21 +1,21 @@ -#pragma once - -#include "test_utils.h" - +#pragma once + +#include "test_utils.h" + #include <ydb/core/testlib/test_pq_client.h> - + #include <ydb/library/aclib/aclib.h> - - + + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h> - -#include <util/system/env.h> - -namespace NKikimr::NPersQueueTests { - + +#include <util/system/env.h> + +namespace NKikimr::NPersQueueTests { + #define SET_LOCALS \ @@ -25,15 +25,15 @@ namespace NKikimr::NPersQueueTests { Y_UNUSED(runtime); \ - using namespace Tests; - using namespace NKikimrClient; - using namespace Ydb::PersQueue; - using namespace Ydb::PersQueue::V1; - using namespace NThreading; - using namespace NNetClassifier; - + using namespace Tests; + using namespace NKikimrClient; + using namespace Ydb::PersQueue; + using namespace Ydb::PersQueue::V1; + using namespace NThreading; + using namespace NNetClassifier; + class TPersQueueV1TestServerBase { - public: + public: virtual void AlterSettings(NKikimr::Tests::TServerSettings& settings) { Y_UNUSED(settings); } @@ -46,22 +46,22 @@ namespace NKikimr::NPersQueueTests { Server->ServerSettings.PQConfig.SetMetaCacheTimeoutSec(30); AlterSettings(Server->ServerSettings); Server->StartServer(false); - if (TenantModeEnabled()) { + if (TenantModeEnabled()) { Server->AnnoyingClient->SetNoConfigMode(); Server->ServerSettings.PQConfig.SetSourceIdTablePath("some unused path"); - } + } Cerr << "Init PQ - start server on port " << Server->GrpcPort << Endl; Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024); - EnablePQLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::FLAT_TX_SCHEMESHARD}); + EnablePQLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::FLAT_TX_SCHEMESHARD}); EnablePQLogs({NKikimrServices::PERSQUEUE}, NLog::EPriority::PRI_INFO); - + Server->AnnoyingClient->FullInit(); Server->AnnoyingClient->CreateConsumer("user"); - if (TenantModeEnabled()) { + if (TenantModeEnabled()) { Cerr << "=== Will create fst-class topics\n"; Server->AnnoyingClient->CreateTopicNoLegacy("/Root/acc/topic1", 1); Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/acc/topic1", 1); - } else { + } else { Cerr << "=== Will create legacy-style topics\n"; Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic2dc", 1); Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc2--acc--topic2dc", 1, true, false); @@ -69,114 +69,114 @@ namespace NKikimr::NPersQueueTests { Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic1", 1); Server->WaitInit("topic1"); Sleep(TDuration::Seconds(10)); - } - + } + InsecureChannel = grpc::CreateChannel("localhost:" + ToString(Server->GrpcPort), grpc::InsecureChannelCredentials()); - ServiceStub = Ydb::PersQueue::V1::PersQueueService::NewStub(InsecureChannel); - InitializeWritePQService(TenantModeEnabled() ? "Root/acc/topic1" : "topic1"); + ServiceStub = Ydb::PersQueue::V1::PersQueueService::NewStub(InsecureChannel); + InitializeWritePQService(TenantModeEnabled() ? "Root/acc/topic1" : "topic1"); NYdb::TDriverConfig driverCfg; driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << Server->GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); YdbDriver.reset(new NYdb::TDriver(driverCfg)); PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*YdbDriver); - } - - void EnablePQLogs(const TVector<NKikimrServices::EServiceKikimr> services, - NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) - { - for (auto s : services) { + } + + void EnablePQLogs(const TVector<NKikimrServices::EServiceKikimr> services, + NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) + { + for (auto s : services) { Server->CleverServer->GetRuntime()->SetLogPriority(s, prio); - } - } - - void InitializeWritePQService(const TString &topicToWrite) { - while (true) { - Sleep(TDuration::MilliSeconds(100)); - - Ydb::PersQueue::V1::StreamingWriteClientMessage req; - Ydb::PersQueue::V1::StreamingWriteServerMessage resp; - grpc::ClientContext context; - - auto stream = ServiceStub->StreamingWrite(&context); - UNIT_ASSERT(stream); - - req.mutable_init_request()->set_topic(topicToWrite); - req.mutable_init_request()->set_message_group_id("12345678"); - - if (!stream->Write(req)) { - UNIT_ASSERT_C(stream->Read(&resp), "Context error: " << context.debug_error_string()); - UNIT_ASSERT_C(resp.status() == Ydb::StatusIds::UNAVAILABLE, - "Response: " << resp << ", Context error: " << context.debug_error_string()); - continue; - } - - AssertSuccessfullStreamingOperation(stream->Read(&resp), stream); - if (resp.status() == Ydb::StatusIds::UNAVAILABLE) { - continue; - } - - if (stream->WritesDone()) { - auto status = stream->Finish(); - Cerr << "Finish: " << (int) status.error_code() << " " << status.error_message() << "\n"; - } - - break; - } - } - - public: - static bool TenantModeEnabled() { - return !GetEnv("PERSQUEUE_NEW_SCHEMECACHE").empty(); - } - - void ModifyTopicACL(const TString& topic, const NACLib::TDiffACL& acl) { - if (TenantModeEnabled()) { - TFsPath path(topic); + } + } + + void InitializeWritePQService(const TString &topicToWrite) { + while (true) { + Sleep(TDuration::MilliSeconds(100)); + + Ydb::PersQueue::V1::StreamingWriteClientMessage req; + Ydb::PersQueue::V1::StreamingWriteServerMessage resp; + grpc::ClientContext context; + + auto stream = ServiceStub->StreamingWrite(&context); + UNIT_ASSERT(stream); + + req.mutable_init_request()->set_topic(topicToWrite); + req.mutable_init_request()->set_message_group_id("12345678"); + + if (!stream->Write(req)) { + UNIT_ASSERT_C(stream->Read(&resp), "Context error: " << context.debug_error_string()); + UNIT_ASSERT_C(resp.status() == Ydb::StatusIds::UNAVAILABLE, + "Response: " << resp << ", Context error: " << context.debug_error_string()); + continue; + } + + AssertSuccessfullStreamingOperation(stream->Read(&resp), stream); + if (resp.status() == Ydb::StatusIds::UNAVAILABLE) { + continue; + } + + if (stream->WritesDone()) { + auto status = stream->Finish(); + Cerr << "Finish: " << (int) status.error_code() << " " << status.error_message() << "\n"; + } + + break; + } + } + + public: + static bool TenantModeEnabled() { + return !GetEnv("PERSQUEUE_NEW_SCHEMECACHE").empty(); + } + + void ModifyTopicACL(const TString& topic, const NACLib::TDiffACL& acl) { + if (TenantModeEnabled()) { + TFsPath path(topic); Server->AnnoyingClient->ModifyACL(path.Dirname(), path.Basename(), acl.SerializeAsString()); - } else { + } else { Server->AnnoyingClient->ModifyACL("/Root/PQ", topic, acl.SerializeAsString()); - } - WaitACLModification(); - - } - - TString GetRoot() const { - return !TenantModeEnabled() ? "/Root/PQ" : ""; - } - - TString GetTopic() { - return TenantModeEnabled() ? "/Root/acc/topic1" : "rt3.dc1--topic1"; - } - - TString GetTopicPath() { - return TenantModeEnabled() ? "/Root/acc/topic1" : "topic1"; - } - - TString GetTopicPathMultipleDC() const { - return "acc/topic2dc"; - } - - public: + } + WaitACLModification(); + + } + + TString GetRoot() const { + return !TenantModeEnabled() ? "/Root/PQ" : ""; + } + + TString GetTopic() { + return TenantModeEnabled() ? "/Root/acc/topic1" : "rt3.dc1--topic1"; + } + + TString GetTopicPath() { + return TenantModeEnabled() ? "/Root/acc/topic1" : "topic1"; + } + + TString GetTopicPathMultipleDC() const { + return "acc/topic2dc"; + } + + public: THolder<NPersQueue::TTestServer> Server; TSimpleSharedPtr<TPortManager> PortManager; - std::shared_ptr<grpc::Channel> InsecureChannel; - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> ServiceStub; + std::shared_ptr<grpc::Channel> InsecureChannel; + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> ServiceStub; std::shared_ptr<NYdb::TDriver> YdbDriver; THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; - }; - + }; + class TPersQueueV1TestServer : public TPersQueueV1TestServerBase { - public: + public: TPersQueueV1TestServer(bool checkAcl = false) : CheckACL(checkAcl) { - InitAll(); - } - - void InitAll() { + InitAll(); + } + + void InitAll() { InitializePQ(); - } + } void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { if (CheckACL) @@ -184,86 +184,86 @@ namespace NKikimr::NPersQueueTests { } private: bool CheckACL; - }; - + }; + class TPersQueueV1TestServerWithRateLimiter : public TPersQueueV1TestServerBase { private: NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity; - public: + public: TPersQueueV1TestServerWithRateLimiter() : TPersQueueV1TestServerBase() {} - + void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity); } - void InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity) { + void InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity) { LimitedEntity = limitedEntity; InitializePQ(); - InitQuotingPaths(); - } - - void InitQuotingPaths() { + InitQuotingPaths(); + } + + void InitQuotingPaths() { Server->AnnoyingClient->MkDir("/Root", "PersQueue"); Server->AnnoyingClient->MkDir("/Root/PersQueue", "System"); Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters"); - } - - void CreateTopicWithQuota(const TString& path, bool createKesus = true, double writeQuota = 1000.0) { - TVector<TString> pathComponents = SplitPath(path); - const TString account = pathComponents[0]; + } + + void CreateTopicWithQuota(const TString& path, bool createKesus = true, double writeQuota = 1000.0) { + TVector<TString> pathComponents = SplitPath(path); + const TString account = pathComponents[0]; const TString name = NPersQueue::BuildFullTopicName(path, "dc1"); - - if (TenantModeEnabled()) { + + if (TenantModeEnabled()) { Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/" + path, 1); - } else { - Cerr << "Creating topic \"" << name << "\"" << Endl; + } else { + Cerr << "Creating topic \"" << name << "\"" << Endl; Server->AnnoyingClient->CreateTopicNoLegacy(name, 1); - } - - const TString rootPath = "/Root/PersQueue/System/Quoters"; - const TString kesusPath = TStringBuilder() << rootPath << "/" << account; - - if (createKesus) { - Cerr << "Creating kesus \"" << account << "\"" << Endl; + } + + const TString rootPath = "/Root/PersQueue/System/Quoters"; + const TString kesusPath = TStringBuilder() << rootPath << "/" << account; + + if (createKesus) { + Cerr << "Creating kesus \"" << account << "\"" << Endl; const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(rootPath, account); - UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); - + UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); + const auto statusCode = Server->AnnoyingClient->AddQuoterResource( Server->CleverServer->GetRuntime(), kesusPath, "write-quota", writeQuota ); - UNIT_ASSERT_EQUAL_C(statusCode, Ydb::StatusIds::SUCCESS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); - } - - if (pathComponents.size() > 1) { // account (first component) + path of topic - TStringBuilder prefixPath; // path without account - prefixPath << "write-quota"; - for (auto currentComponent = pathComponents.begin() + 1; currentComponent != pathComponents.end(); ++currentComponent) { - prefixPath << "/" << *currentComponent; - Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; + UNIT_ASSERT_EQUAL_C(statusCode, Ydb::StatusIds::SUCCESS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); + } + + if (pathComponents.size() > 1) { // account (first component) + path of topic + TStringBuilder prefixPath; // path without account + prefixPath << "write-quota"; + for (auto currentComponent = pathComponents.begin() + 1; currentComponent != pathComponents.end(); ++currentComponent) { + prefixPath << "/" << *currentComponent; + Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; const auto statusCode = Server->AnnoyingClient->AddQuoterResource( Server->CleverServer->GetRuntime(), kesusPath, prefixPath ); - UNIT_ASSERT_C(statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); - } - } - } + UNIT_ASSERT_C(statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); + } + } + } /* - THolder<IProducer> StartProducer(const TString& topicPath, bool compress = false) { - TString fullPath = TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath; - TProducerSettings producerSettings; + THolder<IProducer> StartProducer(const TString& topicPath, bool compress = false) { + TString fullPath = TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath; + TProducerSettings producerSettings; producerSettings.Server = TServerSetting("localhost", Server->GrpcPort); - producerSettings.Topic = fullPath; - producerSettings.SourceId = "TRateLimiterTestSetupSourceId"; - producerSettings.Codec = compress ? "gzip" : "raw"; - THolder<IProducer> producer = PQLib->CreateProducer(producerSettings); - auto startResult = producer->Start(); - UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, startResult.GetValueSync().Response.status(), "Response: " << startResult.GetValueSync().Response); - return producer; - } + producerSettings.Topic = fullPath; + producerSettings.SourceId = "TRateLimiterTestSetupSourceId"; + producerSettings.Codec = compress ? "gzip" : "raw"; + THolder<IProducer> producer = PQLib->CreateProducer(producerSettings); + auto startResult = producer->Start(); + UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, startResult.GetValueSync().Response.status(), "Response: " << startResult.GetValueSync().Response); + return producer; + } */ - }; + }; } diff --git a/ydb/services/persqueue_v1/ut/pq_data_writer.h b/ydb/services/persqueue_v1/ut/pq_data_writer.h index 9264159139..caef894060 100644 --- a/ydb/services/persqueue_v1/ut/pq_data_writer.h +++ b/ydb/services/persqueue_v1/ut/pq_data_writer.h @@ -20,7 +20,7 @@ public: , Runtime(server.CleverServer->GetRuntime()) { InitializeChannel(); - WaitWritePQServiceInitialization(testTopicPath); + WaitWritePQServiceInitialization(testTopicPath); } void Read(const TString& topic, const TString& clientId, const TString& ticket = "", bool error = false, bool checkACL = false, bool onlyCreate = false) { @@ -153,7 +153,7 @@ public: } } - void WaitWritePQServiceInitialization(const TString& testTopicPath = "topic1") { + void WaitWritePQServiceInitialization(const TString& testTopicPath = "topic1") { bool tried = false; while (true) { if (tried) { @@ -168,8 +168,8 @@ public: auto stream = StubP_->StreamingWrite(&context); UNIT_ASSERT(stream); - - req.mutable_init_request()->set_topic(testTopicPath); + + req.mutable_init_request()->set_topic(testTopicPath); req.mutable_init_request()->set_message_group_id("12345678"); if (!stream->Write(req)) { diff --git a/ydb/services/persqueue_v1/ut/test_utils.h b/ydb/services/persqueue_v1/ut/test_utils.h index c52792b1c6..c02308b0d7 100644 --- a/ydb/services/persqueue_v1/ut/test_utils.h +++ b/ydb/services/persqueue_v1/ut/test_utils.h @@ -11,15 +11,15 @@ #define TEST_CASE_NAME (TypeName(*this).rfind("TTestCase") != TString::npos ? TypeName(*this).substr(TypeName(*this).rfind("TTestCase") + 9) : TypeName(*this)) -static constexpr int DEBUG_LOG_LEVEL = 7; - +static constexpr int DEBUG_LOG_LEVEL = 7; + using namespace NKikimr::NPersQueueTests; -inline void WaitACLModification() { - // TODO: Tests are flacky without sleep after ModifyACL. Can we cleanly await modified ACLs without random waits? Or at least poll for ACL changes. - Sleep(TDuration::Seconds(5)); -} - +inline void WaitACLModification() { + // TODO: Tests are flacky without sleep after ModifyACL. Can we cleanly await modified ACLs without random waits? Or at least poll for ACL changes. + Sleep(TDuration::Seconds(5)); +} + // TODO: Remove and replace all usage with ApiTestSetup #define SETUP_API_TEST_PREREQUISITES()\ const TString topic = "topic1";\ @@ -78,4 +78,4 @@ void AssertStreamingSessionDead(std::unique_ptr<grpc::ClientReaderWriter<TClient auto actualErrorCode = static_cast<Ydb::PersQueue::ErrorCode::ErrorCode>(serverMessage.issues(0).issue_code()); UNIT_ASSERT_C(expectedErrorCode == actualErrorCode, serverMessage); } - + diff --git a/ydb/services/persqueue_v1/ut/ya.make b/ydb/services/persqueue_v1/ut/ya.make index bfc6e3d64c..70c088383d 100644 --- a/ydb/services/persqueue_v1/ut/ya.make +++ b/ydb/services/persqueue_v1/ut/ya.make @@ -12,7 +12,7 @@ CFLAGS( FORK_SUBTESTS() -IF (SANITIZER_TYPE OR WITH_VALGRIND) +IF (SANITIZER_TYPE OR WITH_VALGRIND) TIMEOUT(1800) SIZE(LARGE) TAG(ya:fat) @@ -24,7 +24,7 @@ ENDIF() SRCS( persqueue_ut.cpp - persqueue_common_ut.cpp + persqueue_common_ut.cpp test_utils.h pq_data_writer.h api_test_setup.h diff --git a/ydb/services/persqueue_v1/ya.make b/ydb/services/persqueue_v1/ya.make index 4a951c2846..ade0406fcc 100644 --- a/ydb/services/persqueue_v1/ya.make +++ b/ydb/services/persqueue_v1/ya.make @@ -19,8 +19,8 @@ SRCS( grpc_pq_write.h persqueue.cpp persqueue.h - persqueue_utils.cpp - persqueue_utils.h + persqueue_utils.cpp + persqueue_utils.h ) PEERDIR( diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index 08b5fe14dc..c067928c78 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -79,7 +79,7 @@ public: ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); ServerSettings->SetEnableYq(enableYq); ServerSettings->Formats = new TFormatFactory; - ServerSettings->PQConfig = appConfig.GetPQConfig(); + ServerSettings->PQConfig = appConfig.GetPQConfig(); if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) { ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath()); } diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 0e76f0112b..57c282b9f4 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -107,7 +107,7 @@ class KikimrConfigGenerator(object): enable_metering=False, grpc_tls_data_path=None, yql_config_path=None, - enable_datastreams=False, + enable_datastreams=False, auth_config_path=None, disable_mvcc=False, enable_public_api_external_blobs=False, |