diff options
author | alexalanov <alexalanov@yandex-team.ru> | 2022-02-10 16:50:45 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:45 +0300 |
commit | 9a901118f2934da16fdb24f8d314e1c370ca611f (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 17da6ebd947b47279482761a7fc6a1ebf0c60855 (diff) | |
download | ydb-9a901118f2934da16fdb24f8d314e1c370ca611f.tar.gz |
Restoring authorship annotation for <alexalanov@yandex-team.ru>. Commit 2 of 2.
79 files changed, 2960 insertions, 2960 deletions
diff --git a/library/cpp/actors/core/log.cpp b/library/cpp/actors/core/log.cpp index 921f3736464..5f63b5af580 100644 --- a/library/cpp/actors/core/log.cpp +++ b/library/cpp/actors/core/log.cpp @@ -650,13 +650,13 @@ namespace NActors { } const char* TLoggerActor::FormatLocalTimestamp(TInstant time, char* buf) { - struct tm localTime; - time.LocalTime(&localTime); + struct tm localTime; + time.LocalTime(&localTime); int r = strftime(buf, TimeBufSize, "%Y-%m-%d-%H-%M-%S", &localTime); Y_VERIFY(r != 0); return buf; - } - + } + TAutoPtr<TLogBackend> CreateSysLogBackend(const TString& ident, bool logPError, bool logCons) { int flags = 0; diff --git a/library/cpp/actors/core/log_settings.cpp b/library/cpp/actors/core/log_settings.cpp index 39ab30784d2..f52f2fc5d22 100644 --- a/library/cpp/actors/core/log_settings.cpp +++ b/library/cpp/actors/core/log_settings.cpp @@ -19,7 +19,7 @@ namespace NActors { , DefPriority(defPriority) , DefSamplingPriority(defSamplingPriority) , DefSamplingRate(defSamplingRate) - , UseLocalTimestamps(false) + , UseLocalTimestamps(false) , Format(PLAIN_FULL_FORMAT) , ShortHostName("") , ClusterName("") @@ -41,7 +41,7 @@ namespace NActors { , DefPriority(defPriority) , DefSamplingPriority(defSamplingPriority) , DefSamplingRate(defSamplingRate) - , UseLocalTimestamps(false) + , UseLocalTimestamps(false) , Format(PLAIN_FULL_FORMAT) , ShortHostName("") , ClusterName("") @@ -51,12 +51,12 @@ namespace NActors { void TSettings::Append(EComponent minVal, EComponent maxVal, EComponentToStringFunc func) { Y_VERIFY(minVal >= 0, "NLog::TSettings: minVal must be non-negative"); Y_VERIFY(maxVal > minVal, "NLog::TSettings: maxVal must be greater than minVal"); - + // update bounds if (!MaxVal || minVal < MinVal) { MinVal = minVal; } - + if (!MaxVal || maxVal > MaxVal) { MaxVal = maxVal; @@ -77,7 +77,7 @@ namespace NActors { } ComponentNames.resize(Mask + 1); - } + } // assign new names but validate if newly added members were not used before for (int i = minVal; i <= maxVal; i++) { @@ -209,10 +209,10 @@ namespace NActors { ThrottleDelay = value; } - void TSettings::SetUseLocalTimestamps(bool value) { - UseLocalTimestamps = value; - } - + void TSettings::SetUseLocalTimestamps(bool value) { + UseLocalTimestamps = value; + } + EComponent TSettings::FindComponent(const TStringBuf& componentName) const { if (componentName.empty()) return InvalidComponent; diff --git a/library/cpp/actors/core/log_settings.h b/library/cpp/actors/core/log_settings.h index d9a3f90c163..7fe4504edd9 100644 --- a/library/cpp/actors/core/log_settings.h +++ b/library/cpp/actors/core/log_settings.h @@ -82,7 +82,7 @@ namespace NActors { EPriority DefPriority; EPriority DefSamplingPriority; ui32 DefSamplingRate; - bool UseLocalTimestamps; + bool UseLocalTimestamps; enum ELogFormat { PLAIN_FULL_FORMAT, @@ -163,7 +163,7 @@ namespace NActors { bool IsValidComponent(EComponent component); void SetAllowDrop(bool val); void SetThrottleDelay(TDuration value); - void SetUseLocalTimestamps(bool value); + void SetUseLocalTimestamps(bool value); private: int SetLevelImpl( diff --git a/library/cpp/actors/core/mon.h b/library/cpp/actors/core/mon.h index d62ef58f86b..c450f2338eb 100644 --- a/library/cpp/actors/core/mon.h +++ b/library/cpp/actors/core/mon.h @@ -13,7 +13,7 @@ namespace NActors { RemoteHttpInfo, RemoteHttpInfoRes, RemoteJsonInfoRes, - RemoteBinaryInfoRes, + RemoteBinaryInfoRes, End }; @@ -200,22 +200,22 @@ namespace NActors { struct TEvRemoteBinaryInfoRes: public NActors::TEventBase<TEvRemoteBinaryInfoRes, RemoteBinaryInfoRes> { TEvRemoteBinaryInfoRes() { } - + TEvRemoteBinaryInfoRes(const TString& blob) - : Blob(blob) + : Blob(blob) { } - + TString Blob; - + TString ToStringHeader() const override { - return "TEvRemoteBinaryInfoRes"; - } - + return "TEvRemoteBinaryInfoRes"; + } + bool SerializeToArcadiaStream(TChunkSerializer *serializer) const override { return serializer->WriteString(&Blob); - } - + } + ui32 CalculateSerializedSize() const override { return Blob.size(); } @@ -224,11 +224,11 @@ namespace NActors { return true; } - static IEventBase* Load(TEventSerializedData* bufs) { + static IEventBase* Load(TEventSerializedData* bufs) { return new TEvRemoteBinaryInfoRes(bufs->GetString()); - } - }; - + } + }; + } } diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 85a65f2c3be..f5fedfe19b2 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -73,13 +73,13 @@ struct TKikimrEvents : TEvents { ES_CMS, ES_NODE_BROKER, ES_TX_ALLOCATOR, //4152 - // reserve event space for each RTMR process - ES_RTMR_STORAGE, - ES_RTMR_PROXY, - ES_RTMR_PUSHER, - ES_RTMR_HOST, + // reserve event space for each RTMR process + ES_RTMR_STORAGE, + ES_RTMR_PROXY, + ES_RTMR_PUSHER, + ES_RTMR_HOST, ES_RESOURCE_BROKER, - ES_VIEWER, + ES_VIEWER, ES_SUB_DOMAIN, ES_GRPC_PROXY_STATUS, //OLD ES_SQS, diff --git a/ydb/core/client/server/document_conversion.h b/ydb/core/client/server/document_conversion.h index 7ad6485b338..6f70f09beec 100644 --- a/ydb/core/client/server/document_conversion.h +++ b/ydb/core/client/server/document_conversion.h @@ -1 +1 @@ -#pragma once +#pragma once diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp index 38c16c94dfb..8a4d8a3bbd5 100644 --- a/ydb/core/client/server/msgbus_server.cpp +++ b/ydb/core/client/server/msgbus_server.cpp @@ -336,8 +336,8 @@ TMessageBusSessionIdentHolder::TMessageBusSessionIdentHolder() TMessageBusSessionIdentHolder::TMessageBusSessionIdentHolder(TBusMessageContext &msg) { InitSession(msg); -} - +} + TMessageBusSessionIdentHolder::~TMessageBusSessionIdentHolder() {} @@ -365,16 +365,16 @@ void TBusMessageWatcher::NotifyForget() { if (MessageWatcher) { MessageWatcher->OnMessageDied(GetMessageId()); MessageWatcher = nullptr; - } -} - + } +} + void TBusMessageWatcher::NotifyReply(NBus::TBusMessage *response) { if (MessageWatcher) { MessageWatcher->OnMessageReplied(GetMessageId(), response); MessageWatcher = nullptr; } } - + class TMessageBusMonitorActor : public TActorBootstrapped<TMessageBusMonitorActor> { @@ -581,7 +581,7 @@ void TMessageBusServer::ClientActorRequest(ActorCreationFunc func, TBusMessageCo if (IActor *x = func(msg)) ActorSystem->Register(x, TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); else - msg.SendReplyMove(new TBusResponseStatus(MSTATUS_ERROR)); + msg.SendReplyMove(new TBusResponseStatus(MSTATUS_ERROR)); } void TMessageBusServer::GetTypes(TBusMessageContext &msg) { diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h index 37f55217204..3d40fc90017 100644 --- a/ydb/core/client/server/msgbus_server.h +++ b/ydb/core/client/server/msgbus_server.h @@ -311,6 +311,6 @@ IActor* CreateMessageBusConsoleRequest(TBusMessageContext &msg); IActor* CreateMessageBusTestShardControl(TBusMessageContext &msg); TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, const NKikimrTxUserProxy::TEvProposeTransactionStatus &result); - + } } diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index c079e939adb..7f3146d21a7 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -107,48 +107,48 @@ IActor* CreateMessageBusServerProxy( return new TMessageBusServerProxy(server, pqReadSessionsInfoWorkerFactory); } -TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, +TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, const NKikimrTxUserProxy::TEvProposeTransactionStatus &result) { - TAutoPtr<TBusResponse> response(new TBusResponse()); - response->Record.SetStatus(status); - response->Record.SetProxyErrorCode(result.GetStatus()); - + TAutoPtr<TBusResponse> response(new TBusResponse()); + response->Record.SetStatus(status); + response->Record.SetProxyErrorCode(result.GetStatus()); + response->Record.SetStatusCode(result.GetStatusCode()); response->Record.MutableIssues()->CopyFrom(result.GetIssues()); - if (result.HasExecutionEngineStatus()) - response->Record.SetExecutionEngineStatus(result.GetExecutionEngineStatus()); - if (result.HasExecutionEngineResponseStatus()) - response->Record.SetExecutionEngineResponseStatus(result.GetExecutionEngineResponseStatus()); + if (result.HasExecutionEngineStatus()) + response->Record.SetExecutionEngineStatus(result.GetExecutionEngineStatus()); + if (result.HasExecutionEngineResponseStatus()) + response->Record.SetExecutionEngineResponseStatus(result.GetExecutionEngineResponseStatus()); if (result.HasMiniKQLCompileResults()) response->Record.MutableMiniKQLCompileResults()->CopyFrom(result.GetMiniKQLCompileResults()); - if (result.HasMiniKQLErrors()) - response->Record.SetMiniKQLErrors(result.GetMiniKQLErrors()); - if (result.HasDataShardErrors()) - response->Record.SetDataShardErrors(result.GetDataShardErrors()); - if (result.ComplainingDataShardsSize()) - response->Record.MutableComplainingDataShards()->CopyFrom(result.GetComplainingDataShards()); - if (result.UnresolvedKeysSize()) - response->Record.MutableUnresolvedKeys()->CopyFrom(result.GetUnresolvedKeys()); + if (result.HasMiniKQLErrors()) + response->Record.SetMiniKQLErrors(result.GetMiniKQLErrors()); + if (result.HasDataShardErrors()) + response->Record.SetDataShardErrors(result.GetDataShardErrors()); + if (result.ComplainingDataShardsSize()) + response->Record.MutableComplainingDataShards()->CopyFrom(result.GetComplainingDataShards()); + if (result.UnresolvedKeysSize()) + response->Record.MutableUnresolvedKeys()->CopyFrom(result.GetUnresolvedKeys()); if (result.HasHadFollowerReads()) response->Record.SetHadFollowerReads(result.GetHadFollowerReads()); - - if (result.HasTxId()) - response->Record.SetTxId(result.GetTxId()); - if (result.HasStep()) - response->Record.SetStep(result.GetStep()); - - if (result.HasSchemeShardStatus()) - response->Record.SetSchemeStatus(result.GetSchemeShardStatus()); - if (result.HasSchemeShardReportedId()) - response->Record.SetSchemeTagId(result.GetSchemeShardReportedId()); - + + if (result.HasTxId()) + response->Record.SetTxId(result.GetTxId()); + if (result.HasStep()) + response->Record.SetStep(result.GetStep()); + + if (result.HasSchemeShardStatus()) + response->Record.SetSchemeStatus(result.GetSchemeShardStatus()); + if (result.HasSchemeShardReportedId()) + response->Record.SetSchemeTagId(result.GetSchemeShardReportedId()); + if (result.HasTimings()) response->Record.MutableProxyTimings()->CopyFrom(result.GetTimings()); - - return response.Release(); + + return response.Release(); } - + //void TMessageBusServerProxy::Handle(TEvBusProxy::TEvRequest::TPtr& ev, const TActorContext& ctx); // see msgbus_server_request.cpp //void TMessageBusServerProxy::Handle(TEvBusProxy::TEvPersQueue::TPtr& ev, const TActorContext& ctx); // see msgbus_server_scheme_request.cpp diff --git a/ydb/core/client/server/msgbus_server_request.cpp b/ydb/core/client/server/msgbus_server_request.cpp index c4281c04d79..9223bea9f1a 100644 --- a/ydb/core/client/server/msgbus_server_request.cpp +++ b/ydb/core/client/server/msgbus_server_request.cpp @@ -339,5 +339,5 @@ void TMessageBusServerProxy::Handle(TEvBusProxy::TEvRequest::TPtr& ev, const TAc ctx.Register(new TMessageBusServerRequest(ev->Get())); } -} +} } diff --git a/ydb/core/client/server/msgbus_server_tablet_state.cpp b/ydb/core/client/server/msgbus_server_tablet_state.cpp index 55f9b474c32..c8582fc5501 100644 --- a/ydb/core/client/server/msgbus_server_tablet_state.cpp +++ b/ydb/core/client/server/msgbus_server_tablet_state.cpp @@ -1,4 +1,4 @@ -#include "msgbus_server.h" +#include "msgbus_server.h" #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> diff --git a/ydb/core/client/server/msgbus_server_types.cpp b/ydb/core/client/server/msgbus_server_types.cpp index 60c8050c619..0495040e3cd 100644 --- a/ydb/core/client/server/msgbus_server_types.cpp +++ b/ydb/core/client/server/msgbus_server_types.cpp @@ -13,14 +13,14 @@ namespace NKikimr { namespace NMsgBusProxy { -class TMessageBusGetTypes : public TActorBootstrapped<TMessageBusGetTypes>, public TMessageBusSessionIdentHolder { +class TMessageBusGetTypes : public TActorBootstrapped<TMessageBusGetTypes>, public TMessageBusSessionIdentHolder { public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::MSGBUS_COMMON; } TMessageBusGetTypes(TBusMessageContext &msg, TMaybe<ui64> etag) - : TMessageBusSessionIdentHolder(msg) + : TMessageBusSessionIdentHolder(msg) , Etag(etag) {} diff --git a/ydb/core/client/server/msgbus_servicereq.h b/ydb/core/client/server/msgbus_servicereq.h index c23d1c32f19..704a231dd1d 100644 --- a/ydb/core/client/server/msgbus_servicereq.h +++ b/ydb/core/client/server/msgbus_servicereq.h @@ -1,59 +1,59 @@ -#pragma once - -#include "msgbus_server.h" +#pragma once + +#include "msgbus_server.h" #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <ydb/core/base/appdata.h> #include <ydb/public/lib/base/msgbus.h> -#include <util/network/address.h> - -namespace NKikimr { -namespace NMsgBusProxy { - +#include <util/network/address.h> + +namespace NKikimr { +namespace NMsgBusProxy { + template<typename TDerived, NKikimrServices::TActivity::EType Activity> -class TMessageBusLocalServiceRequest : public TActorBootstrapped<TDerived>, public TMessageBusSessionIdentHolder { -protected: +class TMessageBusLocalServiceRequest : public TActorBootstrapped<TDerived>, public TMessageBusSessionIdentHolder { +protected: TActorId ServiceID; - const TDuration Timeout; - - void SendReplyAndDie(NBus::TBusMessage *reply, const TActorContext &ctx) { + const TDuration Timeout; + + void SendReplyAndDie(NBus::TBusMessage *reply, const TActorContext &ctx) { SendReplyMove(reply); - return this->Die(ctx); - } - + return this->Die(ctx); + } + TMessageBusLocalServiceRequest(TBusMessageContext &msg, const TDuration& timeout) - : TMessageBusSessionIdentHolder(msg) - , ServiceID() - , Timeout(timeout) - {} - -public: - NBus::TBusMessage* CreateErrorReply(EResponseStatus status, const TActorContext &ctx) { + : TMessageBusSessionIdentHolder(msg) + , ServiceID() + , Timeout(timeout) + {} + +public: + NBus::TBusMessage* CreateErrorReply(EResponseStatus status, const TActorContext &ctx) { Y_UNUSED(ctx); return new TBusResponseStatus(status, "undocumented error 3"); - } - - void Bootstrap(const TActorContext &ctx) { - ServiceID = static_cast<TDerived *>(this)->MakeServiceID(ctx); - if (ServiceID) { - IEventBase* request = static_cast<TDerived *>(this)->MakeReq(ctx); - - ctx.Send(ServiceID, request, IEventHandle::FlagTrackDelivery, 0); + } + + void Bootstrap(const TActorContext &ctx) { + ServiceID = static_cast<TDerived *>(this)->MakeServiceID(ctx); + if (ServiceID) { + IEventBase* request = static_cast<TDerived *>(this)->MakeReq(ctx); + + ctx.Send(ServiceID, request, IEventHandle::FlagTrackDelivery, 0); if (Timeout) { - this->Become(&TDerived::StateFunc, ctx, Timeout, new TEvents::TEvWakeup()); + this->Become(&TDerived::StateFunc, ctx, Timeout, new TEvents::TEvWakeup()); } else { - // no timeout - this->Become(&TDerived::StateFunc); - } - } else { - return SendReplyAndDie(static_cast<TDerived *>(this)->CreateErrorReply(MSTATUS_ERROR, ctx), ctx); - } - } - + // no timeout + this->Become(&TDerived::StateFunc); + } + } else { + return SendReplyAndDie(static_cast<TDerived *>(this)->CreateErrorReply(MSTATUS_ERROR, ctx), ctx); + } + } + static constexpr auto ActorActivityType() { return Activity; } -}; - -} -} +}; + +} +} diff --git a/ydb/core/client/server/msgbus_tabletreq.h b/ydb/core/client/server/msgbus_tabletreq.h index 21c33ac2c81..f173ae90976 100644 --- a/ydb/core/client/server/msgbus_tabletreq.h +++ b/ydb/core/client/server/msgbus_tabletreq.h @@ -1,5 +1,5 @@ #pragma once -#include "msgbus_server.h" +#include "msgbus_server.h" #include <ydb/public/lib/base/msgbus.h> #include <ydb/core/base/tablet.h> #include <ydb/core/base/tablet_pipe.h> @@ -7,19 +7,19 @@ #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/core/hfunc.h> #include <ydb/core/protos/services.pb.h> -#include <util/generic/hash_set.h> -#include <util/generic/vector.h> - -#define CHECK_PROTOBUF_FIELD_PRESENCE(pb, field, errslist) \ - if (!pb.Has##field()) { \ - errslist.push_back( #field );\ - } - +#include <util/generic/hash_set.h> +#include <util/generic/vector.h> + +#define CHECK_PROTOBUF_FIELD_PRESENCE(pb, field, errslist) \ + if (!pb.Has##field()) { \ + errslist.push_back( #field );\ + } + namespace NKikimr { namespace NMsgBusProxy { template<typename TDerived, typename TTabletReplyEvent> -class TMessageBusTabletRequest : public TActorBootstrapped<TDerived>, public TMessageBusSessionIdentHolder { +class TMessageBusTabletRequest : public TActorBootstrapped<TDerived>, public TMessageBusSessionIdentHolder { protected: const TDuration Timeout; @@ -72,7 +72,7 @@ protected: } TMessageBusTabletRequest(TBusMessageContext &msg, bool withRetry, TDuration timeout, bool connectToFollower) - : TMessageBusSessionIdentHolder(msg) + : TMessageBusSessionIdentHolder(msg) , Timeout(timeout) , WithRetry(withRetry) , ConnectToFollower(connectToFollower) @@ -123,7 +123,7 @@ template<typename TDerived, typename TTabletReplyEvent, NKikimrServices::TActivi class TMessageBusSimpleTabletRequest : public TMessageBusTabletRequest<TDerived, TTabletReplyEvent> { protected: const ui64 TabletID; - + TMessageBusSimpleTabletRequest(TBusMessageContext &msg, ui64 tabletId, bool withRetry, TDuration timeout, bool connectToFollower) : TMessageBusTabletRequest<TDerived, TTabletReplyEvent>(msg, withRetry, timeout, connectToFollower) , TabletID(tabletId) diff --git a/ydb/core/client/server/ya.make b/ydb/core/client/server/ya.make index e154c0faf80..8cbeec21b77 100644 --- a/ydb/core/client/server/ya.make +++ b/ydb/core/client/server/ya.make @@ -56,7 +56,7 @@ SRCS( msgbus_server_tx_request.cpp msgbus_server_types.cpp msgbus_server_whoami.cpp - msgbus_servicereq.h + msgbus_servicereq.h msgbus_tabletreq.h grpc_server.cpp grpc_server.h diff --git a/ydb/core/driver_lib/base_utils/base_utils.h b/ydb/core/driver_lib/base_utils/base_utils.h index 63575e14156..4809f2824da 100644 --- a/ydb/core/driver_lib/base_utils/base_utils.h +++ b/ydb/core/driver_lib/base_utils/base_utils.h @@ -1,62 +1,62 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> - + #include <ydb/core/protos/config.pb.h> - + #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/stream/file.h> -#include <util/stream/format.h> -#include <util/system/hostname.h> - +#include <util/stream/file.h> +#include <util/stream/format.h> +#include <util/system/hostname.h> + #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/string_utils/parse_size/parse_size.h> #include <library/cpp/svnversion/svnversion.h> - -namespace NKikimr { - -struct TCmdFormatConfig { + +namespace NKikimr { + +struct TCmdFormatConfig { TString Path; - NSize::TSize DiskSize; - NSize::TSize ChunkSize; - NSize::TSize SectorSize; - ui64 Guid; + NSize::TSize DiskSize; + NSize::TSize ChunkSize; + NSize::TSize SectorSize; + ui64 Guid; ui64 MainKey; TString TextMessage; - - TCmdFormatConfig(); - - void Parse(int argc, char **argv); -}; - -struct TCmdFormatInfoConfig { + + TCmdFormatConfig(); + + void Parse(int argc, char **argv); +}; + +struct TCmdFormatInfoConfig { TString Path; ui64 MainKey; - bool IsVerbose; - - TCmdFormatInfoConfig(); - - void Parse(int argc, char **argv); -}; - -struct TCmdNodeByHostConfig { + bool IsVerbose; + + TCmdFormatInfoConfig(); + + void Parse(int argc, char **argv); +}; + +struct TCmdNodeByHostConfig { TString NamingFile; TString Hostname; - i32 Port; - - TCmdNodeByHostConfig(); - - void Parse(int argc, char **argv); -}; - -struct TCmdFormatUtilConfig { + i32 Port; + + TCmdNodeByHostConfig(); + + void Parse(int argc, char **argv); +}; + +struct TCmdFormatUtilConfig { TString FormatFile; - ui32 NodeId; - - TCmdFormatUtilConfig (); - - void Parse(int argc, char **argv); -}; - -} + ui32 NodeId; + + TCmdFormatUtilConfig (); + + void Parse(int argc, char **argv); +}; + +} diff --git a/ydb/core/driver_lib/base_utils/format_info.cpp b/ydb/core/driver_lib/base_utils/format_info.cpp index ac02c9382bc..e5d53a7b803 100644 --- a/ydb/core/driver_lib/base_utils/format_info.cpp +++ b/ydb/core/driver_lib/base_utils/format_info.cpp @@ -1,5 +1,5 @@ #include "format_info.h" -#include "base_utils.h" +#include "base_utils.h" #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h> diff --git a/ydb/core/driver_lib/base_utils/format_info.h b/ydb/core/driver_lib/base_utils/format_info.h index 5768c2ab3a4..d391c49aa42 100644 --- a/ydb/core/driver_lib/base_utils/format_info.h +++ b/ydb/core/driver_lib/base_utils/format_info.h @@ -1,5 +1,5 @@ #pragma once -#include "base_utils.h" +#include "base_utils.h" namespace NKikimr { diff --git a/ydb/core/driver_lib/base_utils/format_util.cpp b/ydb/core/driver_lib/base_utils/format_util.cpp index 503f2bf9763..e6f99b16344 100644 --- a/ydb/core/driver_lib/base_utils/format_util.cpp +++ b/ydb/core/driver_lib/base_utils/format_util.cpp @@ -1,5 +1,5 @@ #include "format_util.h" -#include "base_utils.h" +#include "base_utils.h" namespace NKikimr { diff --git a/ydb/core/driver_lib/base_utils/format_util.h b/ydb/core/driver_lib/base_utils/format_util.h index 9f75c1dbe1a..f3ed6a7c520 100644 --- a/ydb/core/driver_lib/base_utils/format_util.h +++ b/ydb/core/driver_lib/base_utils/format_util.h @@ -1,5 +1,5 @@ #pragma once -#include "base_utils.h" +#include "base_utils.h" namespace NKikimr { diff --git a/ydb/core/driver_lib/base_utils/node_by_host.cpp b/ydb/core/driver_lib/base_utils/node_by_host.cpp index 0add24bce52..81344f4dae2 100644 --- a/ydb/core/driver_lib/base_utils/node_by_host.cpp +++ b/ydb/core/driver_lib/base_utils/node_by_host.cpp @@ -1,5 +1,5 @@ #include "node_by_host.h" -#include "base_utils.h" +#include "base_utils.h" namespace NKikimr { diff --git a/ydb/core/driver_lib/base_utils/node_by_host.h b/ydb/core/driver_lib/base_utils/node_by_host.h index a76ab19a74b..67541e40252 100644 --- a/ydb/core/driver_lib/base_utils/node_by_host.h +++ b/ydb/core/driver_lib/base_utils/node_by_host.h @@ -1,5 +1,5 @@ #pragma once -#include "base_utils.h" +#include "base_utils.h" namespace NKikimr { diff --git a/ydb/core/driver_lib/base_utils/ya.make b/ydb/core/driver_lib/base_utils/ya.make index aed25e068b3..3c72ab274a4 100644 --- a/ydb/core/driver_lib/base_utils/ya.make +++ b/ydb/core/driver_lib/base_utils/ya.make @@ -1,18 +1,18 @@ -LIBRARY(base_utils) - +LIBRARY(base_utils) + OWNER(g:kikimr) - -SRCS( - base_utils.h - format_info.h - format_info.cpp - format_util.h - format_util.cpp - node_by_host.h - node_by_host.cpp -) - -PEERDIR( + +SRCS( + base_utils.h + format_info.h + format_info.cpp + format_util.h + format_util.cpp + node_by_host.h + node_by_host.cpp +) + +PEERDIR( library/cpp/deprecated/enum_codegen library/cpp/grpc/client ydb/core/blobstorage/pdisk @@ -20,8 +20,8 @@ PEERDIR( ydb/core/driver_lib/cli_config_base ydb/core/protos ydb/public/lib/deprecated/client -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/driver_lib/cli_config_base/config_base.cpp b/ydb/core/driver_lib/cli_config_base/config_base.cpp index f4223fcd08c..3672ce3ec2f 100644 --- a/ydb/core/driver_lib/cli_config_base/config_base.cpp +++ b/ydb/core/driver_lib/cli_config_base/config_base.cpp @@ -1,16 +1,16 @@ -#include "config_base.h" - -namespace NKikimr { - +#include "config_base.h" + +namespace NKikimr { + TCommandConfig CommandConfig; -TDuration ParseDuration(const TStringBuf& str) { - TDuration result; - if (TDuration::TryParse(str, result)) - return result; - ythrow yexception() << "wrong TDuration format"; +TDuration ParseDuration(const TStringBuf& str) { + TDuration result; + if (TDuration::TryParse(str, result)) + return result; + ythrow yexception() << "wrong TDuration format"; } - + TCommandConfig::TServerEndpoint TCommandConfig::ParseServerAddress(const TString& address) { TServerEndpoint endpoint = {EServerType::GRpc, address}; if (address.empty()) { @@ -42,31 +42,31 @@ TCommandConfig::TServerEndpoint TCommandConfig::ParseServerAddress(const TString } const TString ArgFormatDescription() { - return R"___( -Common option formats: - NUM - integer number, bounds should be deducted from a command context - STR - string, format should be deducted from a command context - ADDR - network address, e.g. kikimr0000.search.yandex.net (or 5.45.222.113, 2a02:6b8:b000:2000::52d:de71), localhost (or 127.0.0.1, ::1) - PATH - filesystem path, e.g. /Berkanavt/kikimr/bin or directory/file.txt - BYTES - integer size in bytes - possible suffixes: - * k|K - kilobytes ( x 2^10 ) - * m|M - megabytes ( x 2^20 ) - * g|G - gigabytes ( x 2^30 ) - * t|T - terabytes ( x 2^40 ) - examples: 1024, 4k, 100M, 2G, 1T - TIME - floating point duration in seconds - possible suffixes: - * s - seconds ( x 1 ) - * ms - milliseconds ( x 10^-3 ) - * us - microseconds ( x 10^-6 ) - * ns - nanoseconds ( x 10^-9 ) - * m - minutes ( x 60 ) - * h - hours ( x 3600 ) - * d - days ( x 86400 ) - precision: 1us - examples: 1.5, 1.5s, 10.5ms, 100us, 1500ns, 30m, 2.5h, 7d - )___"; -} - -} // namespace NKikimr + return R"___( +Common option formats: + NUM - integer number, bounds should be deducted from a command context + STR - string, format should be deducted from a command context + ADDR - network address, e.g. kikimr0000.search.yandex.net (or 5.45.222.113, 2a02:6b8:b000:2000::52d:de71), localhost (or 127.0.0.1, ::1) + PATH - filesystem path, e.g. /Berkanavt/kikimr/bin or directory/file.txt + BYTES - integer size in bytes + possible suffixes: + * k|K - kilobytes ( x 2^10 ) + * m|M - megabytes ( x 2^20 ) + * g|G - gigabytes ( x 2^30 ) + * t|T - terabytes ( x 2^40 ) + examples: 1024, 4k, 100M, 2G, 1T + TIME - floating point duration in seconds + possible suffixes: + * s - seconds ( x 1 ) + * ms - milliseconds ( x 10^-3 ) + * us - microseconds ( x 10^-6 ) + * ns - nanoseconds ( x 10^-9 ) + * m - minutes ( x 60 ) + * h - hours ( x 3600 ) + * d - days ( x 86400 ) + precision: 1us + examples: 1.5, 1.5s, 10.5ms, 100us, 1500ns, 30m, 2.5h, 7d + )___"; +} + +} // namespace NKikimr diff --git a/ydb/core/driver_lib/cli_config_base/config_base.h b/ydb/core/driver_lib/cli_config_base/config_base.h index 0312099c2ed..6db66dc7d1b 100644 --- a/ydb/core/driver_lib/cli_config_base/config_base.h +++ b/ydb/core/driver_lib/cli_config_base/config_base.h @@ -1,41 +1,41 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/mkql_function_registry.h> #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <ydb/public/lib/deprecated/client/grpc_client.h> #include <ydb/core/util/pb.h> - + #include <google/protobuf/text_format.h> - -#include <util/datetime/base.h> - + +#include <util/datetime/base.h> + #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> - -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> + +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/generic/variant.h> - -#include <util/stream/file.h> -#include <util/stream/format.h> - -namespace NKikimr { - -/// last_getopt parse handler for time durations. -/// Canonical usage: -/// NLastGetopt::TOpt& option = ...; -/// TDuration duration; -/// ... -/// option.StoreMappedResultT<TStringBuf>(&duration, &ParseDuration); -TDuration ParseDuration(const TStringBuf& str); - -// all modes use common argument format + +#include <util/stream/file.h> +#include <util/stream/format.h> + +namespace NKikimr { + +/// last_getopt parse handler for time durations. +/// Canonical usage: +/// NLastGetopt::TOpt& option = ...; +/// TDuration duration; +/// ... +/// option.StoreMappedResultT<TStringBuf>(&duration, &ParseDuration); +TDuration ParseDuration(const TStringBuf& str); + +// all modes use common argument format const TString ArgFormatDescription(); - -struct TCommandConfig { + +struct TCommandConfig { TMaybe<std::variant<NMsgBusProxy::TMsgBusClientConfig, NGRpcProxy::TGRpcClientConfig>> ClientConfig; - + enum class EServerType { MessageBus, GRpc, @@ -48,56 +48,56 @@ struct TCommandConfig { }; static TServerEndpoint ParseServerAddress(const TString& address); -}; - +}; + extern TCommandConfig CommandConfig; -} // namespace NKikimr - - -#define MODE_GEN_HELP(enumName, commandName, help, ...) \ - helpStream << " " << RightPad(commandName, 20) << " - " << help << "\n"; - -#define APPEND_ENUM(name, command, ...) \ - if (!first) commandsList << ", "; commandsList << command; first = false; - -#define MODE_GEN_COMMAND(enumName, commandName, ...) \ - if (mode == defaultValue && stricmp(arg, commandName) == 0) mode = enumName; - -#define CLI_MODES_IMPL(MODES_ENUM, DEFAULT_VALUE, MODES_MAP)\ -\ -enum MODES_ENUM {\ - DEFAULT_VALUE,\ - MODES_MAP(ENUM_VALUE_GEN_NO_VALUE)\ -};\ -\ -template<typename TMode> struct TCliCommands;\ -\ -template<> struct TCliCommands<MODES_ENUM> {\ - MODES_ENUM Mode;\ -\ - TCliCommands(): Mode( (DEFAULT_VALUE) ) {}\ -\ +} // namespace NKikimr + + +#define MODE_GEN_HELP(enumName, commandName, help, ...) \ + helpStream << " " << RightPad(commandName, 20) << " - " << help << "\n"; + +#define APPEND_ENUM(name, command, ...) \ + if (!first) commandsList << ", "; commandsList << command; first = false; + +#define MODE_GEN_COMMAND(enumName, commandName, ...) \ + if (mode == defaultValue && stricmp(arg, commandName) == 0) mode = enumName; + +#define CLI_MODES_IMPL(MODES_ENUM, DEFAULT_VALUE, MODES_MAP)\ +\ +enum MODES_ENUM {\ + DEFAULT_VALUE,\ + MODES_MAP(ENUM_VALUE_GEN_NO_VALUE)\ +};\ +\ +template<typename TMode> struct TCliCommands;\ +\ +template<> struct TCliCommands<MODES_ENUM> {\ + MODES_ENUM Mode;\ +\ + TCliCommands(): Mode( (DEFAULT_VALUE) ) {}\ +\ static TString CommandsCsv() {\ - TStringStream commandsList;\ - bool first = true;\ - MODES_MAP(APPEND_ENUM);\ - return commandsList.Str();\ - }\ -\ + TStringStream commandsList;\ + bool first = true;\ + MODES_MAP(APPEND_ENUM);\ + return commandsList.Str();\ + }\ +\ static TString CommandsDescription(const TString& toolName) {\ - TStringStream helpStream;\ + TStringStream helpStream;\ helpStream << "[global options...] <command> [command options...] [mbus] [messagebus options...]";\ - helpStream << "\n\nAvailable commands are:\n";\ - MODES_MAP(MODE_GEN_HELP);\ + helpStream << "\n\nAvailable commands are:\n";\ + MODES_MAP(MODE_GEN_HELP);\ helpStream << "\nSee '" << toolName << " <command> --help' for mode options\n";\ - return helpStream.Str();\ - } \ -\ - static MODES_ENUM ParseCommand(const char* arg) {\ - static constexpr auto defaultValue = (DEFAULT_VALUE);\ - MODES_ENUM mode = defaultValue;\ - MODES_MAP(MODE_GEN_COMMAND);\ - return mode;\ - }\ -}; + return helpStream.Str();\ + } \ +\ + static MODES_ENUM ParseCommand(const char* arg) {\ + static constexpr auto defaultValue = (DEFAULT_VALUE);\ + MODES_ENUM mode = defaultValue;\ + MODES_MAP(MODE_GEN_COMMAND);\ + return mode;\ + }\ +}; diff --git a/ydb/core/driver_lib/cli_config_base/ya.make b/ydb/core/driver_lib/cli_config_base/ya.make index f0436e1ee61..feb09c8e7b9 100644 --- a/ydb/core/driver_lib/cli_config_base/ya.make +++ b/ydb/core/driver_lib/cli_config_base/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - config_base.h - config_base.cpp -) - -PEERDIR( + +SRCS( + config_base.h + config_base.cpp +) + +PEERDIR( library/cpp/deprecated/enum_codegen ydb/core/util ydb/public/lib/deprecated/client ydb/library/yql/minikql -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h index e95b3e7d2c7..099664792ea 100644 --- a/ydb/core/driver_lib/cli_utils/cli.h +++ b/ydb/core/driver_lib/cli_utils/cli.h @@ -1,7 +1,7 @@ #pragma once - -#include "cli_cmd_config.h" - + +#include "cli_cmd_config.h" + #include <ydb/core/driver_lib/cli_base/cli.h> #include <ydb/core/driver_lib/run/factories.h> @@ -9,9 +9,9 @@ #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/stream/file.h> -#include <util/stream/format.h> -#include <util/system/hostname.h> +#include <util/stream/file.h> +#include <util/stream/format.h> +#include <util/system/hostname.h> #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/string_utils/parse_size/parse_size.h> @@ -19,10 +19,10 @@ #include <google/protobuf/text_format.h> -namespace NKikimr { +namespace NKikimr { + +namespace NDriverClient { -namespace NDriverClient { - void DumpProxyErrorCodes(IOutputStream &o, const NKikimrClient::TResponse &response); void DumpSchemeErrorCode(IOutputStream &o, const NKikimrClient::TResponse &response); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp index ab4148be246..fcc93059966 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp @@ -1,23 +1,23 @@ #include <ydb/core/driver_lib/cli_config_base/config_base.h> -#include "cli_cmd_config.h" +#include "cli_cmd_config.h" #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <util/system/env.h> - -namespace NKikimr { -namespace NDriverClient { - - void TCliCmdConfig::ConfigureBaseLastGetopt(NLastGetopt::TOpts &opts) { - opts.AddHelpOption('h'); - - opts.AddLongOption('s', "server", "server address to connect") + +namespace NKikimr { +namespace NDriverClient { + + void TCliCmdConfig::ConfigureBaseLastGetopt(NLastGetopt::TOpts &opts) { + opts.AddHelpOption('h'); + + opts.AddLongOption('s', "server", "server address to connect") .RequiredArgument("ADDR").StoreResult(&Address); - - opts.SetFreeArgTitle(0, "mbus", "- use to override message bus client default settings"); - opts.SetFreeArgsMin(0); - opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; - } - - void TCliCmdConfig::ConfigureMsgBusLastGetopt(const NLastGetopt::TOptsParseResult& res, int argc, char** argv) { + + opts.SetFreeArgTitle(0, "mbus", "- use to override message bus client default settings"); + opts.SetFreeArgsMin(0); + opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; + } + + void TCliCmdConfig::ConfigureMsgBusLastGetopt(const NLastGetopt::TOptsParseResult& res, int argc, char** argv) { if (Address.empty()) { TString kikimrServer = GetEnv("KIKIMR_SERVER"); if (kikimrServer != nullptr) { @@ -54,8 +54,8 @@ namespace NDriverClient { ClientConfig = MsgBusClientConfig; break; - } - } - -} // namespace NDriverClient -} // namespace NKikimr + } + } + +} // namespace NDriverClient +} // namespace NKikimr diff --git a/ydb/core/driver_lib/cli_utils/cli_cmd_config.h b/ydb/core/driver_lib/cli_utils/cli_cmd_config.h index 8a82fcc6a48..bb0b69dbcb6 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmd_config.h +++ b/ydb/core/driver_lib/cli_utils/cli_cmd_config.h @@ -1,18 +1,18 @@ -#pragma once +#pragma once #include <ydb/public/lib/deprecated/kicli/kicli.h> #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <ydb/public/lib/deprecated/client/grpc_client.h> #include <util/generic/variant.h> - -namespace NKikimr { -namespace NDriverClient { - struct TCliCmdConfig : TNonCopyable { + +namespace NKikimr { +namespace NDriverClient { + struct TCliCmdConfig : TNonCopyable { TMaybe<std::variant<NMsgBusProxy::TMsgBusClientConfig, NGRpcProxy::TGRpcClientConfig>> ClientConfig; NMsgBusProxy::TMsgBusClientConfig MsgBusClientConfig; TString Address; - - void ConfigureBaseLastGetopt(NLastGetopt::TOpts &opts); - void ConfigureMsgBusLastGetopt(const NLastGetopt::TOptsParseResult& res, int argc, char** argv); + + void ConfigureBaseLastGetopt(NLastGetopt::TOpts &opts); + void ConfigureMsgBusLastGetopt(const NLastGetopt::TOptsParseResult& res, int argc, char** argv); template<typename TRequest> NBus::EMessageStatus SyncCall(TAutoPtr<TRequest> request, TAutoPtr<NBus::TBusMessage>& response) const { @@ -48,7 +48,7 @@ namespace NDriverClient { Y_FAIL("Client configuration is not provided"); } } - }; - -} -} + }; + +} +} diff --git a/ydb/core/driver_lib/cli_utils/ya.make b/ydb/core/driver_lib/cli_utils/ya.make index d8cc9c4dfb2..cb1f9670bea 100644 --- a/ydb/core/driver_lib/cli_utils/ya.make +++ b/ydb/core/driver_lib/cli_utils/ya.make @@ -1,13 +1,13 @@ -LIBRARY(cli_utils) - +LIBRARY(cli_utils) + OWNER(g:kikimr) - -SRCS( + +SRCS( cli.cpp - cli.h + cli.h cli_actorsystem_perftest.cpp - cli_cmd_config.h - cli_cmd_config.cpp + cli_cmd_config.h + cli_cmd_config.cpp cli_cmds.h cli_cmds_admin.cpp cli_cmds_bs.cpp @@ -24,19 +24,19 @@ SRCS( cli_cmds_server.cpp cli_cmds_tablet.cpp cli_cmds_tenant.cpp - cli_fakeinitshard.cpp + cli_fakeinitshard.cpp cli_keyvalue.cpp cli_persqueue.cpp cli_persqueue_cluster_discovery.cpp cli_persqueue_stress.cpp cli_load.cpp - cli_minikql_compile_and_exec.cpp - cli_mb_trace.cpp - cli_scheme_cache_append.cpp - cli_scheme_initroot.cpp -) - -PEERDIR( + cli_minikql_compile_and_exec.cpp + cli_mb_trace.cpp + cli_scheme_cache_append.cpp + cli_scheme_initroot.cpp +) + +PEERDIR( library/cpp/deprecated/enum_codegen library/cpp/grpc/client library/cpp/protobuf/json @@ -59,8 +59,8 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/lib/deprecated/client ydb/public/lib/ydb_cli/common -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/driver_lib/run/config.cpp b/ydb/core/driver_lib/run/config.cpp index 297251e4926..01f82b98ed2 100644 --- a/ydb/core/driver_lib/run/config.cpp +++ b/ydb/core/driver_lib/run/config.cpp @@ -3,9 +3,9 @@ namespace NKikimr { TKikimrRunConfig::TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, ui32 nodeId, const TKikimrScopeId& scopeId) - : AppConfig(appConfig) + : AppConfig(appConfig) , NodeId(nodeId) , ScopeId(scopeId) {} - -} // namespace NKikimr + +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index cabaa90b8fc..faf1797413e 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -4,42 +4,42 @@ #include <ydb/core/base/event_filter.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <google/protobuf/text_format.h> namespace NKikimr { -union TBasicKikimrServicesMask { - struct { - bool EnableBasicServices:1; +union TBasicKikimrServicesMask { + struct { + bool EnableBasicServices:1; bool EnableIcbService:1; - bool EnableWhiteBoard:1; - bool EnableBSNodeWarden:1; - bool EnableStateStorageService:1; - bool EnableLocalService:1; + bool EnableWhiteBoard:1; + bool EnableBSNodeWarden:1; + bool EnableStateStorageService:1; + bool EnableLocalService:1; bool EnableSharedCache:1; bool EnableBlobCache:1; - bool EnableLogger:1; + bool EnableLogger:1; bool EnableSchedulerActor:1; - bool EnableProfiler:1; + bool EnableProfiler:1; bool EnableResourceBroker:1; - bool EnableTabletResolver:1; - bool EnableTabletMonitoringProxy:1; - bool EnableTabletCountersAggregator:1; - bool EnableRestartsCountPublisher:1; - bool EnableBootstrapper:1; + bool EnableTabletResolver:1; + bool EnableTabletMonitoringProxy:1; + bool EnableTabletCountersAggregator:1; + bool EnableRestartsCountPublisher:1; + bool EnableBootstrapper:1; bool EnableMediatorTimeCastProxy:1; bool EnableTxProxy:1; - bool EnableMiniKQLCompileService:1; - bool EnableMessageBusServices:1; - bool EnableStatsCollector:1; + bool EnableMiniKQLCompileService:1; + bool EnableMessageBusServices:1; + bool EnableStatsCollector:1; bool EnableSelfPing:1; - bool EnableTabletMonitor:1; - bool EnableViewerService:1; - bool EnableLoadService:1; + bool EnableTabletMonitor:1; + bool EnableViewerService:1; + bool EnableLoadService:1; bool EnableFailureInjectionService:1; - bool EnablePersQueueL2Cache:1; + bool EnablePersQueueL2Cache:1; bool EnableKqp:1; bool EnableMemoryLog:1; bool EnableGRpcService:1; @@ -64,35 +64,35 @@ union TBasicKikimrServicesMask { bool EnableHealthCheckService:1; bool EnableYandexQuery:1; bool EnableSequenceProxyService:1; - }; - - ui64 Raw; - - void DisableAll() { - Raw = 0; - } - - void EnableAll() { - Raw = 0xFFFFFFFFFFFFFFFFLL; - } - - TBasicKikimrServicesMask() { - EnableAll(); - } -}; - -static_assert(sizeof(TBasicKikimrServicesMask) == 8, "expected sizeof(TBasicKikimrServicesMask) == 8"); - - -struct TKikimrRunConfig { - NKikimrConfig::TAppConfig& AppConfig; - ui32 NodeId; + }; + + ui64 Raw; + + void DisableAll() { + Raw = 0; + } + + void EnableAll() { + Raw = 0xFFFFFFFFFFFFFFFFLL; + } + + TBasicKikimrServicesMask() { + EnableAll(); + } +}; + +static_assert(sizeof(TBasicKikimrServicesMask) == 8, "expected sizeof(TBasicKikimrServicesMask) == 8"); + + +struct TKikimrRunConfig { + NKikimrConfig::TAppConfig& AppConfig; + ui32 NodeId; TKikimrScopeId ScopeId; TString PathToConfigCacheFile; - TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, + TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, ui32 nodeId = 0, const TKikimrScopeId& scopeId = {}); -}; +}; } diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index 5af3e42d04d..bb686562912 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -1,59 +1,59 @@ -#include "config_parser.h" -#include "dummy.h" - +#include "config_parser.h" +#include "dummy.h" + #include <library/cpp/actors/core/log_settings.h> #include <ydb/public/lib/base/msgbus.h> #include <ydb/public/lib/deprecated/client/msgbus_client.h> - -#include <util/stream/file.h> -#include <util/stream/format.h> -#include <util/system/hostname.h> + +#include <util/stream/file.h> +#include <util/stream/format.h> +#include <util/system/hostname.h> #include <util/string/printf.h> - + #include <library/cpp/string_utils/parse_size/parse_size.h> - + #include <google/protobuf/text_format.h> - -namespace NKikimr { - -TRunCommandConfigParser::TGlobalOpts::TGlobalOpts() - : StartTcp(false) - , SysLog(false) - , LogLevel(NActors::NLog::PRI_WARN) - , LogSamplingLevel(NActors::NLog::PRI_DEBUG) - , LogSamplingRate(0) -{} - -TRunCommandConfigParser::TRunOpts::TRunOpts() - : NodeId(0) - , StartBusProxy(false) - , BusProxyPort(NMsgBusProxy::TProtocol::DefaultPort) - , MonitoringPort(0) + +namespace NKikimr { + +TRunCommandConfigParser::TGlobalOpts::TGlobalOpts() + : StartTcp(false) + , SysLog(false) + , LogLevel(NActors::NLog::PRI_WARN) + , LogSamplingLevel(NActors::NLog::PRI_DEBUG) + , LogSamplingRate(0) +{} + +TRunCommandConfigParser::TRunOpts::TRunOpts() + : NodeId(0) + , StartBusProxy(false) + , BusProxyPort(NMsgBusProxy::TProtocol::DefaultPort) + , MonitoringPort(0) , MonitoringAddress() , MonitoringThreads(10) - , RestartsCountFile("") - , StartTracingBusProxy(true) - , CompileInflightLimit(100000) -{} - - -TRunCommandConfigParser::TRunCommandConfigParser(TKikimrRunConfig& config) - : Config(config) -{} - -TRunCommandConfigParser::~TRunCommandConfigParser() -{} - -void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts) { - // todo: file configs must be paired with url to request basic values - opts.AddLongOption('u', "url-base", "url base to request configs from").OptionalArgument("URL"); - opts.AddLongOption("sys-file", "actor system config file (use dummy config by default)").OptionalArgument("PATH"); - opts.AddLongOption("naming-file", "static nameservice config file").OptionalArgument("PATH"); - opts.AddLongOption("domains-file", "domain config file").OptionalArgument("PATH").Required(); - opts.AddLongOption("bs-file", "blobstorage config file").OptionalArgument("PATH"); - opts.AddLongOption("log-file", "log config file").OptionalArgument("PATH"); - opts.AddLongOption("ic-file", "interconnect config file").OptionalArgument("PATH"); - opts.AddLongOption("channels-file", "tablet channel profile config file").OptionalArgument("PATH").Required(); + , RestartsCountFile("") + , StartTracingBusProxy(true) + , CompileInflightLimit(100000) +{} + + +TRunCommandConfigParser::TRunCommandConfigParser(TKikimrRunConfig& config) + : Config(config) +{} + +TRunCommandConfigParser::~TRunCommandConfigParser() +{} + +void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts) { + // todo: file configs must be paired with url to request basic values + opts.AddLongOption('u', "url-base", "url base to request configs from").OptionalArgument("URL"); + opts.AddLongOption("sys-file", "actor system config file (use dummy config by default)").OptionalArgument("PATH"); + opts.AddLongOption("naming-file", "static nameservice config file").OptionalArgument("PATH"); + opts.AddLongOption("domains-file", "domain config file").OptionalArgument("PATH").Required(); + opts.AddLongOption("bs-file", "blobstorage config file").OptionalArgument("PATH"); + opts.AddLongOption("log-file", "log config file").OptionalArgument("PATH"); + opts.AddLongOption("ic-file", "interconnect config file").OptionalArgument("PATH"); + opts.AddLongOption("channels-file", "tablet channel profile config file").OptionalArgument("PATH").Required(); opts.AddLongOption("vdisk-file", "vdisk kind config file").OptionalArgument("PATH"); opts.AddLongOption("drivemodel-file", "drive model config file").OptionalArgument("PATH"); opts.AddLongOption("kqp-file", "Kikimr Query Processor config file").OptionalArgument("PATH"); @@ -75,43 +75,43 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("alloc-file", "Allocator config file").OptionalArgument("PATH"); opts.AddLongOption("yql-file", "Yql Analytics config file").OptionalArgument("PATH"); opts.AddLongOption("yq-file", "Yandex Query config file").OptionalArgument("PATH"); -} - -void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResult& res) { - if (res.Has("sys-file")) { +} + +void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResult& res) { + if (res.Has("sys-file")) { Y_VERIFY(ParsePBFromFile(res.Get("sys-file"), Config.AppConfig.MutableActorSystemConfig())); - } else { - auto sysConfig = DummyActorSystemConfig(); - Config.AppConfig.MutableActorSystemConfig()->CopyFrom(*sysConfig); - } - - if (res.Has("naming-file")) { + } else { + auto sysConfig = DummyActorSystemConfig(); + Config.AppConfig.MutableActorSystemConfig()->CopyFrom(*sysConfig); + } + + if (res.Has("naming-file")) { Y_VERIFY(ParsePBFromFile(res.Get("naming-file"), Config.AppConfig.MutableNameserviceConfig())); - } - - if (res.Has("domains-file")) { + } + + if (res.Has("domains-file")) { Y_VERIFY(ParsePBFromFile(res.Get("domains-file"), Config.AppConfig.MutableDomainsConfig())); - } - - if (res.Has("bs-file")) { + } + + if (res.Has("bs-file")) { Y_VERIFY(ParsePBFromFile(res.Get("bs-file"), Config.AppConfig.MutableBlobStorageConfig())); - } - - if (res.Has("log-file")) { + } + + if (res.Has("log-file")) { Y_VERIFY(ParsePBFromFile(res.Get("log-file"), Config.AppConfig.MutableLogConfig())); - } - - if (res.Has("ic-file")) { + } + + if (res.Has("ic-file")) { Y_VERIFY(ParsePBFromFile(res.Get("ic-file"), Config.AppConfig.MutableInterconnectConfig())); - } - - if (res.Has("channels-file")) { + } + + if (res.Has("channels-file")) { Y_VERIFY(ParsePBFromFile(res.Get("channels-file"), Config.AppConfig.MutableChannelProfileConfig())); - } - - if (res.Has("bootstrap-file")) { + } + + if (res.Has("bootstrap-file")) { Y_VERIFY(ParsePBFromFile(res.Get("bootstrap-file"), Config.AppConfig.MutableBootstrapConfig())); - } + } if (res.Has("vdisk-file")) { Y_VERIFY(ParsePBFromFile(res.Get("vdisk-file"), Config.AppConfig.MutableVDiskConfig())); @@ -206,149 +206,149 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu if (res.Has("yq-file")) { Y_VERIFY(ParsePBFromFile(res.Get("yq-file"), Config.AppConfig.MutableYandexQueryConfig())); } -} - -void TRunCommandConfigParser::SetupGlobalOpts(NLastGetopt::TOpts& opts) { - opts.AddLongOption("cluster-name", "which cluster this node belongs to") - .DefaultValue("unknown").OptionalArgument("STR").StoreResult(&GlobalOpts.ClusterName); - opts.AddLongOption("log-level", "default logging level").OptionalArgument("1-7") - .DefaultValue(ToString(GlobalOpts.LogLevel)).StoreResult(&GlobalOpts.LogLevel); - opts.AddLongOption("log-sampling-level", "sample logs equal to or above this level").OptionalArgument("1-7") - .DefaultValue(ToString(GlobalOpts.LogSamplingLevel)).StoreResult(&GlobalOpts.LogSamplingLevel); - opts.AddLongOption("log-sampling-rate", - "log only each Nth message with priority matching sampling level; 0 turns log sampling off") - .OptionalArgument(Sprintf("0,%" PRIu32, Max<ui32>())) - .DefaultValue(ToString(GlobalOpts.LogSamplingRate)).StoreResult(&GlobalOpts.LogSamplingRate); - opts.AddLongOption("log-format", "log format to use; short skips the priority and timestamp") - .DefaultValue("full").OptionalArgument("full|short|json").StoreResult(&GlobalOpts.LogFormat); - opts.AddLongOption("syslog", "send to syslog instead of stderr").NoArgument(); - opts.AddLongOption("tcp", "start tcp interconnect").NoArgument(); +} + +void TRunCommandConfigParser::SetupGlobalOpts(NLastGetopt::TOpts& opts) { + opts.AddLongOption("cluster-name", "which cluster this node belongs to") + .DefaultValue("unknown").OptionalArgument("STR").StoreResult(&GlobalOpts.ClusterName); + opts.AddLongOption("log-level", "default logging level").OptionalArgument("1-7") + .DefaultValue(ToString(GlobalOpts.LogLevel)).StoreResult(&GlobalOpts.LogLevel); + opts.AddLongOption("log-sampling-level", "sample logs equal to or above this level").OptionalArgument("1-7") + .DefaultValue(ToString(GlobalOpts.LogSamplingLevel)).StoreResult(&GlobalOpts.LogSamplingLevel); + opts.AddLongOption("log-sampling-rate", + "log only each Nth message with priority matching sampling level; 0 turns log sampling off") + .OptionalArgument(Sprintf("0,%" PRIu32, Max<ui32>())) + .DefaultValue(ToString(GlobalOpts.LogSamplingRate)).StoreResult(&GlobalOpts.LogSamplingRate); + opts.AddLongOption("log-format", "log format to use; short skips the priority and timestamp") + .DefaultValue("full").OptionalArgument("full|short|json").StoreResult(&GlobalOpts.LogFormat); + opts.AddLongOption("syslog", "send to syslog instead of stderr").NoArgument(); + opts.AddLongOption("tcp", "start tcp interconnect").NoArgument(); opts.AddLongOption("udf", "Load shared library with UDF by given path").AppendTo(&GlobalOpts.UDFsPaths); opts.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory").StoreResult(&GlobalOpts.UDFsDir); -} - - -void TRunCommandConfigParser::ParseGlobalOpts(const NLastGetopt::TOptsParseResult& res) { - if (res.Has("tcp")) - GlobalOpts.StartTcp = true; - - if (res.Has("syslog")) - GlobalOpts.SysLog = true; -} - -void TRunCommandConfigParser::ParseRunOpts(int argc, char **argv) { - using namespace NLastGetopt; - TOpts opts = TOpts::Default(); - - opts.AddLongOption('n', "node", "Node ID") +} + + +void TRunCommandConfigParser::ParseGlobalOpts(const NLastGetopt::TOptsParseResult& res) { + if (res.Has("tcp")) + GlobalOpts.StartTcp = true; + + if (res.Has("syslog")) + GlobalOpts.SysLog = true; +} + +void TRunCommandConfigParser::ParseRunOpts(int argc, char **argv) { + using namespace NLastGetopt; + TOpts opts = TOpts::Default(); + + opts.AddLongOption('n', "node", "Node ID") .RequiredArgument(Sprintf("NUM (0,%d]", NActors::TActorId::MaxNodeId)).Required().StoreResult(&RunOpts.NodeId); - opts.AddLongOption("proxy", "Bind to proxy(-ies)").RequiredArgument("ADDR").AppendTo(&RunOpts.ProxyBindToProxy); - opts.AddLongOption("mon-port", "Monitoring port").OptionalArgument("NUM").StoreResult(&RunOpts.MonitoringPort); + opts.AddLongOption("proxy", "Bind to proxy(-ies)").RequiredArgument("ADDR").AppendTo(&RunOpts.ProxyBindToProxy); + opts.AddLongOption("mon-port", "Monitoring port").OptionalArgument("NUM").StoreResult(&RunOpts.MonitoringPort); opts.AddLongOption("mon-address", "Monitoring address").OptionalArgument("ADDR").StoreResult(&RunOpts.MonitoringAddress); opts.AddLongOption("mon-threads", "Monitoring http server threads").RequiredArgument("NUM").StoreResult(&RunOpts.MonitoringThreads); - - SetupLastGetOptForConfigFiles(opts); - opts.AddLongOption("bootstrap-file", "Bootstrap config file").OptionalArgument("PATH"); + + SetupLastGetOptForConfigFiles(opts); + opts.AddLongOption("bootstrap-file", "Bootstrap config file").OptionalArgument("PATH"); opts.AddLongOption("feature-flags-file", "File with feature flags to turn new features on/off").OptionalArgument("PATH"); - opts.AddLongOption('r', "restarts-count-file", "State for restarts monitoring counter,\nuse empty string to disable\n") - .OptionalArgument("PATH").DefaultValue(RunOpts.RestartsCountFile).StoreResult(&RunOpts.RestartsCountFile); - opts.AddLongOption("compile-inflight-limit", "Limit on parallel programs compilation").OptionalArgument("NUM").StoreResult(&RunOpts.CompileInflightLimit); - opts.AddHelpOption('h'); - opts.SetFreeArgTitle(0, "mbus", "- use to start message bus proxy and override it's settings"); - opts.SetFreeArgsMin(0); - opts.ArgPermutation_ = REQUIRE_ORDER; - - TOptsParseResult res(&opts, argc, argv); - ParseConfigFiles(res); - - if ((res.GetFreeArgCount() > 0) && (res.GetFreeArgs().at(0) == "mbus")) { - size_t freeArgsPos = res.GetFreeArgsPos(); - argc -= freeArgsPos; - argv += freeArgsPos; - TOpts msgBusOpts = TOpts::Default(); - SetMsgBusDefaults(RunOpts.ProxyBusSessionConfig, RunOpts.ProxyBusQueueConfig); - msgBusOpts.AddLongOption("port", "Message bus proxy port").OptionalArgument("NUM") - .DefaultValue(ToString(RunOpts.BusProxyPort)).StoreResult(&RunOpts.BusProxyPort); - msgBusOpts.AddLongOption("trace-path", "Path for trace files").Optional().RequiredArgument("PATH").StoreResult(&RunOpts.TracePath); - msgBusOpts.AddHelpOption('h'); - RunOpts.ProxyBusQueueConfig.ConfigureLastGetopt(msgBusOpts, ""); - RunOpts.ProxyBusSessionConfig.ConfigureLastGetopt(msgBusOpts, ""); - TOptsParseResult mbusRes(&msgBusOpts, argc, argv); + opts.AddLongOption('r', "restarts-count-file", "State for restarts monitoring counter,\nuse empty string to disable\n") + .OptionalArgument("PATH").DefaultValue(RunOpts.RestartsCountFile).StoreResult(&RunOpts.RestartsCountFile); + opts.AddLongOption("compile-inflight-limit", "Limit on parallel programs compilation").OptionalArgument("NUM").StoreResult(&RunOpts.CompileInflightLimit); + opts.AddHelpOption('h'); + opts.SetFreeArgTitle(0, "mbus", "- use to start message bus proxy and override it's settings"); + opts.SetFreeArgsMin(0); + opts.ArgPermutation_ = REQUIRE_ORDER; + + TOptsParseResult res(&opts, argc, argv); + ParseConfigFiles(res); + + if ((res.GetFreeArgCount() > 0) && (res.GetFreeArgs().at(0) == "mbus")) { + size_t freeArgsPos = res.GetFreeArgsPos(); + argc -= freeArgsPos; + argv += freeArgsPos; + TOpts msgBusOpts = TOpts::Default(); + SetMsgBusDefaults(RunOpts.ProxyBusSessionConfig, RunOpts.ProxyBusQueueConfig); + msgBusOpts.AddLongOption("port", "Message bus proxy port").OptionalArgument("NUM") + .DefaultValue(ToString(RunOpts.BusProxyPort)).StoreResult(&RunOpts.BusProxyPort); + msgBusOpts.AddLongOption("trace-path", "Path for trace files").Optional().RequiredArgument("PATH").StoreResult(&RunOpts.TracePath); + msgBusOpts.AddHelpOption('h'); + RunOpts.ProxyBusQueueConfig.ConfigureLastGetopt(msgBusOpts, ""); + RunOpts.ProxyBusSessionConfig.ConfigureLastGetopt(msgBusOpts, ""); + TOptsParseResult mbusRes(&msgBusOpts, argc, argv); Y_UNUSED(mbusRes); - RunOpts.StartTracingBusProxy = !RunOpts.TracePath.empty(); - RunOpts.StartBusProxy = true; - } - + RunOpts.StartTracingBusProxy = !RunOpts.TracePath.empty(); + RunOpts.StartBusProxy = true; + } + Y_VERIFY(RunOpts.NodeId > 0 && RunOpts.NodeId <= NActors::TActorId::MaxNodeId); -} - -void TRunCommandConfigParser::ApplyParsedOptions() { - // apply global options - Config.AppConfig.MutableInterconnectConfig()->SetStartTcp(GlobalOpts.StartTcp); - auto logConfig = Config.AppConfig.MutableLogConfig(); - logConfig->SetSysLog(GlobalOpts.SysLog); - logConfig->SetDefaultLevel(GlobalOpts.LogLevel); - logConfig->SetDefaultSamplingLevel(GlobalOpts.LogSamplingLevel); - logConfig->SetDefaultSamplingRate(GlobalOpts.LogSamplingRate); - logConfig->SetFormat(GlobalOpts.LogFormat); - logConfig->SetClusterName(GlobalOpts.ClusterName); - +} + +void TRunCommandConfigParser::ApplyParsedOptions() { + // apply global options + Config.AppConfig.MutableInterconnectConfig()->SetStartTcp(GlobalOpts.StartTcp); + auto logConfig = Config.AppConfig.MutableLogConfig(); + logConfig->SetSysLog(GlobalOpts.SysLog); + logConfig->SetDefaultLevel(GlobalOpts.LogLevel); + logConfig->SetDefaultSamplingLevel(GlobalOpts.LogSamplingLevel); + logConfig->SetDefaultSamplingRate(GlobalOpts.LogSamplingRate); + logConfig->SetFormat(GlobalOpts.LogFormat); + logConfig->SetClusterName(GlobalOpts.ClusterName); + // apply options affecting UDF paths Config.AppConfig.SetUDFsDir(GlobalOpts.UDFsDir); for (const auto& path : GlobalOpts.UDFsPaths) { Config.AppConfig.AddUDFsPaths(path); } - // apply run options - Config.NodeId = RunOpts.NodeId; - auto messageBusConfig = Config.AppConfig.MutableMessageBusConfig(); - - messageBusConfig->SetStartBusProxy(RunOpts.StartBusProxy); - messageBusConfig->SetBusProxyPort(RunOpts.BusProxyPort); - - auto queueConfig = messageBusConfig->MutableProxyBusQueueConfig(); - queueConfig->SetName(RunOpts.ProxyBusQueueConfig.Name); - queueConfig->SetNumWorkers(RunOpts.ProxyBusQueueConfig.NumWorkers); - - auto sessionConfig = messageBusConfig->MutableProxyBusSessionConfig(); - - // TODO use macro from messagebus header file - sessionConfig->SetName(RunOpts.ProxyBusSessionConfig.Name); - sessionConfig->SetNumRetries(RunOpts.ProxyBusSessionConfig.NumRetries); - sessionConfig->SetRetryInterval(RunOpts.ProxyBusSessionConfig.RetryInterval); - sessionConfig->SetReconnectWhenIdle(RunOpts.ProxyBusSessionConfig.ReconnectWhenIdle); - sessionConfig->SetMaxInFlight(RunOpts.ProxyBusSessionConfig.MaxInFlight); - sessionConfig->SetPerConnectionMaxInFlight(RunOpts.ProxyBusSessionConfig.PerConnectionMaxInFlight); - sessionConfig->SetPerConnectionMaxInFlightBySize(RunOpts.ProxyBusSessionConfig.PerConnectionMaxInFlightBySize); - sessionConfig->SetMaxInFlightBySize(RunOpts.ProxyBusSessionConfig.MaxInFlightBySize); - sessionConfig->SetTotalTimeout(RunOpts.ProxyBusSessionConfig.TotalTimeout); - sessionConfig->SetSendTimeout(RunOpts.ProxyBusSessionConfig.SendTimeout); - sessionConfig->SetConnectTimeout(RunOpts.ProxyBusSessionConfig.ConnectTimeout); - sessionConfig->SetDefaultBufferSize(RunOpts.ProxyBusSessionConfig.DefaultBufferSize); - sessionConfig->SetMaxBufferSize(RunOpts.ProxyBusSessionConfig.MaxBufferSize); - sessionConfig->SetSocketRecvBufferSize(RunOpts.ProxyBusSessionConfig.SocketRecvBufferSize); - sessionConfig->SetSocketSendBufferSize(RunOpts.ProxyBusSessionConfig.SocketSendBufferSize); - sessionConfig->SetSocketToS(RunOpts.ProxyBusSessionConfig.SocketToS); - sessionConfig->SetSendThreshold(RunOpts.ProxyBusSessionConfig.SendThreshold); - sessionConfig->SetCork(RunOpts.ProxyBusSessionConfig.Cork.MilliSeconds()); - sessionConfig->SetMaxMessageSize(RunOpts.ProxyBusSessionConfig.MaxMessageSize); - sessionConfig->SetTcpNoDelay(RunOpts.ProxyBusSessionConfig.TcpNoDelay); - sessionConfig->SetTcpCork(RunOpts.ProxyBusSessionConfig.TcpCork); - sessionConfig->SetExecuteOnMessageInWorkerPool(RunOpts.ProxyBusSessionConfig.ExecuteOnMessageInWorkerPool); - sessionConfig->SetExecuteOnReplyInWorkerPool(RunOpts.ProxyBusSessionConfig.ExecuteOnReplyInWorkerPool); - sessionConfig->SetListenPort(RunOpts.ProxyBusSessionConfig.ListenPort); - - for (auto proxy : RunOpts.ProxyBindToProxy) { - messageBusConfig->AddProxyBindToProxy(proxy); - } - messageBusConfig->SetStartTracingBusProxy(RunOpts.StartTracingBusProxy); - messageBusConfig->SetTracePath(RunOpts.TracePath); - + // apply run options + Config.NodeId = RunOpts.NodeId; + auto messageBusConfig = Config.AppConfig.MutableMessageBusConfig(); + + messageBusConfig->SetStartBusProxy(RunOpts.StartBusProxy); + messageBusConfig->SetBusProxyPort(RunOpts.BusProxyPort); + + auto queueConfig = messageBusConfig->MutableProxyBusQueueConfig(); + queueConfig->SetName(RunOpts.ProxyBusQueueConfig.Name); + queueConfig->SetNumWorkers(RunOpts.ProxyBusQueueConfig.NumWorkers); + + auto sessionConfig = messageBusConfig->MutableProxyBusSessionConfig(); + + // TODO use macro from messagebus header file + sessionConfig->SetName(RunOpts.ProxyBusSessionConfig.Name); + sessionConfig->SetNumRetries(RunOpts.ProxyBusSessionConfig.NumRetries); + sessionConfig->SetRetryInterval(RunOpts.ProxyBusSessionConfig.RetryInterval); + sessionConfig->SetReconnectWhenIdle(RunOpts.ProxyBusSessionConfig.ReconnectWhenIdle); + sessionConfig->SetMaxInFlight(RunOpts.ProxyBusSessionConfig.MaxInFlight); + sessionConfig->SetPerConnectionMaxInFlight(RunOpts.ProxyBusSessionConfig.PerConnectionMaxInFlight); + sessionConfig->SetPerConnectionMaxInFlightBySize(RunOpts.ProxyBusSessionConfig.PerConnectionMaxInFlightBySize); + sessionConfig->SetMaxInFlightBySize(RunOpts.ProxyBusSessionConfig.MaxInFlightBySize); + sessionConfig->SetTotalTimeout(RunOpts.ProxyBusSessionConfig.TotalTimeout); + sessionConfig->SetSendTimeout(RunOpts.ProxyBusSessionConfig.SendTimeout); + sessionConfig->SetConnectTimeout(RunOpts.ProxyBusSessionConfig.ConnectTimeout); + sessionConfig->SetDefaultBufferSize(RunOpts.ProxyBusSessionConfig.DefaultBufferSize); + sessionConfig->SetMaxBufferSize(RunOpts.ProxyBusSessionConfig.MaxBufferSize); + sessionConfig->SetSocketRecvBufferSize(RunOpts.ProxyBusSessionConfig.SocketRecvBufferSize); + sessionConfig->SetSocketSendBufferSize(RunOpts.ProxyBusSessionConfig.SocketSendBufferSize); + sessionConfig->SetSocketToS(RunOpts.ProxyBusSessionConfig.SocketToS); + sessionConfig->SetSendThreshold(RunOpts.ProxyBusSessionConfig.SendThreshold); + sessionConfig->SetCork(RunOpts.ProxyBusSessionConfig.Cork.MilliSeconds()); + sessionConfig->SetMaxMessageSize(RunOpts.ProxyBusSessionConfig.MaxMessageSize); + sessionConfig->SetTcpNoDelay(RunOpts.ProxyBusSessionConfig.TcpNoDelay); + sessionConfig->SetTcpCork(RunOpts.ProxyBusSessionConfig.TcpCork); + sessionConfig->SetExecuteOnMessageInWorkerPool(RunOpts.ProxyBusSessionConfig.ExecuteOnMessageInWorkerPool); + sessionConfig->SetExecuteOnReplyInWorkerPool(RunOpts.ProxyBusSessionConfig.ExecuteOnReplyInWorkerPool); + sessionConfig->SetListenPort(RunOpts.ProxyBusSessionConfig.ListenPort); + + for (auto proxy : RunOpts.ProxyBindToProxy) { + messageBusConfig->AddProxyBindToProxy(proxy); + } + messageBusConfig->SetStartTracingBusProxy(RunOpts.StartTracingBusProxy); + messageBusConfig->SetTracePath(RunOpts.TracePath); + Config.AppConfig.MutableBootstrapConfig()->MutableCompileServiceConfig()->SetInflightLimit(RunOpts.CompileInflightLimit); - Config.AppConfig.MutableMonitoringConfig()->SetMonitoringPort(RunOpts.MonitoringPort); + Config.AppConfig.MutableMonitoringConfig()->SetMonitoringPort(RunOpts.MonitoringPort); Config.AppConfig.MutableMonitoringConfig()->SetMonitoringAddress(RunOpts.MonitoringAddress); Config.AppConfig.MutableMonitoringConfig()->SetMonitoringThreads(RunOpts.MonitoringThreads); - Config.AppConfig.MutableRestartsCountConfig()->SetRestartsCountFile(RunOpts.RestartsCountFile); -} - -} // namespace NKikimr + Config.AppConfig.MutableRestartsCountConfig()->SetRestartsCountFile(RunOpts.RestartsCountFile); +} + +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/config_parser.h b/ydb/core/driver_lib/run/config_parser.h index 7ebde9a4c57..5825ec6c49e 100644 --- a/ydb/core/driver_lib/run/config_parser.h +++ b/ydb/core/driver_lib/run/config_parser.h @@ -1,65 +1,65 @@ -#pragma once - -#include "config.h" +#pragma once + +#include "config.h" #include <ydb/core/protos/config.pb.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> - + #include <library/cpp/getopt/last_getopt.h> - + #include <library/cpp/messagebus/session_config.h> #include <library/cpp/messagebus/queue_config.h> - -namespace NKikimr { - -class TRunCommandConfigParser { -public: - TRunCommandConfigParser(TKikimrRunConfig& config); - virtual ~TRunCommandConfigParser(); - - virtual void ParseRunOpts(int argc, char **argv); - - virtual void SetupGlobalOpts(NLastGetopt::TOpts& opts); - virtual void ParseGlobalOpts(const NLastGetopt::TOptsParseResult& res); - virtual void SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts); - virtual void ParseConfigFiles(const NLastGetopt::TOptsParseResult& res); - virtual void ApplyParsedOptions(); - -protected: - struct TGlobalOpts { - bool StartTcp; // interconnect config - bool SysLog; // log settings - ui32 LogLevel; // log settings - ui32 LogSamplingLevel; // log settings - ui32 LogSamplingRate; // log settings + +namespace NKikimr { + +class TRunCommandConfigParser { +public: + TRunCommandConfigParser(TKikimrRunConfig& config); + virtual ~TRunCommandConfigParser(); + + virtual void ParseRunOpts(int argc, char **argv); + + virtual void SetupGlobalOpts(NLastGetopt::TOpts& opts); + virtual void ParseGlobalOpts(const NLastGetopt::TOptsParseResult& res); + virtual void SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts); + virtual void ParseConfigFiles(const NLastGetopt::TOptsParseResult& res); + virtual void ApplyParsedOptions(); + +protected: + struct TGlobalOpts { + bool StartTcp; // interconnect config + bool SysLog; // log settings + ui32 LogLevel; // log settings + ui32 LogSamplingLevel; // log settings + ui32 LogSamplingRate; // log settings TString LogFormat;// log settings TString ClusterName; // log settings TString UDFsDir; // directory to recurcively load UDFs from TVector<TString> UDFsPaths; // fine tunned UDFs list to load - - TGlobalOpts(); - }; - - struct TRunOpts { - ui32 NodeId; - bool StartBusProxy; - ui32 BusProxyPort; - NBus::TBusQueueConfig ProxyBusQueueConfig; - NBus::TBusServerSessionConfig ProxyBusSessionConfig; + + TGlobalOpts(); + }; + + struct TRunOpts { + ui32 NodeId; + bool StartBusProxy; + ui32 BusProxyPort; + NBus::TBusQueueConfig ProxyBusQueueConfig; + NBus::TBusServerSessionConfig ProxyBusSessionConfig; TVector<ui64> ProxyBindToProxy; - ui32 MonitoringPort; + ui32 MonitoringPort; TString MonitoringAddress; ui32 MonitoringThreads; TString RestartsCountFile; - bool StartTracingBusProxy; + bool StartTracingBusProxy; TString TracePath; - size_t CompileInflightLimit; // MiniKQLCompileService - - TRunOpts(); - }; - - TKikimrRunConfig& Config; - TGlobalOpts GlobalOpts; - TRunOpts RunOpts; -}; - -} + size_t CompileInflightLimit; // MiniKQLCompileService + + TRunOpts(); + }; + + TKikimrRunConfig& Config; + TGlobalOpts GlobalOpts; + TRunOpts RunOpts; +}; + +} diff --git a/ydb/core/driver_lib/run/driver.h b/ydb/core/driver_lib/run/driver.h index 249b1a94efc..7ad96718d11 100644 --- a/ydb/core/driver_lib/run/driver.h +++ b/ydb/core/driver_lib/run/driver.h @@ -1,20 +1,20 @@ -#pragma once - +#pragma once + #include <ydb/core/driver_lib/cli_config_base/config_base.h> - + #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/stream/file.h> -#include <util/stream/format.h> - +#include <util/stream/file.h> +#include <util/stream/format.h> + #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/string_utils/parse_size/parse_size.h> #include <library/cpp/svnversion/svnversion.h> - - -namespace NKikimr { - -#define MODE_MAP(XX) \ - XX(EDM_RUN, "run", "run kikimr node") \ + + +namespace NKikimr { + +#define MODE_MAP(XX) \ + XX(EDM_RUN, "run", "run kikimr node") \ XX(EDM_SERVER, "server", "run kikimr node") \ XX(EDM_ADMIN, "admin", "admin running kikimr") \ XX(EDM_DB, "db", "admin running kikimr") \ @@ -25,11 +25,11 @@ namespace NKikimr { XX(EDM_CMS, "cms", "admin running kikimr") \ XX(EDM_DISCOVERY, "discovery", "discover endpoints") \ XX(EDM_WHOAMI, "whoami", "admin running kikimr") \ - XX(EDM_FORMAT_INFO, "format-info", "read pdisk format info") \ - XX(EDM_FORMAT_UTIL, "format-util", "query blob storage format configuration file") \ - XX(EDM_NODE_BY_HOST, "node-by-host", "get node id by hostname") \ - XX(EDM_SCHEME_INITROOT, "scheme-initroot", "init scheme root") \ - XX(EDM_COMPILE_AND_EXEC_MINIKQL, "minikql-exec", "compile and execute MiniKQL program") \ + XX(EDM_FORMAT_INFO, "format-info", "read pdisk format info") \ + XX(EDM_FORMAT_UTIL, "format-util", "query blob storage format configuration file") \ + XX(EDM_NODE_BY_HOST, "node-by-host", "get node id by hostname") \ + XX(EDM_SCHEME_INITROOT, "scheme-initroot", "init scheme root") \ + XX(EDM_COMPILE_AND_EXEC_MINIKQL, "minikql-exec", "compile and execute MiniKQL program") \ XX(EDM_TRACE, "mb-trace", "control message bus trace") \ XX(EDM_KEYVALUE_REQUEST, "keyvalue-request", "send protobuf request to a keyvalue tablet") \ XX(EDM_PERSQUEUE_REQUEST, "persqueue-request", "send protobuf request to a persqueue tablet") \ @@ -37,8 +37,8 @@ namespace NKikimr { XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \ XX(EDM_LOAD_REQUEST, "bs-load-test", "send protobuf request to blobstorage test load actor (https://wiki.yandex-team.ru/kikimr/developers/BSLoadTest/)") \ XX(EDM_ACTORSYS_PERFTEST, "actorsys-perf-test", "make actorsystem performance test") \ - -CLI_MODES_IMPL(EDriverMode, EDM_NO, MODE_MAP); - -} + +CLI_MODES_IMPL(EDriverMode, EDM_NO, MODE_MAP); + +} diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 604107d0930..819c1478d19 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -1,13 +1,13 @@ -#include "config.h" +#include "config.h" #include "kikimr_services_initializers.h" -#include "service_initializer.h" +#include "service_initializer.h" #include "version.h" - + #include <ydb/core/actorlib_impl/destruct_actor.h> #include <ydb/core/actorlib_impl/load_network.h> #include <ydb/core/actorlib_impl/mad_squirrel.h> #include <ydb/core/actorlib_impl/node_identifier.h> - + #include <ydb/core/base/appdata.h> #include <ydb/core/base/config_units.h> #include <ydb/core/base/counters.h> @@ -59,7 +59,7 @@ #include <ydb/core/kqp/kqp.h> #include <ydb/core/kqp/rm/kqp_rm.h> - + #include <ydb/core/metering/metering.h> #include <ydb/core/mind/address_classification/net_classifier.h> @@ -86,9 +86,9 @@ #include <ydb/core/protos/services.pb.h> #include <ydb/core/protos/console_config.pb.h> - + #include <ydb/core/quoter/quoter_service.h> - + #include <ydb/core/scheme/scheme_type_registry.h> #include <ydb/core/security/ticket_parser.h> @@ -103,7 +103,7 @@ #include <ydb/core/tablet/tablet_list_renderer.h> #include <ydb/core/tablet/tablet_monitoring_proxy.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> - + #include <ydb/core/tracing/tablet_info.h> #include <ydb/core/tx/coordinator/coordinator.h> @@ -122,11 +122,11 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/long_tx_service/public/events.h> #include <ydb/core/tx/long_tx_service/long_tx_service.h> - + #include <ydb/core/util/failure_injection.h> #include <ydb/core/util/memory_tracker.h> #include <ydb/core/util/sig.h> - + #include <ydb/core/viewer/viewer.h> #include <ydb/public/lib/deprecated/client/msgbus_client.h> @@ -175,7 +175,7 @@ #include <library/cpp/logger/log.h> #include <library/cpp/monlib/messagebus/mon_messagebus.h> - + #include <library/cpp/svnversion/svnversion.h> #include <library/cpp/lwtrace/mon/mon_lwtrace.h> @@ -186,20 +186,20 @@ #include <util/system/hostname.h> -namespace NKikimr { - -namespace NKikimrServicesInitializers { - +namespace NKikimr { + +namespace NKikimrServicesInitializers { + ui32 TYandexQueryInitializer::IcPort = 0; -IKikimrServicesInitializer::IKikimrServicesInitializer(const TKikimrRunConfig& runConfig) - : Config(runConfig.AppConfig) +IKikimrServicesInitializer::IKikimrServicesInitializer(const TKikimrRunConfig& runConfig) + : Config(runConfig.AppConfig) , NodeId(runConfig.NodeId) , ScopeId(runConfig.ScopeId) {} - -// TBasicServicesInitializer - + +// TBasicServicesInitializer + template <class TConfig> static TCpuMask ParseAffinity(const TConfig& cfg) { TCpuMask result; @@ -288,11 +288,11 @@ void AddExecutorPool( cpuManager.United.emplace_back(std::move(united)); break; } - default: + default: Y_FAIL(); - } -} - + } +} + static TUnitedWorkersConfig CreateUnitedWorkersConfig(const NKikimrConfig::TActorSystemConfig::TUnitedWorkers& config, ui32 unitedThreads) { TUnitedWorkersConfig result; result.CpuCount = unitedThreads; @@ -343,20 +343,20 @@ static TCpuManagerConfig CreateCpuManagerConfig(const NKikimrConfig::TActorSyste } static TSchedulerConfig CreateSchedulerConfig(const NKikimrConfig::TActorSystemConfig::TScheduler &config) { - const ui64 resolution = config.HasResolution() ? config.GetResolution() : 1024; + const ui64 resolution = config.HasResolution() ? config.GetResolution() : 1024; Y_VERIFY_DEBUG((resolution & (resolution - 1)) == 0); // resolution must be power of 2 - const ui64 spinThreshold = config.HasSpinThreshold() ? config.GetSpinThreshold() : 0; - const ui64 progressThreshold = config.HasProgressThreshold() ? config.GetProgressThreshold() : 10000; + const ui64 spinThreshold = config.HasSpinThreshold() ? config.GetSpinThreshold() : 0; + const ui64 progressThreshold = config.HasProgressThreshold() ? config.GetProgressThreshold() : 10000; const bool useSchedulerActor = config.HasUseSchedulerActor() ? config.GetUseSchedulerActor() : false; - + return TSchedulerConfig(resolution, spinThreshold, progressThreshold, useSchedulerActor); -} - +} + TBasicServicesInitializer::TBasicServicesInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) + : IKikimrServicesInitializer(runConfig) { -} - +} + static ui32 GetInterconnectThreadPoolId(const NKikimr::TAppData* appData) { Y_VERIFY_DEBUG(appData != nullptr); auto item = appData->ServicePools.find("Interconnect"); @@ -504,18 +504,18 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter return result; } -void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { Y_VERIFY(Config.HasActorSystemConfig()); - + auto& systemConfig = Config.GetActorSystemConfig(); Y_VERIFY(systemConfig.HasScheduler()); Y_VERIFY(systemConfig.ExecutorSize()); - const ui32 systemPoolId = appData->SystemPoolId; - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters = appData->Counters; - - setup->NodeId = NodeId; + const ui32 systemPoolId = appData->SystemPoolId; + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters = appData->Counters; + + setup->NodeId = NodeId; setup->MaxActivityType = GetActivityTypeCount(); setup->CpuManager = CreateCpuManagerConfig(systemConfig, setup->MaxActivityType); for (ui32 poolId = 0; poolId != setup->GetExecutorsCount(); ++poolId) { @@ -525,30 +525,30 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(TActorId(), TActorSetupCmd(CreateMadSquirrel(), TMailboxType::HTSwap, poolId))); } } - } + } auto schedulerConfig = CreateSchedulerConfig(systemConfig.GetScheduler()); schedulerConfig.MonCounters = GetServiceCounters(counters, "utils"); setup->Scheduler.Reset(CreateSchedulerThread(schedulerConfig)); setup->LocalServices.emplace_back(MakeIoDispatcherActorId(), TActorSetupCmd(CreateIoDispatcherActor( schedulerConfig.MonCounters->GetSubgroup("subsystem", "io_dispatcher")), TMailboxType::HTSwap, systemPoolId)); - + NLwTraceMonPage::DashboardRegistry().Register(NActors::LWTraceDashboards(setup)); - if (Config.HasNameserviceConfig()) { + if (Config.HasNameserviceConfig()) { const auto& nsConfig = Config.GetNameserviceConfig(); const TActorId resolverId = NDnsResolver::MakeDnsResolverActorId(); const TActorId nameserviceId = GetNameserviceActorId(); - + ui32 numNodes = 0; TSet<TString> dataCenters; - TIntrusivePtr<TTableNameserverSetup> table(new TTableNameserverSetup()); + TIntrusivePtr<TTableNameserverSetup> table(new TTableNameserverSetup()); for (const auto &node : nsConfig.GetNode()) { - const ui32 nodeId = node.GetNodeId(); + const ui32 nodeId = node.GetNodeId(); const TString host = node.HasHost() ? node.GetHost() : TString(); - const ui32 port = node.GetPort(); - + const ui32 port = node.GetPort(); + const TString resolveHost = node.HasInterconnectHost() ? node.GetInterconnectHost() : host; @@ -566,8 +566,8 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s ++numNodes; dataCenters.insert(location.GetDataCenterId()); - } - + } + NDnsResolver::TOnDemandDnsResolverOptions resolverOptions; resolverOptions.MonCounters = GetServiceCounters(counters, "utils")->GetSubgroup("subsystem", "dns_resolver"); IActor *resolver = NDnsResolver::CreateOnDemandDnsResolver(resolverOptions); @@ -599,14 +599,14 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->LocalServices.emplace_back( nameserviceId, TActorSetupCmd(nameservice, TMailboxType::HTSwap, systemPoolId)); - + if (Config.HasInterconnectConfig() && Config.GetInterconnectConfig().GetStartTcp()) { const auto& icConfig = Config.GetInterconnectConfig(); - TChannelsConfig channels; + TChannelsConfig channels; auto settings = GetInterconnectSettings(icConfig, numNodes, dataCenters.size()); ui32 interconnectPoolId = GetInterconnectThreadPoolId(appData); - + for (const auto& channel : icConfig.GetChannel()) { const auto index = channel.GetIndex(); ui32 weight = 0; @@ -616,13 +616,13 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s } else if (channel.HasQuota()) { weight = channel.GetQuota(); } - + Y_VERIFY(index < 1U << IEventHandle::ChannelBits, "Channel index is too large: got %" PRIu32 ", should be less than %" PRIu32, index, 1U << IEventHandle::ChannelBits); Y_VERIFY(weight > 0U && weight <= std::numeric_limits<ui16>::max(), "Channel weight is out of allowed range: got %" PRIu32 ", should be > 0 and < %" PRIu32, weight, std::numeric_limits<ui16>::max()); channels.insert({ui16(index), TChannelSettings{ui16(weight)}}); - } - + } + // create poller actor (whether platform supports it) setup->LocalServices.emplace_back(MakePollerActorId(), TActorSetupCmd(CreatePollerActor(), TMailboxType::ReadAsFilled, systemPoolId)); @@ -639,7 +639,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s icCommon->HandshakeBallastSize = icConfig.GetHandshakeBallastSize(); icCommon->LocalScopeId = ScopeId.GetInterconnectScopeId(); icCommon->Cookie = icConfig.GetSuppressConnectivityCheck() ? TString() : CreateGuidAsString(); - + #define CHANNEL(NAME) {TInterconnectChannels::NAME, #NAME} icCommon->ChannelName = { CHANNEL(IC_COMMON), @@ -701,7 +701,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s Y_VERIFY(!table->StaticNodeTable.empty()); ui32 maxNode = 0; - for (const auto &node : table->StaticNodeTable) { + for (const auto &node : table->StaticNodeTable) { maxNode = Max(maxNode, node.first); } setup->Interconnect.ProxyActors.resize(maxNode + 1); @@ -710,12 +710,12 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s std::unordered_set<ui32> staticIds; for (const auto& node : table->StaticNodeTable) { - const ui32 destId = node.first; - if (destId != NodeId) { + const ui32 destId = node.first; + if (destId != NodeId) { staticIds.insert(destId); setup->Interconnect.ProxyActors[destId] = TActorSetupCmd(new TInterconnectProxyTCP(destId, icCommon), TMailboxType::ReadAsFilled, interconnectPoolId); - } else { + } else { TYandexQueryInitializer::SetIcPort(node.second.second); icCommon->TechnicalSelfHostName = node.second.Host; TString address = "::"; //bind ipv6 interfaces by default @@ -730,8 +730,8 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s } setup->LocalServices.emplace_back(MakeInterconnectListenerActorId(false), TActorSetupCmd(listener, TMailboxType::ReadAsFilled, interconnectPoolId)); - } - } + } + } // Prepare listener for dynamic node. if (Config.GetDynamicNodeConfig().HasNodeInfo()) { @@ -758,10 +758,10 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, systemPoolId)); //IC_Load::InitializeService(setup, appData, maxNode); - } - } -} - + } + } +} + // TImmediateControlBoardInitializer TImmediateControlBoardInitializer::TImmediateControlBoardInitializer(const TKikimrRunConfig& runConfig) @@ -783,16 +783,16 @@ void TImmediateControlBoardInitializer::InitializeServices(NActors::TActorSystem } -// TBSNodeWardenInitializer - -TBSNodeWardenInitializer::TBSNodeWardenInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +// TBSNodeWardenInitializer + +TBSNodeWardenInitializer::TBSNodeWardenInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { TIntrusivePtr<TNodeWardenConfig> nodeWardenConfig(new TNodeWardenConfig(new TRealPDiskServiceFactory())); - if (Config.HasBlobStorageConfig()) { + if (Config.HasBlobStorageConfig()) { const auto& bsc = Config.GetBlobStorageConfig(); appData->StaticBlobStorageConfig->MergeFrom(bsc.GetServiceSet()); nodeWardenConfig->FeatureFlags = Config.GetFeatureFlags(); @@ -810,7 +810,7 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se nodeWardenConfig->CacheVDisks = bsc.GetCacheVDisks(); nodeWardenConfig->EnableVDiskCooldownTimeout = true; } - + ObtainTenantKey(&nodeWardenConfig->TenantKey, Config.GetKeyConfig()); ObtainStaticKey(&nodeWardenConfig->StaticKey); ObtainPDiskKey(&nodeWardenConfig->PDiskKey, Config.GetPDiskKeyConfig()); @@ -821,10 +821,10 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se setup->LocalServices.emplace_back(MakeUniversalSchedulerActorId(), TActorSetupCmd(CreateUniversalSchedulerActor(), TMailboxType::ReadAsFilled, appData->SystemPoolId)); -} - -// TStateStorageServiceInitializer - +} + +// TStateStorageServiceInitializer + template<typename TCreateFunc> void StartLocalStateStorageReplicas(TCreateFunc createFunc, TStateStorageInfo *info, ui32 poolId, TActorSystemSetup &setup) { ui32 index = 0; @@ -834,30 +834,30 @@ void StartLocalStateStorageReplicas(TCreateFunc createFunc, TStateStorageInfo *i setup.LocalServices.emplace_back( replica, TActorSetupCmd(createFunc(info, index), TMailboxType::ReadAsFilled, poolId)); - } + } ++index; - } - } -} - -TStateStorageServiceInitializer::TStateStorageServiceInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TStateStorageServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - // setup state storage stuff - const ui32 maxssid = 255; - bool knownss[maxssid + 1] = {}; - for (const NKikimrConfig::TDomainsConfig::TStateStorage &ssconf : Config.GetDomainsConfig().GetStateStorage()) { - const ui32 ssid = ssconf.GetSSId(); + } + } +} + +TStateStorageServiceInitializer::TStateStorageServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TStateStorageServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + // setup state storage stuff + const ui32 maxssid = 255; + bool knownss[maxssid + 1] = {}; + for (const NKikimrConfig::TDomainsConfig::TStateStorage &ssconf : Config.GetDomainsConfig().GetStateStorage()) { + const ui32 ssid = ssconf.GetSSId(); Y_VERIFY(ssid <= maxssid); - knownss[ssid] = true; - + knownss[ssid] = true; + TIntrusivePtr<TStateStorageInfo> ssrInfo; TIntrusivePtr<TStateStorageInfo> ssbInfo; TIntrusivePtr<TStateStorageInfo> sbrInfo; - + BuildStateStorageInfos(ssconf, ssrInfo, ssbInfo, sbrInfo); StartLocalStateStorageReplicas(CreateStateStorageReplica, ssrInfo.Get(), appData->SystemPoolId, *setup); @@ -867,44 +867,44 @@ void TStateStorageServiceInitializer::InitializeServices(NActors::TActorSystemSe setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeStateStorageProxyID(ssid), TActorSetupCmd(CreateStateStorageProxy(ssrInfo.Get(), ssbInfo.Get(), sbrInfo.Get()), TMailboxType::ReadAsFilled, - appData->SystemPoolId))); - } - for (ui32 ssid = 0; ssid <= maxssid; ++ssid) { - if (!knownss[ssid]) + appData->SystemPoolId))); + } + for (ui32 ssid = 0; ssid <= maxssid; ++ssid) { + if (!knownss[ssid]) setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeStateStorageProxyID(ssid), - TActorSetupCmd(CreateStateStorageProxyStub(), + TActorSetupCmd(CreateStateStorageProxyStub(), TMailboxType::HTSwap, - appData->SystemPoolId))); - } + appData->SystemPoolId))); + } setup->LocalServices.emplace_back( TActorId(), TActorSetupCmd(CreateTenantNodeEnumerationPublisher(), TMailboxType::HTSwap, appData->SystemPoolId) ); -} - -// TLocalServiceInitializer - -TLocalServiceInitializer::TLocalServiceInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - -void TLocalServiceInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +} + +// TLocalServiceInitializer + +TLocalServiceInitializer::TLocalServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + +void TLocalServiceInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { // choose pool id for important tablets ui32 importantPoolId = appData->UserPoolId; if (Config.GetFeatureFlags().GetImportantTabletsUseSystemPool()) { importantPoolId = appData->SystemPoolId; } - // setup local + // setup local TLocalConfig::TPtr localConfig(new TLocalConfig()); localConfig->TabletClassInfo[appData->DefaultTabletTypes.SchemeShard] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreateFlatTxSchemeShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.DataShard] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreateDataShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); - localConfig->TabletClassInfo[appData->DefaultTabletTypes.KeyValue] = TLocalConfig::TTabletClassInfo( + localConfig->TabletClassInfo[appData->DefaultTabletTypes.KeyValue] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreateKeyValueFlat, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.PersQueue] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreatePersQueue, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); @@ -928,7 +928,7 @@ void TLocalServiceInitializer::InitializeServices( new TTabletSetupInfo(&NSequenceShard::CreateSequenceShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.ReplicationController] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&NReplication::CreateController, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); - + TTenantPoolConfig::TPtr tenantPoolConfig = new TTenantPoolConfig(Config.GetTenantPoolConfig(), localConfig); if (!tenantPoolConfig->IsEnabled && (!tenantPoolConfig->StaticSlots.empty() || !tenantPoolConfig->DynamicSlots.empty())) @@ -940,8 +940,8 @@ void TLocalServiceInitializer::InitializeServices( for (const auto &domain : Config.GetDomainsConfig().GetDomain()) { tenantPoolConfig->AddStaticSlot(domain.GetName()); } - } - + } + setup->LocalServices.push_back(std::make_pair(MakeTenantPoolRootID(), TActorSetupCmd(CreateTenantPool(tenantPoolConfig), TMailboxType::ReadAsFilled, 0))); @@ -954,8 +954,8 @@ void TLocalServiceInitializer::InitializeServices( NTestShard::CreateStateServerInterfaceActor(), TMailboxType::ReadAsFilled, 0)); NKesus::AddKesusProbesList(); -} - +} + // TSharedCacheInitializer TSharedCacheInitializer::TSharedCacheInitializer(const TKikimrRunConfig& runConfig) @@ -1013,28 +1013,28 @@ void TBlobCacheInitializer::InitializeServices( TActorSetupCmd(NBlobCache::CreateBlobCache(DEFAULT_CACHE_SIZE_BYTES, blobCacheGroup), TMailboxType::ReadAsFilled, appData->UserPoolId))); } -// TLoggerInitializer - -TLoggerInitializer::TLoggerInitializer(const TKikimrRunConfig& runConfig, +// TLoggerInitializer + +TLoggerInitializer::TLoggerInitializer(const TKikimrRunConfig& runConfig, TIntrusivePtr<NActors::NLog::TSettings> logSettings, std::shared_ptr<TLogBackend> logBackend) - : IKikimrServicesInitializer(runConfig) + : IKikimrServicesInitializer(runConfig) , LogSettings(logSettings) , LogBackend(logBackend) , PathToConfigCacheFile(runConfig.PathToConfigCacheFile) { } -void TLoggerInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +void TLoggerInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { const TIntrusivePtr<NMonitoring::TDynamicCounters> utilsCounters = GetServiceCounters(appData->Counters, "utils"); - - // log settings must be initialized before calling this method + + // log settings must be initialized before calling this method NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, LogBackend, utilsCounters); NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::HTSwap, appData->IOPoolId); std::pair<NActors::TActorId, NActors::TActorSetupCmd> loggerActorPair(LogSettings->LoggerActorId, loggerActorCmd); - setup->LocalServices.push_back(loggerActorPair); + setup->LocalServices.push_back(loggerActorPair); IActor *configurator; if (PathToConfigCacheFile && !appData->FeatureFlags.GetEnableConfigurationCache()) { @@ -1045,8 +1045,8 @@ void TLoggerInitializer::InitializeServices( setup->LocalServices.emplace_back(TActorId(), TActorSetupCmd(configurator, TMailboxType::HTSwap, appData->UserPoolId)); -} - +} + // TSchedulerActorInitializer TSchedulerActorInitializer::TSchedulerActorInitializer(const TKikimrRunConfig& runConfig) @@ -1064,33 +1064,33 @@ void TSchedulerActorInitializer::InitializeServices( } } -// TProfilerInitializer - -TProfilerInitializer::TProfilerInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TProfilerInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +// TProfilerInitializer + +TProfilerInitializer::TProfilerInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TProfilerInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { const TIntrusivePtr<NMonitoring::TDynamicCounters> utilsCounters = GetServiceCounters(appData->Counters, "utils"); - + TActorSetupCmd profilerSetup(CreateProfilerActor(utilsCounters, "/var/tmp"), TMailboxType::HTSwap, 0); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeProfilerID(NodeId), profilerSetup)); -} - +} + // TResourceBrokerInitializer - + TResourceBrokerInitializer::TResourceBrokerInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - + : IKikimrServicesInitializer(runConfig) { +} + void TResourceBrokerInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { NKikimrResourceBroker::TResourceBrokerConfig config = NResourceBroker::MakeDefaultConfig(); - if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasCompactionBroker()) { + if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasCompactionBroker()) { Y_FAIL("Legacy CompactionBroker configuration is no longer supported"); } @@ -1100,63 +1100,63 @@ void TResourceBrokerInitializer::InitializeServices( if (Config.HasResourceBrokerConfig()) { NResourceBroker::MergeConfigUpdates(config, Config.GetResourceBrokerConfig()); - } - + } + auto counters = GetServiceCounters(appData->Counters, "tablets"); TActorSetupCmd actorSetup = { NResourceBroker::CreateResourceBrokerActor(config, counters), TMailboxType::ReadAsFilled, appData->UserPoolId }; setup->LocalServices.push_back(std::make_pair(NResourceBroker::MakeResourceBrokerID(), actorSetup)); -} - -// TRestartsCountPublisher - -void TRestartsCountPublisher::PublishRestartsCount(const NMonitoring::TDynamicCounters::TCounterPtr& counter, +} + +// TRestartsCountPublisher + +void TRestartsCountPublisher::PublishRestartsCount(const NMonitoring::TDynamicCounters::TCounterPtr& counter, const TString& restartsCountFile) { if (restartsCountFile.size()) { - try { + try { TUnbufferedFileInput fileInput(restartsCountFile); const TString content = fileInput.ReadAll(); - *counter = FromString<ui32>(content); - } catch (yexception) { - *counter = 0; - } + *counter = FromString<ui32>(content); + } catch (yexception) { + *counter = 0; + } TUnbufferedFileOutput fileOutput(restartsCountFile); - fileOutput.Write(ToString(*counter+1)); - } -} - -TRestartsCountPublisher::TRestartsCountPublisher(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TRestartsCountPublisher::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { + fileOutput.Write(ToString(*counter+1)); + } +} + +TRestartsCountPublisher::TRestartsCountPublisher(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TRestartsCountPublisher::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { Y_UNUSED(setup); const TIntrusivePtr<NMonitoring::TDynamicCounters> utilsCounters = GetServiceCounters(appData->Counters, "utils"); - - if (Config.HasRestartsCountConfig()) { - const auto& restartsCountConfig = Config.GetRestartsCountConfig(); - if (restartsCountConfig.HasRestartsCountFile()) { + + if (Config.HasRestartsCountConfig()) { + const auto& restartsCountConfig = Config.GetRestartsCountConfig(); + if (restartsCountConfig.HasRestartsCountFile()) { PublishRestartsCount(utilsCounters->GetCounter("RestartsCount", false), restartsCountConfig.GetRestartsCountFile()); - } - } -} - -// TTabletResolverInitializer - -TTabletResolverInitializer::TTabletResolverInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TTabletResolverInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - TIntrusivePtr<TTabletResolverConfig> tabletResolverConfig(new TTabletResolverConfig()); + } + } +} + +// TTabletResolverInitializer + +TTabletResolverInitializer::TTabletResolverInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TTabletResolverInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + TIntrusivePtr<TTabletResolverConfig> tabletResolverConfig(new TTabletResolverConfig()); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeTabletResolverID(), TActorSetupCmd(CreateTabletResolver(tabletResolverConfig), TMailboxType::ReadAsFilled, appData->SystemPoolId))); - -} - + +} + // TTabletPipePeNodeCachesInitializer TTabletPipePeNodeCachesInitializer::TTabletPipePeNodeCachesInitializer(const TKikimrRunConfig& runConfig) @@ -1188,32 +1188,32 @@ void TTabletPipePeNodeCachesInitializer::InitializeServices( TActorSetupCmd(CreatePipePeNodeCache(followerPipeConfig), TMailboxType::ReadAsFilled, appData->UserPoolId)); } -// TTabletMonitoringProxyInitializer - -TTabletMonitoringProxyInitializer::TTabletMonitoringProxyInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TTabletMonitoringProxyInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +// TTabletMonitoringProxyInitializer + +TTabletMonitoringProxyInitializer::TTabletMonitoringProxyInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TTabletMonitoringProxyInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { NTabletMonitoringProxy::TTabletMonitoringProxyConfig proxyConfig; proxyConfig.SetRetryLimitCount(Config.GetMonitoringConfig().GetTabletMonitoringRetries()); TActorSetupCmd tabletMonitoringProxySetup(NTabletMonitoringProxy::CreateTabletMonitoringProxy(std::move(proxyConfig)), TMailboxType::ReadAsFilled, appData->UserPoolId); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NTabletMonitoringProxy::MakeTabletMonitoringProxyID(), tabletMonitoringProxySetup)); - -} - -// TTabletCountersAggregatorInitializer - -TTabletCountersAggregatorInitializer::TTabletCountersAggregatorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TTabletCountersAggregatorInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { + +} + +// TTabletCountersAggregatorInitializer + +TTabletCountersAggregatorInitializer::TTabletCountersAggregatorInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TTabletCountersAggregatorInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { { TActorSetupCmd tabletCountersAggregatorSetup(CreateTabletCountersAggregator(false), TMailboxType::ReadAsFilled, appData->UserPoolId); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeTabletCountersAggregatorID(NodeId, false), tabletCountersAggregatorSetup)); @@ -1222,8 +1222,8 @@ void TTabletCountersAggregatorInitializer::InitializeServices( TActorSetupCmd tabletCountersAggregatorSetup(CreateTabletCountersAggregator(true), TMailboxType::ReadAsFilled, appData->UserPoolId); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeTabletCountersAggregatorID(NodeId, true), tabletCountersAggregatorSetup)); } -} - +} + //TGRpcProxyStatusInitializer TGRpcProxyStatusInitializer::TGRpcProxyStatusInitializer(const TKikimrRunConfig& runConfig) @@ -1239,24 +1239,24 @@ void TGRpcProxyStatusInitializer::InitializeServices( } -// This code is shared between default kikimr bootstrapper and alternative bootstrapper - -static TIntrusivePtr<TTabletSetupInfo> CreateTablet( +// This code is shared between default kikimr bootstrapper and alternative bootstrapper + +static TIntrusivePtr<TTabletSetupInfo> CreateTablet( const TString& typeName, - const TIntrusivePtr<TTabletStorageInfo>& tabletInfo, - const TAppData* appData, - const TIntrusivePtr<ITabletFactory>& customTablets = nullptr) -{ - TIntrusivePtr<TTabletSetupInfo> tabletSetup; - if (customTablets) { - tabletSetup = customTablets->CreateTablet(typeName, tabletInfo, *appData); - if (tabletSetup) { - return tabletSetup; - } - } - + const TIntrusivePtr<TTabletStorageInfo>& tabletInfo, + const TAppData* appData, + const TIntrusivePtr<ITabletFactory>& customTablets = nullptr) +{ + TIntrusivePtr<TTabletSetupInfo> tabletSetup; + if (customTablets) { + tabletSetup = customTablets->CreateTablet(typeName, tabletInfo, *appData); + if (tabletSetup) { + return tabletSetup; + } + } + TTabletTypes::EType tabletType = TTabletTypes::StrToType(typeName); - + ui32 workPoolId = appData->UserPoolId; if (appData->FeatureFlags.GetImportantTabletsUseSystemPool()) { switch (tabletType) { @@ -1272,25 +1272,25 @@ static TIntrusivePtr<TTabletSetupInfo> CreateTablet( tabletSetup = MakeTabletSetupInfo(tabletType, workPoolId, appData->SystemPoolId); - if (tabletInfo->TabletType == TTabletTypes::TYPE_INVALID) { - tabletInfo->TabletType = tabletType; - } + if (tabletInfo->TabletType == TTabletTypes::TYPE_INVALID) { + tabletInfo->TabletType = tabletType; + } + + return tabletSetup; +} + +// TBootstrapperInitializer - return tabletSetup; -} - -// TBootstrapperInitializer - -TBootstrapperInitializer::TBootstrapperInitializer( +TBootstrapperInitializer::TBootstrapperInitializer( const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { -} - -void TBootstrapperInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - if (Config.HasBootstrapConfig()) { - for (const auto &boot : Config.GetBootstrapConfig().GetTablet()) { +} + +void TBootstrapperInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + if (Config.HasBootstrapConfig()) { + for (const auto &boot : Config.GetBootstrapConfig().GetTablet()) { if (boot.GetAllowDynamicConfiguration()) { setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( TActorId(), @@ -1299,18 +1299,18 @@ void TBootstrapperInitializer::InitializeServices( const bool standby = boot.HasStandBy() && boot.GetStandBy(); for (const ui32 bootstrapperNode : boot.GetNode()) { if (bootstrapperNode == NodeId) { - + TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(boot.GetInfo())); - + auto tabletType = BootstrapperTypeToTabletType(boot.GetType()); - + auto tabletSetupInfo = CreateTablet( - TTabletTypes::TypeToStr(tabletType), - info, - appData); - + TTabletTypes::TypeToStr(tabletType), + info, + appData); + TIntrusivePtr<TBootstrapperInfo> bi = new TBootstrapperInfo(tabletSetupInfo.Get()); - + if (boot.NodeSize() != 1) { bi->OtherNodes.reserve(boot.NodeSize() - 1); for (ui32 x : boot.GetNode()) @@ -1323,87 +1323,87 @@ void TBootstrapperInitializer::InitializeServices( } setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeBootstrapperID(info->TabletID, bootstrapperNode), TActorSetupCmd(CreateBootstrapper(info.Get(), bi.Get(), standby), TMailboxType::HTSwap, appData->SystemPoolId))); - } - } - } - } - } -} - -// alternative bootstrapper - -TTabletsInitializer::TTabletsInitializer( - const TKikimrRunConfig& runConfig, - TIntrusivePtr<ITabletFactory> customTablets) - : IKikimrServicesInitializer(runConfig) - , CustomTablets(customTablets) -{ -} - -void TTabletsInitializer::InitializeServices( - TActorSystemSetup* setup, - const TAppData* appData) -{ - if (!Config.HasTabletsConfig() || Config.GetTabletsConfig().TabletSize() == 0) { - return; - } - - for (const auto& tabletConfig: Config.GetTabletsConfig().GetTablet()) { - for (ui32 bootstrapperNode: tabletConfig.GetNode()) { - if (bootstrapperNode == setup->NodeId) { + } + } + } + } + } +} + +// alternative bootstrapper + +TTabletsInitializer::TTabletsInitializer( + const TKikimrRunConfig& runConfig, + TIntrusivePtr<ITabletFactory> customTablets) + : IKikimrServicesInitializer(runConfig) + , CustomTablets(customTablets) +{ +} + +void TTabletsInitializer::InitializeServices( + TActorSystemSetup* setup, + const TAppData* appData) +{ + if (!Config.HasTabletsConfig() || Config.GetTabletsConfig().TabletSize() == 0) { + return; + } + + for (const auto& tabletConfig: Config.GetTabletsConfig().GetTablet()) { + for (ui32 bootstrapperNode: tabletConfig.GetNode()) { + if (bootstrapperNode == setup->NodeId) { auto tabletInfo = TabletStorageInfoFromProto(tabletConfig.GetInfo()); - - auto tabletType = tabletConfig.GetType(); - auto tabletSetup = CreateTablet(tabletType, tabletInfo, appData, CustomTablets); - if (!tabletSetup) { - ythrow yexception() - << "unknown tablet type: " << tabletConfig.GetType(); - } - - setup->LocalServices.push_back(std::make_pair( - MakeBootstrapperID(tabletInfo->TabletID, bootstrapperNode), - TActorSetupCmd( - CreateBootstrapper( - tabletInfo.Get(), - new TBootstrapperInfo(tabletSetup.Get()), - tabletConfig.GetStandBy()), + + auto tabletType = tabletConfig.GetType(); + auto tabletSetup = CreateTablet(tabletType, tabletInfo, appData, CustomTablets); + if (!tabletSetup) { + ythrow yexception() + << "unknown tablet type: " << tabletConfig.GetType(); + } + + setup->LocalServices.push_back(std::make_pair( + MakeBootstrapperID(tabletInfo->TabletID, bootstrapperNode), + TActorSetupCmd( + CreateBootstrapper( + tabletInfo.Get(), + new TBootstrapperInfo(tabletSetup.Get()), + tabletConfig.GetStandBy()), TMailboxType::ReadAsFilled, - appData->SystemPoolId))); - } - } - } -} - + appData->SystemPoolId))); + } + } + } +} + // TMediatorTimeCastProxyInitializer - + TMediatorTimeCastProxyInitializer::TMediatorTimeCastProxyInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - + : IKikimrServicesInitializer(runConfig) { +} + void TMediatorTimeCastProxyInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( MakeMediatorTimecastProxyID(), TActorSetupCmd(CreateMediatorTimecastProxy(), TMailboxType::ReadAsFilled, appData->SystemPoolId))); -} - -// TMiniKQLCompileServiceInitializer - +} + +// TMiniKQLCompileServiceInitializer + TMiniKQLCompileServiceInitializer::TMiniKQLCompileServiceInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { -} - -void TMiniKQLCompileServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +} + +void TMiniKQLCompileServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { const auto compileInFlight = Config.GetBootstrapConfig().GetCompileServiceConfig().GetInflightLimit(); IActor* compileService = CreateMiniKQLCompileService(compileInFlight); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeMiniKQLCompileServiceID(), TActorSetupCmd(compileService, TMailboxType::ReadAsFilled, appData->UserPoolId))); -} - +} + static bool IsServiceInitialized(NActors::TActorSystemSetup* setup, TActorId service) { for (auto &pr : setup->LocalServices) @@ -1412,16 +1412,16 @@ static bool IsServiceInitialized(NActors::TActorSystemSetup* setup, TActorId ser return false; } -// TMessageBusServicesInitializer - -TMessageBusServicesInitializer::TMessageBusServicesInitializer(const TKikimrRunConfig& runConfig, +// TMessageBusServicesInitializer + +TMessageBusServicesInitializer::TMessageBusServicesInitializer(const TKikimrRunConfig& runConfig, NMsgBusProxy::IMessageBusServer& busServer) - : IKikimrServicesInitializer(runConfig) + : IKikimrServicesInitializer(runConfig) , BusServer(busServer) { -} - -void TMessageBusServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { +} + +void TMessageBusServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { if (!IsServiceInitialized(setup, NMsgBusProxy::CreateMsgBusProxyId()) && Config.HasMessageBusConfig() && Config.GetMessageBusConfig().GetStartBusProxy()) { if (IActor *proxy = BusServer.CreateProxy()) { @@ -1504,7 +1504,7 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), TActorSetupCmd(cache, TMailboxType::ReadAsFilled, appData->UserPoolId)); } - } + } if (!IsServiceInitialized(setup, NGRpcService::CreateGRpcRequestProxyId())) { auto grpcReqProxy = NGRpcService::CreateGRpcRequestProxy(Config); @@ -1620,23 +1620,23 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se TActorSetupCmd(CreateGrpcPublisherServiceActor(std::move(endpoints)), TMailboxType::ReadAsFilled, appData->UserPoolId) ); } -} - -#ifdef ACTORSLIB_COLLECT_EXEC_STATS -// TStatsCollectorInitializer - -TStatsCollectorInitializer::TStatsCollectorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TStatsCollectorInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - - if (appData->Mon) { - IActor* statsCollector = CreateStatsCollector( - 1, // seconds - *setup, +} + +#ifdef ACTORSLIB_COLLECT_EXEC_STATS +// TStatsCollectorInitializer + +TStatsCollectorInitializer::TStatsCollectorInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TStatsCollectorInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + + if (appData->Mon) { + IActor* statsCollector = CreateStatsCollector( + 1, // seconds + *setup, appData->Counters); setup->LocalServices.emplace_back( TActorId(), @@ -1644,7 +1644,7 @@ void TStatsCollectorInitializer::InitializeServices( statsCollector, TMailboxType::HTSwap, appData->UserPoolId)); - + IActor* memStatsCollector = CreateMemStatsCollector( 1, // seconds appData->Counters); @@ -1655,8 +1655,8 @@ void TStatsCollectorInitializer::InitializeServices( TMailboxType::HTSwap, appData->UserPoolId)); - IActor* procStatCollector = CreateProcStatCollector( - 5, // seconds + IActor* procStatCollector = CreateProcStatCollector( + 5, // seconds appData->Counters); setup->LocalServices.emplace_back( TActorId(), @@ -1664,10 +1664,10 @@ void TStatsCollectorInitializer::InitializeServices( procStatCollector, TMailboxType::HTSwap, appData->UserPoolId)); - } -} -#endif - + } +} +#endif + // TSelfPingInitializer TSelfPingInitializer::TSelfPingInitializer(const TKikimrRunConfig& runConfig) @@ -1697,21 +1697,21 @@ void TSelfPingInitializer::InitializeServices( } } -// TWhiteBoardServiceInitializer - -TWhiteBoardServiceInitializer::TWhiteBoardServiceInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TWhiteBoardServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - IActor* tabletStateService = NNodeWhiteboard::CreateNodeWhiteboardService(); +// TWhiteBoardServiceInitializer + +TWhiteBoardServiceInitializer::TWhiteBoardServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TWhiteBoardServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + IActor* tabletStateService = NNodeWhiteboard::CreateNodeWhiteboardService(); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NNodeWhiteboard::MakeNodeWhiteboardServiceId(NodeId), - TActorSetupCmd(tabletStateService, + TActorSetupCmd(tabletStateService, TMailboxType::HTSwap, - appData->SystemPoolId))); -} - + appData->SystemPoolId))); +} + // TNodeIdentifierInitializer TNodeIdentifierInitializer::TNodeIdentifierInitializer(const TKikimrRunConfig& runConfig) @@ -1727,26 +1727,26 @@ void TNodeIdentifierInitializer::InitializeServices(NActors::TActorSystemSetup* appData->IOPoolId))); } -// TTabletMonitorInitializer - -TTabletMonitorInitializer::TTabletMonitorInitializer( - const TKikimrRunConfig& runConfig, - const TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier>& tabletStateClassifier, - const TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer>& tabletListRenderer) - : IKikimrServicesInitializer(runConfig) - , TabletStateClassifier(tabletStateClassifier) - , TabletListRenderer(tabletListRenderer) { -} - -void TTabletMonitorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - IActor* nodeTabletMonitor = NNodeTabletMonitor::CreateNodeTabletMonitor(TabletStateClassifier, TabletListRenderer); +// TTabletMonitorInitializer + +TTabletMonitorInitializer::TTabletMonitorInitializer( + const TKikimrRunConfig& runConfig, + const TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier>& tabletStateClassifier, + const TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer>& tabletListRenderer) + : IKikimrServicesInitializer(runConfig) + , TabletStateClassifier(tabletStateClassifier) + , TabletListRenderer(tabletListRenderer) { +} + +void TTabletMonitorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + IActor* nodeTabletMonitor = NNodeTabletMonitor::CreateNodeTabletMonitor(TabletStateClassifier, TabletListRenderer); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NNodeTabletMonitor::MakeNodeTabletMonitorID(NodeId), - TActorSetupCmd(nodeTabletMonitor, + TActorSetupCmd(nodeTabletMonitor, TMailboxType::HTSwap, - appData->UserPoolId))); -} - + appData->UserPoolId))); +} + // TViewerInitializer TViewerInitializer::TViewerInitializer(const TKikimrRunConfig& runConfig) @@ -1756,10 +1756,10 @@ TViewerInitializer::TViewerInitializer(const TKikimrRunConfig& runConfig) void TViewerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - using namespace NViewer; - IActor* viewer = CreateViewer(KikimrRunConfig); - SetupPQVirtualHandlers(dynamic_cast<IViewer*>(viewer)); - SetupDBVirtualHandlers(dynamic_cast<IViewer*>(viewer)); + using namespace NViewer; + IActor* viewer = CreateViewer(KikimrRunConfig); + SetupPQVirtualHandlers(dynamic_cast<IViewer*>(viewer)); + SetupDBVirtualHandlers(dynamic_cast<IViewer*>(viewer)); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeViewerID(NodeId), TActorSetupCmd(viewer, TMailboxType::HTSwap, @@ -1965,7 +1965,7 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu setup->LocalServices.push_back(std::make_pair( NKqp::MakeKqpProxyID(NodeId), TActorSetupCmd(proxy, TMailboxType::HTSwap, appData->UserPoolId))); - } + } } TMemoryLogInitializer::TMemoryLogInitializer( @@ -2297,5 +2297,5 @@ void TYandexQueryInitializer::InitializeServices(TActorSystemSetup* setup, const ); } -} // namespace NKikimrServicesInitializers -} // namespace NKikimr +} // namespace NKikimrServicesInitializers +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 85d4d66afc9..407ce1bb7ba 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -1,17 +1,17 @@ -#pragma once -#include "config.h" +#pragma once +#include "config.h" #include "factories.h" -#include "service_initializer.h" - +#include "service_initializer.h" + #include <library/cpp/actors/util/affinity.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/statestorage.h> #include <ydb/core/tablet/tablet_setup.h> #include <ydb/core/tablet/node_tablet_monitor.h> #include <ydb/core/tablet_flat/shared_sausagecache.h> - + #include <ydb/core/protos/config.pb.h> - + #include <ydb/public/lib/base/msgbus.h> #include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h> @@ -23,38 +23,38 @@ #include <library/cpp/actors/core/scheduler_basic.h> #include <library/cpp/actors/interconnect/poller_tcp.h> #include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/vector.h> - -namespace NKikimr { - -namespace NKikimrServicesInitializers { - -class IKikimrServicesInitializer : public IServiceInitializer { -protected: - const NKikimrConfig::TAppConfig& Config; - const ui32 NodeId; + +#include <util/generic/vector.h> + +namespace NKikimr { + +namespace NKikimrServicesInitializers { + +class IKikimrServicesInitializer : public IServiceInitializer { +protected: + const NKikimrConfig::TAppConfig& Config; + const ui32 NodeId; const TKikimrScopeId ScopeId; - -public: - IKikimrServicesInitializer(const TKikimrRunConfig& runConfig); -}; - -// base, nameservice, interconnect -class TBasicServicesInitializer : public IKikimrServicesInitializer { + +public: + IKikimrServicesInitializer(const TKikimrRunConfig& runConfig); +}; + +// base, nameservice, interconnect +class TBasicServicesInitializer : public IKikimrServicesInitializer { static IExecutorPool* CreateExecutorPool(const NKikimrConfig::TActorSystemConfig::TExecutor& poolConfig, const NKikimrConfig::TActorSystemConfig& systemConfig, ui32 poolId, ui32 maxActivityType); - - static ISchedulerThread* CreateScheduler(const NKikimrConfig::TActorSystemConfig::TScheduler &config); - -public: + + static ISchedulerThread* CreateScheduler(const NKikimrConfig::TActorSystemConfig::TScheduler &config); + +public: TBasicServicesInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TImmediateControlBoardInitializer : public IKikimrServicesInitializer { public: TImmediateControlBoardInitializer(const TKikimrRunConfig& runConfig); @@ -62,28 +62,28 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TBSNodeWardenInitializer : public IKikimrServicesInitializer { -public: - TBSNodeWardenInitializer(const TKikimrRunConfig& runConfig); - +class TBSNodeWardenInitializer : public IKikimrServicesInitializer { +public: + TBSNodeWardenInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + // ss: proxies, replicas; board: replicas; scheme_board: replicas -class TStateStorageServiceInitializer : public IKikimrServicesInitializer { -public: - TStateStorageServiceInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - -class TLocalServiceInitializer : public IKikimrServicesInitializer { -public: - TLocalServiceInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +class TStateStorageServiceInitializer : public IKikimrServicesInitializer { +public: + TStateStorageServiceInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; +}; + +class TLocalServiceInitializer : public IKikimrServicesInitializer { +public: + TLocalServiceInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; +}; + class TSharedCacheInitializer : public IKikimrServicesInitializer { public: TSharedCacheInitializer(const TKikimrRunConfig& runConfig); @@ -98,19 +98,19 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TLoggerInitializer : public IKikimrServicesInitializer { - TIntrusivePtr<NActors::NLog::TSettings> LogSettings; +class TLoggerInitializer : public IKikimrServicesInitializer { + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; std::shared_ptr<TLogBackend> LogBackend; TString PathToConfigCacheFile; - -public: - TLoggerInitializer(const TKikimrRunConfig& runConfig, + +public: + TLoggerInitializer(const TKikimrRunConfig& runConfig, TIntrusivePtr<NActors::NLog::TSettings> logSettings, std::shared_ptr<TLogBackend> logBackend); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TSchedulerActorInitializer : public IKikimrServicesInitializer { public: TSchedulerActorInitializer(const TKikimrRunConfig& runConfig); @@ -118,27 +118,27 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TProfilerInitializer : public IKikimrServicesInitializer { -public: - TProfilerInitializer(const TKikimrRunConfig& runConfig); - +class TProfilerInitializer : public IKikimrServicesInitializer { +public: + TProfilerInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TResourceBrokerInitializer : public IKikimrServicesInitializer { -public: +public: TResourceBrokerInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TTabletResolverInitializer : public IKikimrServicesInitializer { -public: - TTabletResolverInitializer(const TKikimrRunConfig& runConfig); - +public: + TTabletResolverInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TTabletPipePeNodeCachesInitializer : public IKikimrServicesInitializer { public: TTabletPipePeNodeCachesInitializer(const TKikimrRunConfig& runConfig); @@ -147,19 +147,19 @@ public: }; class TTabletMonitoringProxyInitializer : public IKikimrServicesInitializer { -public: - TTabletMonitoringProxyInitializer(const TKikimrRunConfig& runConfig); - +public: + TTabletMonitoringProxyInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TTabletCountersAggregatorInitializer : public IKikimrServicesInitializer { -public: - TTabletCountersAggregatorInitializer(const TKikimrRunConfig& runConfig); - +public: + TTabletCountersAggregatorInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TGRpcProxyStatusInitializer : public IKikimrServicesInitializer { public: TGRpcProxyStatusInitializer(const TKikimrRunConfig& runConfig); @@ -167,41 +167,41 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TRestartsCountPublisher : public IKikimrServicesInitializer { - static void PublishRestartsCount(const NMonitoring::TDynamicCounters::TCounterPtr& counter, +class TRestartsCountPublisher : public IKikimrServicesInitializer { + static void PublishRestartsCount(const NMonitoring::TDynamicCounters::TCounterPtr& counter, const TString& restartsCountFile); - -public: - TRestartsCountPublisher(const TKikimrRunConfig& runConfig); - + +public: + TRestartsCountPublisher(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TBootstrapperInitializer : public IKikimrServicesInitializer { -public: +public: TBootstrapperInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - -// alternative (RTMR-style) bootstrapper -class TTabletsInitializer : public IKikimrServicesInitializer { - TIntrusivePtr<ITabletFactory> CustomTablets; - -public: - TTabletsInitializer(const TKikimrRunConfig& runConfig, - TIntrusivePtr<ITabletFactory> customTablets); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; +}; + +// alternative (RTMR-style) bootstrapper +class TTabletsInitializer : public IKikimrServicesInitializer { + TIntrusivePtr<ITabletFactory> CustomTablets; + +public: + TTabletsInitializer(const TKikimrRunConfig& runConfig, + TIntrusivePtr<ITabletFactory> customTablets); + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; +}; + class TMediatorTimeCastProxyInitializer : public IKikimrServicesInitializer { -public: +public: TMediatorTimeCastProxyInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TTxProxyInitializer : public IKikimrServicesInitializer { TVector<ui64> CollectAllAllocatorsFromAllDomains(const NKikimr::TAppData* appData); @@ -225,24 +225,24 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TMiniKQLCompileServiceInitializer : public IKikimrServicesInitializer { -public: +class TMiniKQLCompileServiceInitializer : public IKikimrServicesInitializer { +public: TMiniKQLCompileServiceInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + // msgbus: proxy, trace -class TMessageBusServicesInitializer : public IKikimrServicesInitializer { - NMsgBusProxy::IMessageBusServer& BusServer; - -public: - TMessageBusServicesInitializer(const TKikimrRunConfig& runConfig, +class TMessageBusServicesInitializer : public IKikimrServicesInitializer { + NMsgBusProxy::IMessageBusServer& BusServer; + +public: + TMessageBusServicesInitializer(const TKikimrRunConfig& runConfig, NMsgBusProxy::IMessageBusServer& busServer); - + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + // ticket_parser and so on... class TSecurityServicesInitializer : public IKikimrServicesInitializer { public: @@ -263,16 +263,16 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -#ifdef ACTORSLIB_COLLECT_EXEC_STATS -// stats_collector, procstats_collector -class TStatsCollectorInitializer : public IKikimrServicesInitializer { -public: - TStatsCollectorInitializer(const TKikimrRunConfig& runConfig); - +#ifdef ACTORSLIB_COLLECT_EXEC_STATS +// stats_collector, procstats_collector +class TStatsCollectorInitializer : public IKikimrServicesInitializer { +public: + TStatsCollectorInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; -#endif - +}; +#endif + class TSelfPingInitializer : public IKikimrServicesInitializer { public: TSelfPingInitializer(const TKikimrRunConfig& runConfig); @@ -280,13 +280,13 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TWhiteBoardServiceInitializer : public IKikimrServicesInitializer { -public: - TWhiteBoardServiceInitializer(const TKikimrRunConfig& runConfig); - +class TWhiteBoardServiceInitializer : public IKikimrServicesInitializer { +public: + TWhiteBoardServiceInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TNodeIdentifierInitializer : public IKikimrServicesInitializer { public: TNodeIdentifierInitializer(const TKikimrRunConfig& runConfig); @@ -294,18 +294,18 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TTabletMonitorInitializer : public IKikimrServicesInitializer { - TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier> TabletStateClassifier; - TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer> TabletListRenderer; - -public: - TTabletMonitorInitializer(const TKikimrRunConfig& runConfig, - const TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier>& tabletStateClassifier, - const TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer>& tabletListRenderer); - +class TTabletMonitorInitializer : public IKikimrServicesInitializer { + TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier> TabletStateClassifier; + TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer> TabletListRenderer; + +public: + TTabletMonitorInitializer(const TKikimrRunConfig& runConfig, + const TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier>& tabletStateClassifier, + const TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer>& tabletListRenderer); + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - +}; + class TViewerInitializer : public IKikimrServicesInitializer { const TKikimrRunConfig& KikimrRunConfig; @@ -499,5 +499,5 @@ private: static ui32 IcPort; }; -} // namespace NKikimrServicesInitializers -} // namespace NKikimr +} // namespace NKikimrServicesInitializers +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index c927c1ebaa8..f0a9315685b 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -1,19 +1,19 @@ #include "main.h" -#include "driver.h" +#include "driver.h" -// add support for base utils +// add support for base utils #include <ydb/core/driver_lib/base_utils/format_info.h> #include <ydb/core/driver_lib/base_utils/format_util.h> #include <ydb/core/driver_lib/base_utils/node_by_host.h> -// add support for CLI utils +// add support for CLI utils #include <ydb/core/driver_lib/cli_utils/cli.h> - -// add support for running kikimr node + +// add support for running kikimr node #include <ydb/core/driver_lib/run/config.h> #include <ydb/core/driver_lib/run/config_parser.h> #include <ydb/core/driver_lib/run/run.h> - + // allocator info #include <library/cpp/malloc/api/malloc.h> @@ -55,19 +55,19 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> #ifndef _win_ mlockall(MCL_CURRENT); #endif - using namespace NLastGetopt; - using TDriverModeParser = TCliCommands<EDriverMode>; + using namespace NLastGetopt; + using TDriverModeParser = TCliCommands<EDriverMode>; - NKikimrConfig::TAppConfig appConfig; + NKikimrConfig::TAppConfig appConfig; TCommandConfig cmdConf; - TKikimrRunConfig runConfig(appConfig); + TKikimrRunConfig runConfig(appConfig); + + TRunCommandConfigParser configParser(runConfig); - TRunCommandConfigParser configParser(runConfig); - - TOpts opts = TOpts::Default(); + TOpts opts = TOpts::Default(); opts.SetTitle("KiKiMR client/server binary"); - - configParser.SetupGlobalOpts(opts); + + configParser.SetupGlobalOpts(opts); NMsgBusProxy::TMsgBusClientConfig mbusConfig; mbusConfig.ConfigureLastGetopt(opts, "mb-"); NDriverClient::HideOptions(opts); @@ -83,32 +83,32 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> opts.SetFreeArgTitle(0, "<command>", TDriverModeParser::CommandsCsv()); opts.SetCmdLineDescr(NDriverClient::NewClientCommandsDescription(factories)); - opts.AddHelpOption('h'); - opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; - - TOptsParseResult res(&opts, argc, argv); - - size_t freeArgsPos = res.GetFreeArgsPos(); - argc -= freeArgsPos; - argv += freeArgsPos; - - EDriverMode mode = TDriverModeParser::ParseCommand(*argv); - - if (mode == EDM_NO) { - fprintf(stderr, "Unknown command '%s'\n\n", *argv); + opts.AddHelpOption('h'); + opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; + + TOptsParseResult res(&opts, argc, argv); + + size_t freeArgsPos = res.GetFreeArgsPos(); + argc -= freeArgsPos; + argv += freeArgsPos; + + EDriverMode mode = TDriverModeParser::ParseCommand(*argv); + + if (mode == EDM_NO) { + fprintf(stderr, "Unknown command '%s'\n\n", *argv); opts.PrintUsage(TString("")); - exit(1); - } - - configParser.ParseGlobalOpts(res); - - switch (mode) { + exit(1); + } + + configParser.ParseGlobalOpts(res); + + switch (mode) { case EDM_RUN: - { - configParser.ParseRunOpts(argc, argv); - configParser.ApplyParsedOptions(); + { + configParser.ParseRunOpts(argc, argv); + configParser.ApplyParsedOptions(); return MainRun(runConfig, factories); - } + } case EDM_ADMIN: case EDM_DB: case EDM_TABLET: diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index b9fcfc3a612..a4f74aa4e0f 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1,7 +1,7 @@ #include "run.h" #include "dummy.h" -#include "service_initializer.h" -#include "kikimr_services_initializers.h" +#include "service_initializer.h" +#include "kikimr_services_initializers.h" #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/hfunc.h> @@ -12,18 +12,18 @@ #include <library/cpp/actors/core/process_stats.h> #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/core/log_settings.h> - + #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/executor_pool_io.h> #include <library/cpp/actors/core/scheduler_basic.h> - + #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/poller_tcp.h> #include <library/cpp/actors/interconnect/interconnect_tcp_proxy.h> #include <library/cpp/actors/interconnect/interconnect_tcp_server.h> #include <library/cpp/actors/interconnect/interconnect_mon.h> #include <ydb/core/actorlib_impl/mad_squirrel.h> - + #include <ydb/core/control/immediate_control_board_actor.h> #include <library/cpp/actors/protos/services_common.pb.h> @@ -121,29 +121,29 @@ #include <util/folder/dirut.h> #include <util/system/file.h> #include <util/system/getpid.h> -#include <util/system/hostname.h> +#include <util/system/hostname.h> #include <ydb/core/tracing/tablet_info.h> namespace NKikimr { -class TDomainsInitializer : public IAppDataInitializer { - const NKikimrConfig::TAppConfig& Config; - -public: - TDomainsInitializer(const TKikimrRunConfig& runConfig) - : Config(runConfig.AppConfig) - { - } - - virtual void Initialize(NKikimr::TAppData* appData) override - { - // setup domain info - appData->DomainsInfo = new TDomainsInfo(); - for (const NKikimrConfig::TDomainsConfig::TDomain &domain : Config.GetDomainsConfig().GetDomain()) { - const ui32 domainId = domain.GetDomainId(); - const ui64 schemeRoot = domain.HasSchemeRoot() ? domain.GetSchemeRoot() : 0; - const ui64 planResolution = domain.HasPlanResolution() ? domain.GetPlanResolution() : 500; +class TDomainsInitializer : public IAppDataInitializer { + const NKikimrConfig::TAppConfig& Config; + +public: + TDomainsInitializer(const TKikimrRunConfig& runConfig) + : Config(runConfig.AppConfig) + { + } + + virtual void Initialize(NKikimr::TAppData* appData) override + { + // setup domain info + appData->DomainsInfo = new TDomainsInfo(); + for (const NKikimrConfig::TDomainsConfig::TDomain &domain : Config.GetDomainsConfig().GetDomain()) { + const ui32 domainId = domain.GetDomainId(); + const ui64 schemeRoot = domain.HasSchemeRoot() ? domain.GetSchemeRoot() : 0; + const ui64 planResolution = domain.HasPlanResolution() ? domain.GetPlanResolution() : 500; const TString domainName = domain.HasName() ? domain.GetName() : Sprintf("domain-%" PRIu32, domainId); TDomainsInfo::TDomain::TStoragePoolKinds poolTypes; for (auto &type : domain.GetStoragePoolTypes()) { @@ -178,11 +178,11 @@ public: } appData->DomainsInfo->AddDomain(domainPtr.Release()); - } - - for (const NKikimrConfig::TDomainsConfig::THiveConfig &hiveConfig : Config.GetDomainsConfig().GetHiveConfig()) { - appData->DomainsInfo->AddHive(hiveConfig.GetHiveUid(), hiveConfig.GetHive()); - } + } + + for (const NKikimrConfig::TDomainsConfig::THiveConfig &hiveConfig : Config.GetDomainsConfig().GetHiveConfig()) { + appData->DomainsInfo->AddHive(hiveConfig.GetHiveUid(), hiveConfig.GetHive()); + } for (const NKikimrConfig::TDomainsConfig::TNamedCompactionPolicy &policy : Config.GetDomainsConfig().GetNamedCompactionPolicy()) { appData->DomainsInfo->AddCompactionPolicy(policy.GetName(), new NLocalDb::TCompactionPolicy(policy.GetPolicy())); @@ -206,79 +206,79 @@ public: appData->EnableKqpSpilling = Config.GetTableServiceConfig().GetSpillingServiceConfig().GetLocalFileConfig().GetEnable(); appData->CompactionConfig = Config.GetCompactionConfig(); - } + } }; -class TChannelProfilesInitializer : public IAppDataInitializer { - const NKikimrConfig::TAppConfig& Config; +class TChannelProfilesInitializer : public IAppDataInitializer { + const NKikimrConfig::TAppConfig& Config; -public: - TChannelProfilesInitializer(const TKikimrRunConfig& runConfig) - : Config(runConfig.AppConfig) - { - } +public: + TChannelProfilesInitializer(const TKikimrRunConfig& runConfig) + : Config(runConfig.AppConfig) + { + } - virtual void Initialize(NKikimr::TAppData* appData) override - { - if (!Config.HasChannelProfileConfig()) { - return; - } - // setup channel profiles - appData->ChannelProfiles = new TChannelProfiles(); - ui32 idx = 0; + virtual void Initialize(NKikimr::TAppData* appData) override + { + if (!Config.HasChannelProfileConfig()) { + return; + } + // setup channel profiles + appData->ChannelProfiles = new TChannelProfiles(); + ui32 idx = 0; Y_VERIFY(Config.GetChannelProfileConfig().ProfileSize() > 0); - for (const NKikimrConfig::TChannelProfileConfig::TProfile &profile : Config.GetChannelProfileConfig().GetProfile()) { - const ui32 profileId = profile.GetProfileId(); + for (const NKikimrConfig::TChannelProfileConfig::TProfile &profile : Config.GetChannelProfileConfig().GetProfile()) { + const ui32 profileId = profile.GetProfileId(); Y_VERIFY(profileId == idx, "Duplicate, missing or out of order profileId %" PRIu32 " (expected %" PRIu32 ")", - profileId, idx); - ++idx; - const ui32 channels = profile.ChannelSize(); + profileId, idx); + ++idx; + const ui32 channels = profile.ChannelSize(); Y_VERIFY(channels >= 2); - + appData->ChannelProfiles->Profiles.emplace_back(); TChannelProfiles::TProfile &outProfile = appData->ChannelProfiles->Profiles.back(); - ui32 channelIdx = 0; - for (const NKikimrConfig::TChannelProfileConfig::TProfile::TChannel &channel : profile.GetChannel()) { + ui32 channelIdx = 0; + for (const NKikimrConfig::TChannelProfileConfig::TProfile::TChannel &channel : profile.GetChannel()) { Y_VERIFY(channel.HasErasureSpecies()); Y_VERIFY(channel.HasPDiskCategory()); TString name = channel.GetErasureSpecies(); - TBlobStorageGroupType::EErasureSpecies erasure = TBlobStorageGroupType::ErasureSpeciesByName(name); - if (erasure == TBlobStorageGroupType::ErasureSpeciesCount) { - ythrow yexception() << "wrong erasure species \"" << name << "\""; - } - const ui64 pDiskCategory = channel.GetPDiskCategory(); + TBlobStorageGroupType::EErasureSpecies erasure = TBlobStorageGroupType::ErasureSpeciesByName(name); + if (erasure == TBlobStorageGroupType::ErasureSpeciesCount) { + ythrow yexception() << "wrong erasure species \"" << name << "\""; + } + const ui64 pDiskCategory = channel.GetPDiskCategory(); const NKikimrBlobStorage::TVDiskKind::EVDiskKind vDiskCategory = static_cast<NKikimrBlobStorage::TVDiskKind::EVDiskKind>(channel.GetVDiskCategory()); const TString kind = channel.GetStoragePoolKind(); outProfile.Channels.push_back(TChannelProfiles::TProfile::TChannel(erasure, pDiskCategory, vDiskCategory, kind)); - ++channelIdx; - } + ++channelIdx; + } } } -}; - - -class TProxySchemeCacheInitializer : public IAppDataInitializer { - const NKikimrConfig::TAppConfig& Config; - -public: - TProxySchemeCacheInitializer(const TKikimrRunConfig& runConfig) - : Config(runConfig.AppConfig) - { - } - - virtual void Initialize(NKikimr::TAppData* appData) override - { - if (Config.HasBootstrapConfig()) { - if (Config.GetBootstrapConfig().HasProxySchemeCacheNodes()) - appData->ProxySchemeCacheNodes = Config.GetBootstrapConfig().GetProxySchemeCacheNodes(); - if (Config.GetBootstrapConfig().HasProxySchemeCacheDistNodes()) - appData->ProxySchemeCacheDistrNodes = Config.GetBootstrapConfig().GetProxySchemeCacheDistNodes(); +}; + + +class TProxySchemeCacheInitializer : public IAppDataInitializer { + const NKikimrConfig::TAppConfig& Config; + +public: + TProxySchemeCacheInitializer(const TKikimrRunConfig& runConfig) + : Config(runConfig.AppConfig) + { + } + + virtual void Initialize(NKikimr::TAppData* appData) override + { + if (Config.HasBootstrapConfig()) { + if (Config.GetBootstrapConfig().HasProxySchemeCacheNodes()) + appData->ProxySchemeCacheNodes = Config.GetBootstrapConfig().GetProxySchemeCacheNodes(); + if (Config.GetBootstrapConfig().HasProxySchemeCacheDistNodes()) + appData->ProxySchemeCacheDistrNodes = Config.GetBootstrapConfig().GetProxySchemeCacheDistNodes(); } } -}; +}; class TDynamicNameserviceInitializer : public IAppDataInitializer { @@ -320,28 +320,28 @@ public: TKikimrRunner::TKikimrRunner(std::shared_ptr<TModuleFactories> factories) : ModuleFactories(std::move(factories)) , Counters(MakeIntrusive<NMonitoring::TDynamicCounters>()) - , PollerThreads(new NInterconnect::TPollerThreads) -{ -} + , PollerThreads(new NInterconnect::TPollerThreads) +{ +} -TKikimrRunner::~TKikimrRunner() { - if (!!ActorSystem) { +TKikimrRunner::~TKikimrRunner() { + if (!!ActorSystem) { // Stop ActorSystem first, so no one actor can call any grpc stuff. - ActorSystem->Stop(); + ActorSystem->Stop(); // After that stop sending any requests to actors // by destroing grpc subsystem. for (auto& serv : GRpcServers) { serv.second.Destroy(); } - ActorSystem.Destroy(); - } -} + ActorSystem.Destroy(); + } +} -void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool includeHostName) -{ - const auto& appConfig = runConfig.AppConfig; +void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool includeHostName) +{ + const auto& appConfig = runConfig.AppConfig; NActors::TMon::TConfig monConfig; monConfig.Port = appConfig.HasMonitoringConfig() ? appConfig.GetMonitoringConfig().GetMonitoringPort() : 0; if (monConfig.Port) { @@ -349,14 +349,14 @@ void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool monConfig.Threads = appConfig.GetMonitoringConfig().GetMonitoringThreads(); monConfig.Address = appConfig.GetMonitoringConfig().GetMonitoringAddress(); monConfig.RedirectMainPageTo = appConfig.GetMonitoringConfig().GetRedirectMainPageTo(); - if (includeHostName) { - if (appConfig.HasNameserviceConfig() && appConfig.GetNameserviceConfig().NodeSize() > 0) { - for (const auto& it : appConfig.GetNameserviceConfig().GetNode()) { - if (it.HasNodeId() && it.GetNodeId() == runConfig.NodeId) { - if (it.HasHost()) { + if (includeHostName) { + if (appConfig.HasNameserviceConfig() && appConfig.GetNameserviceConfig().NodeSize() > 0) { + for (const auto& it : appConfig.GetNameserviceConfig().GetNode()) { + if (it.HasNodeId() && it.GetNodeId() == runConfig.NodeId) { + if (it.HasHost()) { monConfig.Host = it.GetHost(); - } - break; + } + break; } } } @@ -380,8 +380,8 @@ void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool Monitoring->Register(NHttp::CreatePing()); ActorsMonPage = Monitoring->RegisterIndexPage("actors", "Actors"); } - } -} + } +} void TKikimrRunner::InitializeMonitoringLogin(const TKikimrRunConfig&) { @@ -402,44 +402,44 @@ void TKikimrRunner::InitializeMessageBus( const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories ) { - if (runConfig.AppConfig.HasMessageBusConfig() && runConfig.AppConfig.GetMessageBusConfig().GetStartBusProxy()) { - const auto& msgbusConfig = runConfig.AppConfig.GetMessageBusConfig(); - - // deserialize queue and session configs - - const auto& queueConfig = msgbusConfig.GetProxyBusQueueConfig(); - ProxyBusQueueConfig.Name = queueConfig.GetName(); - ProxyBusQueueConfig.NumWorkers = queueConfig.GetNumWorkers(); - - const auto& sessionConfig = msgbusConfig.GetProxyBusSessionConfig(); - - ProxyBusSessionConfig.Name = sessionConfig.GetName(); - ProxyBusSessionConfig.NumRetries = sessionConfig.GetNumRetries(); - ProxyBusSessionConfig.RetryInterval = sessionConfig.GetRetryInterval(); - ProxyBusSessionConfig.ReconnectWhenIdle = sessionConfig.GetReconnectWhenIdle(); - ProxyBusSessionConfig.MaxInFlight = sessionConfig.GetMaxInFlight(); - ProxyBusSessionConfig.PerConnectionMaxInFlight = sessionConfig.GetPerConnectionMaxInFlight(); - ProxyBusSessionConfig.PerConnectionMaxInFlightBySize = sessionConfig.GetPerConnectionMaxInFlightBySize(); - ProxyBusSessionConfig.MaxInFlightBySize = sessionConfig.GetMaxInFlightBySize(); - ProxyBusSessionConfig.TotalTimeout = sessionConfig.GetTotalTimeout(); - ProxyBusSessionConfig.SendTimeout = sessionConfig.GetSendTimeout(); - ProxyBusSessionConfig.ConnectTimeout = sessionConfig.GetConnectTimeout(); - ProxyBusSessionConfig.DefaultBufferSize = sessionConfig.GetDefaultBufferSize(); - ProxyBusSessionConfig.MaxBufferSize = sessionConfig.GetMaxBufferSize(); - ProxyBusSessionConfig.SocketRecvBufferSize = sessionConfig.GetSocketRecvBufferSize(); - ProxyBusSessionConfig.SocketSendBufferSize = sessionConfig.GetSocketSendBufferSize(); - ProxyBusSessionConfig.SocketToS = sessionConfig.GetSocketToS(); - ProxyBusSessionConfig.SendThreshold = sessionConfig.GetSendThreshold(); - ProxyBusSessionConfig.Cork = TDuration::MilliSeconds(sessionConfig.GetCork()); - ProxyBusSessionConfig.MaxMessageSize = sessionConfig.GetMaxMessageSize(); - ProxyBusSessionConfig.TcpNoDelay = sessionConfig.GetTcpNoDelay(); - ProxyBusSessionConfig.TcpCork = sessionConfig.GetTcpCork(); - ProxyBusSessionConfig.ExecuteOnMessageInWorkerPool = sessionConfig.GetExecuteOnMessageInWorkerPool(); - ProxyBusSessionConfig.ExecuteOnReplyInWorkerPool = sessionConfig.GetExecuteOnReplyInWorkerPool(); - ProxyBusSessionConfig.ListenPort = sessionConfig.GetListenPort(); - - Bus.Reset(NBus::CreateMessageQueue(ProxyBusQueueConfig, "server proxy")); - if (msgbusConfig.GetStartTracingBusProxy()) { + if (runConfig.AppConfig.HasMessageBusConfig() && runConfig.AppConfig.GetMessageBusConfig().GetStartBusProxy()) { + const auto& msgbusConfig = runConfig.AppConfig.GetMessageBusConfig(); + + // deserialize queue and session configs + + const auto& queueConfig = msgbusConfig.GetProxyBusQueueConfig(); + ProxyBusQueueConfig.Name = queueConfig.GetName(); + ProxyBusQueueConfig.NumWorkers = queueConfig.GetNumWorkers(); + + const auto& sessionConfig = msgbusConfig.GetProxyBusSessionConfig(); + + ProxyBusSessionConfig.Name = sessionConfig.GetName(); + ProxyBusSessionConfig.NumRetries = sessionConfig.GetNumRetries(); + ProxyBusSessionConfig.RetryInterval = sessionConfig.GetRetryInterval(); + ProxyBusSessionConfig.ReconnectWhenIdle = sessionConfig.GetReconnectWhenIdle(); + ProxyBusSessionConfig.MaxInFlight = sessionConfig.GetMaxInFlight(); + ProxyBusSessionConfig.PerConnectionMaxInFlight = sessionConfig.GetPerConnectionMaxInFlight(); + ProxyBusSessionConfig.PerConnectionMaxInFlightBySize = sessionConfig.GetPerConnectionMaxInFlightBySize(); + ProxyBusSessionConfig.MaxInFlightBySize = sessionConfig.GetMaxInFlightBySize(); + ProxyBusSessionConfig.TotalTimeout = sessionConfig.GetTotalTimeout(); + ProxyBusSessionConfig.SendTimeout = sessionConfig.GetSendTimeout(); + ProxyBusSessionConfig.ConnectTimeout = sessionConfig.GetConnectTimeout(); + ProxyBusSessionConfig.DefaultBufferSize = sessionConfig.GetDefaultBufferSize(); + ProxyBusSessionConfig.MaxBufferSize = sessionConfig.GetMaxBufferSize(); + ProxyBusSessionConfig.SocketRecvBufferSize = sessionConfig.GetSocketRecvBufferSize(); + ProxyBusSessionConfig.SocketSendBufferSize = sessionConfig.GetSocketSendBufferSize(); + ProxyBusSessionConfig.SocketToS = sessionConfig.GetSocketToS(); + ProxyBusSessionConfig.SendThreshold = sessionConfig.GetSendThreshold(); + ProxyBusSessionConfig.Cork = TDuration::MilliSeconds(sessionConfig.GetCork()); + ProxyBusSessionConfig.MaxMessageSize = sessionConfig.GetMaxMessageSize(); + ProxyBusSessionConfig.TcpNoDelay = sessionConfig.GetTcpNoDelay(); + ProxyBusSessionConfig.TcpCork = sessionConfig.GetTcpCork(); + ProxyBusSessionConfig.ExecuteOnMessageInWorkerPool = sessionConfig.GetExecuteOnMessageInWorkerPool(); + ProxyBusSessionConfig.ExecuteOnReplyInWorkerPool = sessionConfig.GetExecuteOnReplyInWorkerPool(); + ProxyBusSessionConfig.ListenPort = sessionConfig.GetListenPort(); + + Bus.Reset(NBus::CreateMessageQueue(ProxyBusQueueConfig, "server proxy")); + if (msgbusConfig.GetStartTracingBusProxy()) { BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer( Bus.Get(), ProxyBusSessionConfig, @@ -448,21 +448,21 @@ void TKikimrRunner::InitializeMessageBus( msgbusConfig.GetBusProxyPort()) ); } - else { + else { BusServer.Reset(NMsgBusProxy::CreateMsgBusServer( Bus.Get(), ProxyBusSessionConfig, factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr, msgbusConfig.GetBusProxyPort() )); - } - - if (Monitoring) { - BusMonPage.Reset(new NMonitoring::TBusNgMonPage()); - Monitoring->Register(BusMonPage.Get()); - } + } + + if (Monitoring) { + BusMonPage.Reset(new NMonitoring::TBusNgMonPage()); + Monitoring->Register(BusMonPage.Get()); + } } -} +} static TString ReadFile(const TString& fileName) { TFileInput f(fileName); @@ -483,7 +483,7 @@ void TKikimrRunner::InitializeKqpController(const TKikimrRunConfig& runConfig) { void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { const auto& appConfig = runConfig.AppConfig; - + auto fillFn = [&](const NKikimrConfig::TGRpcConfig& grpcConfig, NGrpc::TGRpcServer& server, NGrpc::TServerOptions& opts) { const auto& services = grpcConfig.GetServices(); @@ -829,13 +829,13 @@ void TKikimrRunner::InitializeAllocator(const TKikimrRunConfig& runConfig) { } } -void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) -{ - const auto& cfg = runConfig.AppConfig; - const ui32 sysPoolId = cfg.GetActorSystemConfig().HasSysExecutor() ? cfg.GetActorSystemConfig().GetSysExecutor() : 0; - const ui32 userPoolId = cfg.GetActorSystemConfig().HasUserExecutor() ? cfg.GetActorSystemConfig().GetUserExecutor() : 0; - const ui32 ioPoolId = cfg.GetActorSystemConfig().HasIoExecutor() ? cfg.GetActorSystemConfig().GetIoExecutor() : 0; - const ui32 batchPoolId = cfg.GetActorSystemConfig().HasBatchExecutor() ? cfg.GetActorSystemConfig().GetBatchExecutor() : 0; +void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) +{ + const auto& cfg = runConfig.AppConfig; + const ui32 sysPoolId = cfg.GetActorSystemConfig().HasSysExecutor() ? cfg.GetActorSystemConfig().GetSysExecutor() : 0; + const ui32 userPoolId = cfg.GetActorSystemConfig().HasUserExecutor() ? cfg.GetActorSystemConfig().GetUserExecutor() : 0; + const ui32 ioPoolId = cfg.GetActorSystemConfig().HasIoExecutor() ? cfg.GetActorSystemConfig().GetIoExecutor() : 0; + const ui32 batchPoolId = cfg.GetActorSystemConfig().HasBatchExecutor() ? cfg.GetActorSystemConfig().GetBatchExecutor() : 0; TMap<TString, ui32> servicePools; for (ui32 i = 0; i < cfg.GetActorSystemConfig().ServiceExecutorSize(); ++i) { auto item = cfg.GetActorSystemConfig().GetServiceExecutor(i); @@ -843,13 +843,13 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) const ui32 pool = item.GetExecutorId(); servicePools.insert(std::pair<TString, ui32>(service, pool)); } - - AppData.Reset(new TAppData(sysPoolId, userPoolId, ioPoolId, batchPoolId, + + AppData.Reset(new TAppData(sysPoolId, userPoolId, ioPoolId, batchPoolId, servicePools, - TypeRegistry.Get(), - FunctionRegistry.Get(), + TypeRegistry.Get(), + FunctionRegistry.Get(), FormatFactory.Get(), - &KikimrShouldContinue)); + &KikimrShouldContinue)); AppData->DataShardExportFactory = ModuleFactories ? ModuleFactories->DataShardExportFactory.get() : nullptr; AppData->SqsEventsWriterFactory = ModuleFactories ? ModuleFactories->SqsEventsWriterFactory.get() : nullptr; AppData->PersQueueMirrorReaderFactory = ModuleFactories ? ModuleFactories->PersQueueMirrorReaderFactory.get() : nullptr; @@ -863,12 +863,12 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) ? ModuleFactories->FolderServiceFactory : nullptr; - AppData->Counters = Counters; - AppData->Mon = Monitoring.Get(); - AppData->BusMonPage = BusMonPage.Get(); + AppData->Counters = Counters; + AppData->Mon = Monitoring.Get(); + AppData->BusMonPage = BusMonPage.Get(); AppData->PollerThreads = PollerThreads; AppData->LocalScopeId = runConfig.ScopeId; - + // setup streaming config if (runConfig.AppConfig.GetGRpcConfig().HasStreamingConfig()) { AppData->StreamingConfig.CopyFrom(runConfig.AppConfig.GetGRpcConfig().GetStreamingConfig()); @@ -922,23 +922,23 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) if (runConfig.AppConfig.GetBootstrapConfig().HasEnableIntrospection()) AppData->EnableIntrospection = runConfig.AppConfig.GetBootstrapConfig().GetEnableIntrospection(); - TAppDataInitializersList appDataInitializers; - // setup domain info - appDataInitializers.AddAppDataInitializer(new TDomainsInitializer(runConfig)); - // setup channel profiles - appDataInitializers.AddAppDataInitializer(new TChannelProfilesInitializer(runConfig)); - // setup proxy scheme cache - appDataInitializers.AddAppDataInitializer(new TProxySchemeCacheInitializer(runConfig)); + TAppDataInitializersList appDataInitializers; + // setup domain info + appDataInitializers.AddAppDataInitializer(new TDomainsInitializer(runConfig)); + // setup channel profiles + appDataInitializers.AddAppDataInitializer(new TChannelProfilesInitializer(runConfig)); + // setup proxy scheme cache + appDataInitializers.AddAppDataInitializer(new TProxySchemeCacheInitializer(runConfig)); // setup dynamic nameservice appDataInitializers.AddAppDataInitializer(new TDynamicNameserviceInitializer(runConfig)); // setup cms appDataInitializers.AddAppDataInitializer(new TCmsInitializer(runConfig)); - - appDataInitializers.Initialize(AppData.Get()); -} - -void TKikimrRunner::InitializeLogSettings(const TKikimrRunConfig& runConfig) -{ + + appDataInitializers.Initialize(AppData.Get()); +} + +void TKikimrRunner::InitializeLogSettings(const TKikimrRunConfig& runConfig) +{ if (ModuleFactories && ModuleFactories->LogBackendFactory) { auto logBackend = ModuleFactories->LogBackendFactory->CreateLogBackend(runConfig, Counters); LogBackend.reset(logBackend.Release()); @@ -947,17 +947,17 @@ void TKikimrRunner::InitializeLogSettings(const TKikimrRunConfig& runConfig) LogBackend.reset(logBackend.Release()); } - if (!runConfig.AppConfig.HasLogConfig()) - return; - - auto logConfig = runConfig.AppConfig.GetLogConfig(); + if (!runConfig.AppConfig.HasLogConfig()) + return; + + auto logConfig = runConfig.AppConfig.GetLogConfig(); LogSettings.Reset(new NActors::NLog::TSettings(NActors::TActorId(runConfig.NodeId, "logger"), - NKikimrServices::LOGGER, - (NActors::NLog::EPriority)logConfig.GetDefaultLevel(), - (NActors::NLog::EPriority)logConfig.GetDefaultSamplingLevel(), + NKikimrServices::LOGGER, + (NActors::NLog::EPriority)logConfig.GetDefaultLevel(), + (NActors::NLog::EPriority)logConfig.GetDefaultSamplingLevel(), logConfig.GetDefaultSamplingRate(), logConfig.GetTimeThresholdMs())); - + LogSettings->Append( NActorsServices::EServiceCommon_MIN, NActorsServices::EServiceCommon_MAX, @@ -968,84 +968,84 @@ void TKikimrRunner::InitializeLogSettings(const TKikimrRunConfig& runConfig) NKikimrServices::EServiceKikimr_MAX, NKikimrServices::EServiceKikimr_Name ); - - LogSettings->ClusterName = logConfig.HasClusterName() ? logConfig.GetClusterName() : ""; - - if (logConfig.GetFormat() == "full") { - LogSettings->Format = NLog::TSettings::PLAIN_FULL_FORMAT; - } else if (logConfig.GetFormat() == "short") { - LogSettings->Format = NLog::TSettings::PLAIN_SHORT_FORMAT; - } else if (logConfig.GetFormat() == "json") { - LogSettings->Format = NLog::TSettings::JSON_FORMAT; - } else { + + LogSettings->ClusterName = logConfig.HasClusterName() ? logConfig.GetClusterName() : ""; + + if (logConfig.GetFormat() == "full") { + LogSettings->Format = NLog::TSettings::PLAIN_FULL_FORMAT; + } else if (logConfig.GetFormat() == "short") { + LogSettings->Format = NLog::TSettings::PLAIN_SHORT_FORMAT; + } else if (logConfig.GetFormat() == "json") { + LogSettings->Format = NLog::TSettings::JSON_FORMAT; + } else { Y_FAIL("Unknown log format: \"%s\"", logConfig.GetFormat().data()); - } - + } + if (logConfig.HasAllowDropEntries()) { LogSettings->SetAllowDrop(logConfig.GetAllowDropEntries()); } - if (logConfig.HasUseLocalTimestamps()) { - LogSettings->SetUseLocalTimestamps(logConfig.GetUseLocalTimestamps()); - } - - if (LogSettings->Format == NLog::TSettings::JSON_FORMAT) { + if (logConfig.HasUseLocalTimestamps()) { + LogSettings->SetUseLocalTimestamps(logConfig.GetUseLocalTimestamps()); + } + + if (LogSettings->Format == NLog::TSettings::JSON_FORMAT) { TString fullHostName = HostName(); - size_t firstDot = fullHostName.find_first_of('.'); - LogSettings->ShortHostName = fullHostName.substr(0, firstDot); - } -} - -void TKikimrRunner::ApplyLogSettings(const TKikimrRunConfig& runConfig) -{ - if (!runConfig.AppConfig.HasLogConfig()) - return; - - auto logConfig = runConfig.AppConfig.GetLogConfig(); - for (const auto& entry : logConfig.GetEntry()) { + size_t firstDot = fullHostName.find_first_of('.'); + LogSettings->ShortHostName = fullHostName.substr(0, firstDot); + } +} + +void TKikimrRunner::ApplyLogSettings(const TKikimrRunConfig& runConfig) +{ + if (!runConfig.AppConfig.HasLogConfig()) + return; + + auto logConfig = runConfig.AppConfig.GetLogConfig(); + for (const auto& entry : logConfig.GetEntry()) { const TString& componentName = entry.GetComponent(); - - NLog::EComponent component; + + NLog::EComponent component; if (componentName.empty()) { - component = NLog::InvalidComponent; - } else { - component = LogSettings->FindComponent(componentName); + component = NLog::InvalidComponent; + } else { + component = LogSettings->FindComponent(componentName); Y_VERIFY(component != NLog::InvalidComponent, "Invalid component name in log configuration file: \"%s\"", componentName.data()); - } - + } + TString explanation; if (entry.HasLevel()) { Y_VERIFY(LogSettings->SetLevel((NLog::EPriority)entry.GetLevel(), component, explanation) == 0); } - if (entry.HasSamplingLevel()) { + if (entry.HasSamplingLevel()) { Y_VERIFY(LogSettings->SetSamplingLevel((NLog::EPriority)entry.GetSamplingLevel(), component, explanation) == 0); - } - if (entry.HasSamplingRate()) { + } + if (entry.HasSamplingRate()) { Y_VERIFY(LogSettings->SetSamplingRate(entry.GetSamplingRate(), component, explanation) == 0); - } - } -} - + } + } +} + void TKikimrRunner::InitializeActorSystem( const TKikimrRunConfig& runConfig, TIntrusivePtr<TServiceInitializersList> serviceInitializers, const TBasicKikimrServicesMask& servicesMask) -{ +{ THolder<TActorSystemSetup> setup(new TActorSystemSetup()); - - serviceInitializers->InitializeServices(setup.Get(), AppData.Get()); - + + serviceInitializers->InitializeServices(setup.Get(), AppData.Get()); + if (Monitoring) { setup->LocalServices.emplace_back(NCrossRef::MakeCrossRefActorId(), TActorSetupCmd(NCrossRef::CreateCrossRefActor(), TMailboxType::HTSwap, AppData->SystemPoolId)); } - ApplyLogSettings(runConfig); - - ActorSystem.Reset(new TActorSystem(setup, AppData.Get(), LogSettings)); - - if (Monitoring) { + ApplyLogSettings(runConfig); + + ActorSystem.Reset(new TActorSystem(setup, AppData.Get(), LogSettings)); + + if (Monitoring) { if (servicesMask.EnableLogger) { Monitoring->RegisterActorPage( ActorsMonPage, @@ -1125,95 +1125,95 @@ void TKikimrRunner::InitializeActorSystem( if (YqSharedResources) { YqSharedResources->Init(ActorSystem.Get()); } -} - -TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializersList( - const TKikimrRunConfig& runConfig, - const TBasicKikimrServicesMask& serviceMask) { - - using namespace NKikimrServicesInitializers; - TIntrusivePtr<TServiceInitializersList> sil(new TServiceInitializersList); +} + +TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializersList( + const TKikimrRunConfig& runConfig, + const TBasicKikimrServicesMask& serviceMask) { + + using namespace NKikimrServicesInitializers; + TIntrusivePtr<TServiceInitializersList> sil(new TServiceInitializersList); if (serviceMask.EnableMemoryLog) { sil->AddServiceInitializer(new TMemoryLogInitializer(runConfig)); } - if (serviceMask.EnableBasicServices) { + if (serviceMask.EnableBasicServices) { sil->AddServiceInitializer(new TBasicServicesInitializer(runConfig)); - } + } if (serviceMask.EnableIcbService) { sil->AddServiceInitializer(new TImmediateControlBoardInitializer(runConfig)); } - if (serviceMask.EnableWhiteBoard) { - sil->AddServiceInitializer(new TWhiteBoardServiceInitializer(runConfig)); - } + if (serviceMask.EnableWhiteBoard) { + sil->AddServiceInitializer(new TWhiteBoardServiceInitializer(runConfig)); + } if (serviceMask.EnableNodeIdentifier) { sil->AddServiceInitializer(new TNodeIdentifierInitializer(runConfig)); } - if (serviceMask.EnableBSNodeWarden) { - sil->AddServiceInitializer(new TBSNodeWardenInitializer(runConfig)); - } + if (serviceMask.EnableBSNodeWarden) { + sil->AddServiceInitializer(new TBSNodeWardenInitializer(runConfig)); + } if (serviceMask.EnableSchemeBoardMonitoring) { sil->AddServiceInitializer(new TSchemeBoardMonitoringInitializer(runConfig)); } - if (serviceMask.EnableStateStorageService) { - sil->AddServiceInitializer(new TStateStorageServiceInitializer(runConfig)); - } - if (serviceMask.EnableLocalService) { - sil->AddServiceInitializer(new TLocalServiceInitializer(runConfig)); - } + if (serviceMask.EnableStateStorageService) { + sil->AddServiceInitializer(new TStateStorageServiceInitializer(runConfig)); + } + if (serviceMask.EnableLocalService) { + sil->AddServiceInitializer(new TLocalServiceInitializer(runConfig)); + } if (serviceMask.EnableSharedCache) { sil->AddServiceInitializer(new TSharedCacheInitializer(runConfig)); } if (serviceMask.EnableBlobCache) { sil->AddServiceInitializer(new TBlobCacheInitializer(runConfig)); } - if (serviceMask.EnableLogger) { + if (serviceMask.EnableLogger) { sil->AddServiceInitializer(new TLoggerInitializer(runConfig, LogSettings, LogBackend)); - } + } if (serviceMask.EnableSchedulerActor) { sil->AddServiceInitializer(new TSchedulerActorInitializer(runConfig)); } - if (serviceMask.EnableProfiler) { - sil->AddServiceInitializer(new TProfilerInitializer(runConfig)); - } + if (serviceMask.EnableProfiler) { + sil->AddServiceInitializer(new TProfilerInitializer(runConfig)); + } if (serviceMask.EnableResourceBroker) { sil->AddServiceInitializer(new TResourceBrokerInitializer(runConfig)); - } - if (serviceMask.EnableTabletResolver) { - sil->AddServiceInitializer(new TTabletResolverInitializer(runConfig)); + } + if (serviceMask.EnableTabletResolver) { + sil->AddServiceInitializer(new TTabletResolverInitializer(runConfig)); sil->AddServiceInitializer(new TTabletPipePeNodeCachesInitializer(runConfig)); - } - if (serviceMask.EnableTabletMonitoringProxy) { - sil->AddServiceInitializer(new TTabletMonitoringProxyInitializer(runConfig)); - } - if (serviceMask.EnableTabletCountersAggregator) { - sil->AddServiceInitializer(new TTabletCountersAggregatorInitializer(runConfig)); - } + } + if (serviceMask.EnableTabletMonitoringProxy) { + sil->AddServiceInitializer(new TTabletMonitoringProxyInitializer(runConfig)); + } + if (serviceMask.EnableTabletCountersAggregator) { + sil->AddServiceInitializer(new TTabletCountersAggregatorInitializer(runConfig)); + } if (serviceMask.EnableGRpcProxyStatus) { sil->AddServiceInitializer(new TGRpcProxyStatusInitializer(runConfig)); } - if (serviceMask.EnableRestartsCountPublisher) { - sil->AddServiceInitializer(new TRestartsCountPublisher(runConfig)); - } - if (serviceMask.EnableBootstrapper) { + if (serviceMask.EnableRestartsCountPublisher) { + sil->AddServiceInitializer(new TRestartsCountPublisher(runConfig)); + } + if (serviceMask.EnableBootstrapper) { sil->AddServiceInitializer(new TBootstrapperInitializer(runConfig)); - } + } if (serviceMask.EnableMediatorTimeCastProxy) { sil->AddServiceInitializer(new TMediatorTimeCastProxyInitializer(runConfig)); - } + } if (serviceMask.EnableTxProxy) { sil->AddServiceInitializer(new TTxProxyInitializer(runConfig)); } - + if (serviceMask.EnableSecurityServices) { sil->AddServiceInitializer(new TSecurityServicesInitializer(runConfig, ModuleFactories)); } if (BusServer && serviceMask.EnableMessageBusServices) { sil->AddServiceInitializer(new TMessageBusServicesInitializer(runConfig, *BusServer)); - } - + } + if (serviceMask.EnableMiniKQLCompileService) { sil->AddServiceInitializer(new TMiniKQLCompileServiceInitializer(runConfig)); } @@ -1226,40 +1226,40 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TGRpcServicesInitializer(runConfig, ModuleFactories)); } -#ifdef ACTORSLIB_COLLECT_EXEC_STATS - if (serviceMask.EnableStatsCollector) { - sil->AddServiceInitializer(new TStatsCollectorInitializer(runConfig)); - } -#endif +#ifdef ACTORSLIB_COLLECT_EXEC_STATS + if (serviceMask.EnableStatsCollector) { + sil->AddServiceInitializer(new TStatsCollectorInitializer(runConfig)); + } +#endif if (serviceMask.EnableSelfPing) { sil->AddServiceInitializer(new TSelfPingInitializer(runConfig)); } - if (serviceMask.EnableTabletMonitor) { - sil->AddServiceInitializer(new NKikimrServicesInitializers::TTabletMonitorInitializer( - runConfig, - new NNodeTabletMonitor::TTabletStateClassifier(), - new NNodeTabletMonitor::TTabletListRenderer())); - } + if (serviceMask.EnableTabletMonitor) { + sil->AddServiceInitializer(new NKikimrServicesInitializers::TTabletMonitorInitializer( + runConfig, + new NNodeTabletMonitor::TTabletStateClassifier(), + new NNodeTabletMonitor::TTabletListRenderer())); + } - if (serviceMask.EnableViewerService) { + if (serviceMask.EnableViewerService) { sil->AddServiceInitializer(new TViewerInitializer(runConfig)); - } - if (serviceMask.EnableLoadService) { - sil->AddServiceInitializer(new TLoadInitializer(runConfig)); - } + } + if (serviceMask.EnableLoadService) { + sil->AddServiceInitializer(new TLoadInitializer(runConfig)); + } if (serviceMask.EnableFailureInjectionService) { sil->AddServiceInitializer(new TFailureInjectionInitializer(runConfig)); } - if (serviceMask.EnablePersQueueL2Cache) { - sil->AddServiceInitializer(new TPersQueueL2CacheInitializer(runConfig)); - } + if (serviceMask.EnablePersQueueL2Cache) { + sil->AddServiceInitializer(new TPersQueueL2CacheInitializer(runConfig)); + } if (serviceMask.EnableNetClassifier) { sil->AddServiceInitializer(new TNetClassifierInitializer(runConfig)); } if (serviceMask.EnablePersQueueClusterTracker) { sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig)); } - + sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig)); sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig)); @@ -1325,9 +1325,9 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TSequenceProxyServiceInitializer(runConfig)); } - return sil; -} - + return sil; +} + void RegisterBaseTagForMemoryProfiling(TActorSystem* as) { Y_VERIFY(as != nullptr); if (as->MemProfActivityBase != 0) @@ -1347,21 +1347,21 @@ void RegisterBaseTagForMemoryProfiling(TActorSystem* as) { Y_VERIFY(as->MemProfActivityBase != 0); } -void TKikimrRunner::KikimrStart() { - - if (!!Monitoring) { - Monitoring->Start(); +void TKikimrRunner::KikimrStart() { + + if (!!Monitoring) { + Monitoring->Start(); + } + + if (!!PollerThreads) { + PollerThreads->Start(); } - if (!!PollerThreads) { - PollerThreads->Start(); - } - ThreadSigmask(SIG_BLOCK); if (ActorSystem) { RegisterBaseTagForMemoryProfiling(ActorSystem.Get()); - ActorSystem->Start(); - } + ActorSystem->Start(); + } for (auto& server : GRpcServers) { if (server.second) { @@ -1383,11 +1383,11 @@ void TKikimrRunner::KikimrStart() { EnableActorCallstack(); ThreadSigmask(SIG_UNBLOCK); -} +} -void TKikimrRunner::KikimrStop(bool graceful) { +void TKikimrRunner::KikimrStop(bool graceful) { Y_UNUSED(graceful); - + if (EnabledGrpcService) { ActorSystem->Send(new IEventHandle(NGRpcService::CreateGrpcPublisherServiceActorId(), {}, new TEvents::TEvPoisonPill)); } @@ -1402,8 +1402,8 @@ void TKikimrRunner::KikimrStop(bool graceful) { KqpShutdownController->Stop(); } - DisableActorCallstack(); - + DisableActorCallstack(); + if (AppData->FeatureFlags.GetEnableDrainOnShutdown() && GracefulShutdownSupported) { for (ui32 i = 0; i < 300; i++) { auto cnt = drainProgress->GetOnlineTabletsEstimate(); @@ -1465,7 +1465,7 @@ void TKikimrRunner::KikimrStop(bool graceful) { } if (ActorSystem) { - ActorSystem->Stop(); + ActorSystem->Stop(); } for (auto& server : GRpcServers) { @@ -1473,9 +1473,9 @@ void TKikimrRunner::KikimrStop(bool graceful) { } if (Bus) { - Bus->Stop(); - Bus.Drop(); - } + Bus->Stop(); + Bus.Drop(); + } if (YqSharedResources) { YqSharedResources->Stop(); @@ -1490,42 +1490,42 @@ void TKikimrRunner::KikimrStop(bool graceful) { ModuleFactories->DataShardExportFactory->Shutdown(); } } -} - -void TKikimrRunner::BusyLoop() { +} + +void TKikimrRunner::BusyLoop() { auto shouldContinueState = KikimrShouldContinue.PollState(); while (shouldContinueState == TProgramShouldContinue::Continue) { - // TODO make this interval configurable + // TODO make this interval configurable Sleep(TDuration::MilliSeconds(10)); shouldContinueState = KikimrShouldContinue.PollState(); } -} +} TProgramShouldContinue TKikimrRunner::KikimrShouldContinue; -void TKikimrRunner::OnTerminate(int) { +void TKikimrRunner::OnTerminate(int) { KikimrShouldContinue.ShouldStop(0); -} - - -void TKikimrRunner::SetSignalHandlers() { -#ifdef _unix_ - signal(SIGPIPE, SIG_IGN); -#endif - signal(SIGINT, &TKikimrRunner::OnTerminate); - signal(SIGTERM, &TKikimrRunner::OnTerminate); +} + + +void TKikimrRunner::SetSignalHandlers() { +#ifdef _unix_ + signal(SIGPIPE, SIG_IGN); +#endif + signal(SIGINT, &TKikimrRunner::OnTerminate); + signal(SIGTERM, &TKikimrRunner::OnTerminate); #if !defined(_win_) SetAsyncSignalHandler(SIGHUP, [](int) { TLogBackend::ReopenAllBackends(); }); #endif -} - -void TKikimrRunner::InitializeRegistries(const TKikimrRunConfig& runConfig) { +} + +void TKikimrRunner::InitializeRegistries(const TKikimrRunConfig& runConfig) { TypeRegistry.Reset(new NScheme::TKikimrTypeRegistry()); - TypeRegistry->CalculateMetadataEtag(); - + TypeRegistry->CalculateMetadataEtag(); + FunctionRegistry.Reset(NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())->Clone()); FormatFactory.Reset(new TFormatFactory); @@ -1552,8 +1552,8 @@ void TKikimrRunner::InitializeRegistries(const TKikimrRunConfig& runConfig) { #endif NKikHouse::RegisterFormat(*FormatFactory); -} - +} + TIntrusivePtr<TKikimrRunner> TKikimrRunner::CreateKikimrRunner( const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories) { @@ -1574,6 +1574,6 @@ TIntrusivePtr<TKikimrRunner> TKikimrRunner::CreateKikimrRunner( runner->InitializeGracefulShutdown(runConfig); runner->InitializeGRpc(runConfig); return runner; -} +} } // NKikimr diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index 2f4f329cc96..dfd93a24e71 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -1,8 +1,8 @@ #pragma once #include "config.h" #include "factories.h" -#include "service_initializer.h" - +#include "service_initializer.h" + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/actors/core/log_settings.h> #include <library/cpp/actors/interconnect/poller_tcp.h> @@ -19,26 +19,26 @@ #include <ydb/core/tablet/tablet_setup.h> #include <ydb/core/ymq/http/http.h> #include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> - + namespace NKikimr { -class TKikimrRunner : public virtual TThrRefBase { -protected: +class TKikimrRunner : public virtual TThrRefBase { +protected: static TProgramShouldContinue KikimrShouldContinue; - static void OnTerminate(int); - + static void OnTerminate(int); + std::shared_ptr<TModuleFactories> ModuleFactories; - TIntrusivePtr<NScheme::TTypeRegistry> TypeRegistry; + TIntrusivePtr<NScheme::TTypeRegistry> TypeRegistry; TIntrusivePtr<NMiniKQL::IMutableFunctionRegistry> FunctionRegistry; TIntrusivePtr<TFormatFactory> FormatFactory; NYq::IYqSharedResources::TPtr YqSharedResources; - - TAutoPtr<TMon> Monitoring; - NMonitoring::TDynamicCounterPtr Counters; + + TAutoPtr<TMon> Monitoring; + NMonitoring::TDynamicCounterPtr Counters; NMonitoring::TIndexMonPage *ActorsMonPage = nullptr; - + bool EnabledGrpcService = false; bool GracefulShutdownSupported = false; THolder<NSQS::TAsyncHttpServer> SqsHttp; @@ -48,35 +48,35 @@ protected: THolder<NKqp::TKqpShutdownController> KqpShutdownController; TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; - TAutoPtr<TAppData> AppData; - - NBus::TBusQueueConfig ProxyBusQueueConfig; - NBus::TBusSessionConfig ProxyBusSessionConfig; - NBus::TBusMessageQueuePtr Bus; - - TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer; - TIntrusivePtr<NMonitoring::TBusNgMonPage> BusMonPage; - + TAutoPtr<TAppData> AppData; + + NBus::TBusQueueConfig ProxyBusQueueConfig; + NBus::TBusSessionConfig ProxyBusSessionConfig; + NBus::TBusMessageQueuePtr Bus; + + TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer; + TIntrusivePtr<NMonitoring::TBusNgMonPage> BusMonPage; + TVector<std::pair<TString, TAutoPtr<NGrpc::TGRpcServer>>> GRpcServers; - TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; std::shared_ptr<TLogBackend> LogBackend; - TAutoPtr<TActorSystem> ActorSystem; - + TAutoPtr<TActorSystem> ActorSystem; + TKikimrRunner(std::shared_ptr<TModuleFactories> factories = {}); - - virtual ~TKikimrRunner(); - + + virtual ~TKikimrRunner(); + virtual void InitializeRegistries(const TKikimrRunConfig& runConfig); - + void InitializeAllocator(const TKikimrRunConfig& runConfig); - void InitializeLogSettings(const TKikimrRunConfig& runConfig); - - void ApplyLogSettings(const TKikimrRunConfig& runConfig); - - void InitializeMonitoring(const TKikimrRunConfig& runConfig, bool includeHostName = true); - + void InitializeLogSettings(const TKikimrRunConfig& runConfig); + + void ApplyLogSettings(const TKikimrRunConfig& runConfig); + + void InitializeMonitoring(const TKikimrRunConfig& runConfig, bool includeHostName = true); + void InitializeControlBoard(const TKikimrRunConfig& runConfig); void InitializeMonitoringLogin(const TKikimrRunConfig& runConfig); @@ -85,36 +85,36 @@ protected: const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories = nullptr ); - + void InitializeGRpc(const TKikimrRunConfig& runConfig); void InitializeKqpController(const TKikimrRunConfig& runConfig); void InitializeGracefulShutdown(const TKikimrRunConfig& runConfig); - void InitializeAppData(const TKikimrRunConfig& runConfig); - + void InitializeAppData(const TKikimrRunConfig& runConfig); + void InitializeActorSystem( const TKikimrRunConfig& runConfig, TIntrusivePtr<TServiceInitializersList> serviceInitializers, const TBasicKikimrServicesMask& serviceMask = {}); - + TIntrusivePtr<TServiceInitializersList> CreateServiceInitializersList( const TKikimrRunConfig& runConfig, const TBasicKikimrServicesMask& serviceMask = {}); - -public: - static void SetSignalHandlers(); - - virtual void KikimrStart(); - virtual void BusyLoop(); - virtual void KikimrStop(bool graceful); - + +public: + static void SetSignalHandlers(); + + virtual void KikimrStart(); + virtual void BusyLoop(); + virtual void KikimrStop(bool graceful); + static TIntrusivePtr<TKikimrRunner> CreateKikimrRunner( const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); -}; - +}; + int MainRun(const TKikimrRunConfig &runConfig, std::shared_ptr<TModuleFactories> factories); } diff --git a/ydb/core/driver_lib/run/service_initializer.cpp b/ydb/core/driver_lib/run/service_initializer.cpp index 9b61ce88be7..c9b813f6dc7 100644 --- a/ydb/core/driver_lib/run/service_initializer.cpp +++ b/ydb/core/driver_lib/run/service_initializer.cpp @@ -1,27 +1,27 @@ -#include "service_initializer.h" - -namespace NKikimr { - -void TServiceInitializersList::AddServiceInitializer(TIntrusivePtr<IServiceInitializer> serviceInitializer) { - ServiceInitializersList.push_back(serviceInitializer); -} - -void TServiceInitializersList::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - for (auto& serviceInitializer : ServiceInitializersList) { - serviceInitializer->InitializeServices(setup, appData); - } -} - -void TAppDataInitializersList::AddAppDataInitializer(TIntrusivePtr<IAppDataInitializer> appDataInitializer) { - AppDataInitializersList.push_back(appDataInitializer); -} - -void TAppDataInitializersList::Initialize(NKikimr::TAppData* appData) { - for (auto& appDataInitializer : AppDataInitializersList) { - appDataInitializer->Initialize(appData); - } -} - -} // namespace NKikimr +#include "service_initializer.h" + +namespace NKikimr { + +void TServiceInitializersList::AddServiceInitializer(TIntrusivePtr<IServiceInitializer> serviceInitializer) { + ServiceInitializersList.push_back(serviceInitializer); +} + +void TServiceInitializersList::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + for (auto& serviceInitializer : ServiceInitializersList) { + serviceInitializer->InitializeServices(setup, appData); + } +} + +void TAppDataInitializersList::AddAppDataInitializer(TIntrusivePtr<IAppDataInitializer> appDataInitializer) { + AppDataInitializersList.push_back(appDataInitializer); +} + +void TAppDataInitializersList::Initialize(NKikimr::TAppData* appData) { + for (auto& appDataInitializer : AppDataInitializersList) { + appDataInitializer->Initialize(appData); + } +} + +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/service_initializer.h b/ydb/core/driver_lib/run/service_initializer.h index 84525eff7d9..870fd3da689 100644 --- a/ydb/core/driver_lib/run/service_initializer.h +++ b/ydb/core/driver_lib/run/service_initializer.h @@ -1,47 +1,47 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <ydb/core/base/appdata.h> - -#include <util/generic/list.h> -#include <util/generic/ptr.h> - -namespace NKikimr { - -struct IAppDataInitializer : public virtual TThrRefBase { - virtual void Initialize(NKikimr::TAppData* appData) = 0; - -}; - - -class TAppDataInitializersList : public IAppDataInitializer { - + +#include <util/generic/list.h> +#include <util/generic/ptr.h> + +namespace NKikimr { + +struct IAppDataInitializer : public virtual TThrRefBase { + virtual void Initialize(NKikimr::TAppData* appData) = 0; + +}; + + +class TAppDataInitializersList : public IAppDataInitializer { + TList<TIntrusivePtr<IAppDataInitializer> > AppDataInitializersList; - -public: - void AddAppDataInitializer(TIntrusivePtr<IAppDataInitializer> appDataInitializer); - - virtual void Initialize(NKikimr::TAppData* appData) override; -}; - - -struct IServiceInitializer: public virtual TThrRefBase { - virtual void InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) = 0; -}; - - -class TServiceInitializersList : public IServiceInitializer { - + +public: + void AddAppDataInitializer(TIntrusivePtr<IAppDataInitializer> appDataInitializer); + + virtual void Initialize(NKikimr::TAppData* appData) override; +}; + + +struct IServiceInitializer: public virtual TThrRefBase { + virtual void InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) = 0; +}; + + +class TServiceInitializersList : public IServiceInitializer { + TList<TIntrusivePtr<IServiceInitializer> > ServiceInitializersList; - -public: - void AddServiceInitializer(TIntrusivePtr<IServiceInitializer> serviceInitializer); - - virtual void InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) override; -}; - -} // namespace NKikimr + +public: + void AddServiceInitializer(TIntrusivePtr<IServiceInitializer> serviceInitializer); + + virtual void InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) override; +}; + +} // namespace NKikimr diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 2c6a411cc3b..f410ef42f20 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -1,10 +1,10 @@ -LIBRARY(run) - +LIBRARY(run) + OWNER( ddoarn g:kikimr ) - + IF (PROFILE_MEMORY_ALLOCATIONS) CFLAGS( -DPROFILE_MEMORY_ALLOCATIONS @@ -17,31 +17,31 @@ IF (KIKIMR_UDF_DYNAMIC_LINK) ) ENDIF() -SRCS( - config.cpp - config.h - config_parser.cpp - config_parser.h +SRCS( + config.cpp + config.h + config_parser.cpp + config_parser.h driver.h - dummy.cpp - dummy.h + dummy.cpp + dummy.h factories.h factories.cpp - kikimr_services_initializers.cpp - kikimr_services_initializers.h + kikimr_services_initializers.cpp + kikimr_services_initializers.h log_backend.cpp log_backend.h main.h main.cpp - run.cpp - run.h - service_initializer.cpp - service_initializer.h + run.cpp + run.h + service_initializer.cpp + service_initializer.h version.cpp version.h -) - -PEERDIR( +) + +PEERDIR( contrib/libs/protobuf library/cpp/actors/core library/cpp/actors/dnsresolver @@ -148,11 +148,11 @@ PEERDIR( ydb/services/rate_limiter ydb/services/ydb ydb/services/yq -) - +) + YQL_LAST_ABI_VERSION() -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/node_whiteboard/node_whiteboard.h b/ydb/core/node_whiteboard/node_whiteboard.h index a5c4fc44405..cdf7601807c 100644 --- a/ydb/core/node_whiteboard/node_whiteboard.h +++ b/ydb/core/node_whiteboard/node_whiteboard.h @@ -92,7 +92,7 @@ struct TEvWhiteboard{ Record.SetState(state); Record.SetGeneration(generation); } - + TEvTabletStateUpdate(TTabletId tabletId, TFollowerId followerId, NKikimrWhiteboard::TTabletStateInfo::ETabletState state, ui32 generation, bool leader) { Record.SetTabletId(tabletId); Record.SetFollowerId(followerId); @@ -102,9 +102,9 @@ struct TEvWhiteboard{ } TEvTabletStateUpdate(TTabletId tabletId, ui32 userState) { - Record.SetTabletId(tabletId); - Record.SetUserState(userState); - } + Record.SetTabletId(tabletId); + Record.SetUserState(userState); + } }; struct TEvTabletStateRequest : public TEventPB<TEvTabletStateRequest, NKikimrWhiteboard::TEvTabletStateRequest, EvTabletStateRequest> {}; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d43b5d499c6..d64169d4fc0 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -275,14 +275,14 @@ message TLogConfig { } repeated TEntry Entry = 1; - optional bool SysLog = 2 [default = false]; - optional uint32 DefaultLevel = 3 [default = 4]; // NActors::NLog::PRI_WARN - optional uint32 DefaultSamplingLevel = 4 [default = 7]; // NActors::NLog::PRI_DEBUG - optional uint32 DefaultSamplingRate = 5 [default = 0]; - optional string Format = 6 [default = "full"]; // "full" | "short" | "json" - optional string ClusterName = 7; + optional bool SysLog = 2 [default = false]; + optional uint32 DefaultLevel = 3 [default = 4]; // NActors::NLog::PRI_WARN + optional uint32 DefaultSamplingLevel = 4 [default = 7]; // NActors::NLog::PRI_DEBUG + optional uint32 DefaultSamplingRate = 5 [default = 0]; + optional string Format = 6 [default = "full"]; // "full" | "short" | "json" + optional string ClusterName = 7; optional bool AllowDropEntries = 8 [default = true]; - optional bool UseLocalTimestamps = 9 [default = false]; + optional bool UseLocalTimestamps = 9 [default = false]; optional string BackendFileName = 10; optional string SysLogService = 11; optional bool SysLogToStdErr = 12; // writes logs to stderr as well as in syslog @@ -294,7 +294,7 @@ message TCompileServiceConfig { optional uint32 InflightLimit = 1 [default = 100000]; } -message TBootstrap { +message TBootstrap { enum ETabletType { TX_DUMMY = 0; @@ -315,10 +315,10 @@ message TBootstrap { DATASHARD = 31; FLAT_SCHEMESHARD = 32; KEYVALUEFLAT = 33; - - JOBRUNNER_POOL_MANAGER = 40; - FLAT_JOBRUNNER_POOL_MANAGER = 41; - + + JOBRUNNER_POOL_MANAGER = 40; + FLAT_JOBRUNNER_POOL_MANAGER = 41; + CMS = 51; NODE_BROKER = 52; @@ -373,7 +373,7 @@ message TInterconnectConfig { repeated TChannel Channel = 1; optional bool FirstTryBeforePoll = 2; // DEPRECATED - optional bool StartTcp = 3 [default = false]; + optional bool StartTcp = 3 [default = false]; optional uint32 SelfKickDelay = 4; // DEPRECATED optional uint32 HandshakeTimeout = 5; optional uint32 HeartbeatInterval = 6; @@ -434,8 +434,8 @@ message TChannelProfileConfig { repeated TProfile Profile = 1; } - -message TMonitoringConfig { + +message TMonitoringConfig { message TDatabaseLabels { optional bool Enabled = 1 [default = true]; // If labels are enabled and services list is empty @@ -470,7 +470,7 @@ message TMonitoringConfig { repeated TAttributeGroup AttributeGroups = 2; }; - optional uint32 MonitoringPort = 1 [default = 0]; + optional uint32 MonitoringPort = 1 [default = 0]; optional uint32 MonitoringThreads = 2 [default = 10]; optional string MonitoringCaption = 3 [default = "YDB Monitoring"]; optional uint32 TabletMonitoringRetries = 4 [default = 1]; @@ -483,75 +483,75 @@ message TMonitoringConfig { optional string ProcessLocation = 11; optional string AllowOrigin = 12; optional string RedirectMainPageTo = 13 [default = "monitoring/"]; -} - -message TRestartsCountConfig { - optional string RestartsCountFile = 1 [default = ""]; -} - -message TMessageBusConfig { - - message TBusQueueConfig { - optional string Name = 1; - optional uint32 NumWorkers = 2 [default = 1]; - } - - message TBusSessionConfig { - optional string Name = 1 [default = ""]; - optional uint32 NumRetries = 2 [default = 0]; - optional uint32 RetryInterval = 3 [default = 1000]; - optional bool ReconnectWhenIdle = 4 [default = false]; - optional uint64 MaxInFlight = 5 [default = 1000]; - optional uint32 PerConnectionMaxInFlight = 6 [default = 0]; - optional uint32 PerConnectionMaxInFlightBySize = 7 [default = 0]; - optional int64 MaxInFlightBySize = 8 [default = -1]; - optional int64 TotalTimeout = 9 [default = 0]; - optional int64 SendTimeout = 10 [default = 0]; - optional int64 ConnectTimeout = 11 [default = 0]; - optional uint64 DefaultBufferSize = 12 [default = 10240]; - optional uint64 MaxBufferSize = 13 [default = 1048576]; - optional uint32 SocketRecvBufferSize = 14 [default = 0]; - optional uint32 SocketSendBufferSize = 15 [default = 0]; - optional int32 SocketToS = 16 [default = -1]; - optional uint64 SendThreshold = 17 [default = 10240]; - optional uint64 Cork = 18 [default = 0]; // milliseconds - optional uint32 MaxMessageSize = 19 [default = 0x01A00000]; - optional bool TcpNoDelay = 20 [default = false]; - optional bool TcpCork = 21 [default = false]; - optional bool ExecuteOnMessageInWorkerPool = 22 [default = true]; - optional bool ExecuteOnReplyInWorkerPool = 23 [default = true]; - optional uint32 ListenPort = 24 [default = 0]; - } - - optional bool StartBusProxy = 1 [default = false]; - optional uint32 BusProxyPort = 2 [default = 14505]; - optional TBusQueueConfig ProxyBusQueueConfig = 3; - optional TBusSessionConfig ProxyBusSessionConfig = 4; - repeated uint64 ProxyBindToProxy = 5; - optional bool StartTracingBusProxy = 6 [default = true]; // move to TMessageBusTraceServiceConfig - optional string TracePath = 7; +} + +message TRestartsCountConfig { + optional string RestartsCountFile = 1 [default = ""]; +} + +message TMessageBusConfig { + + message TBusQueueConfig { + optional string Name = 1; + optional uint32 NumWorkers = 2 [default = 1]; + } + + message TBusSessionConfig { + optional string Name = 1 [default = ""]; + optional uint32 NumRetries = 2 [default = 0]; + optional uint32 RetryInterval = 3 [default = 1000]; + optional bool ReconnectWhenIdle = 4 [default = false]; + optional uint64 MaxInFlight = 5 [default = 1000]; + optional uint32 PerConnectionMaxInFlight = 6 [default = 0]; + optional uint32 PerConnectionMaxInFlightBySize = 7 [default = 0]; + optional int64 MaxInFlightBySize = 8 [default = -1]; + optional int64 TotalTimeout = 9 [default = 0]; + optional int64 SendTimeout = 10 [default = 0]; + optional int64 ConnectTimeout = 11 [default = 0]; + optional uint64 DefaultBufferSize = 12 [default = 10240]; + optional uint64 MaxBufferSize = 13 [default = 1048576]; + optional uint32 SocketRecvBufferSize = 14 [default = 0]; + optional uint32 SocketSendBufferSize = 15 [default = 0]; + optional int32 SocketToS = 16 [default = -1]; + optional uint64 SendThreshold = 17 [default = 10240]; + optional uint64 Cork = 18 [default = 0]; // milliseconds + optional uint32 MaxMessageSize = 19 [default = 0x01A00000]; + optional bool TcpNoDelay = 20 [default = false]; + optional bool TcpCork = 21 [default = false]; + optional bool ExecuteOnMessageInWorkerPool = 22 [default = true]; + optional bool ExecuteOnReplyInWorkerPool = 23 [default = true]; + optional uint32 ListenPort = 24 [default = 0]; + } + + optional bool StartBusProxy = 1 [default = false]; + optional uint32 BusProxyPort = 2 [default = 14505]; + optional TBusQueueConfig ProxyBusQueueConfig = 3; + optional TBusSessionConfig ProxyBusSessionConfig = 4; + repeated uint64 ProxyBindToProxy = 5; + optional bool StartTracingBusProxy = 6 [default = true]; // move to TMessageBusTraceServiceConfig + optional string TracePath = 7; optional uint64 CompileInflightLimit_Depricated = 8 [default = 100000]; -} - -message TTabletsConfig { - - message TTablet { +} + +message TTabletsConfig { + + message TTablet { optional string Type = 1; - repeated uint32 Node = 2; - optional NKikimrTabletBase.TTabletStorageInfo Info = 3; - optional bool StandBy = 4; - optional uint64 WatchThreshold = 5; - } - - repeated TTablet Tablet = 1; -} - -message TKQPConfig { + repeated uint32 Node = 2; + optional NKikimrTabletBase.TTabletStorageInfo Info = 3; + optional bool StandBy = 4; + optional uint64 WatchThreshold = 5; + } + + repeated TTablet Tablet = 1; +} + +message TKQPConfig { optional bool Enable = 1 [default = true]; repeated NKikimrKqp.TKqpSetting Settings = 10; -} - +} + message TMemoryLogConfig { optional uint64 LogBufferSize = 1; optional uint64 LogGrainSize = 2; @@ -1330,25 +1330,25 @@ message TConfigVersion { repeated TConfigItemVersion Items = 1; } -message TAppConfig { - optional TActorSystemConfig ActorSystemConfig = 1; - optional TLogConfig LogConfig = 2; - optional TStaticNameserviceConfig NameserviceConfig = 3; - optional TInterconnectConfig InterconnectConfig = 4; - optional TDomainsConfig DomainsConfig = 5; - optional TBlobStorageConfig BlobStorageConfig = 6; - optional TChannelProfileConfig ChannelProfileConfig = 7; - optional TBootstrap BootstrapConfig = 8; - optional TMonitoringConfig MonitoringConfig = 10; - optional TRestartsCountConfig RestartsCountConfig = 11; - optional TMessageBusConfig MessageBusConfig = 12; - optional TTabletsConfig TabletsConfig = 13; // alternative bootstrapper configuration +message TAppConfig { + optional TActorSystemConfig ActorSystemConfig = 1; + optional TLogConfig LogConfig = 2; + optional TStaticNameserviceConfig NameserviceConfig = 3; + optional TInterconnectConfig InterconnectConfig = 4; + optional TDomainsConfig DomainsConfig = 5; + optional TBlobStorageConfig BlobStorageConfig = 6; + optional TChannelProfileConfig ChannelProfileConfig = 7; + optional TBootstrap BootstrapConfig = 8; + optional TMonitoringConfig MonitoringConfig = 10; + optional TRestartsCountConfig RestartsCountConfig = 11; + optional TMessageBusConfig MessageBusConfig = 12; + optional TTabletsConfig TabletsConfig = 13; // alternative bootstrapper configuration optional NKikimrBlobStorage.TAllVDiskKinds VDiskConfig = 14; optional NKikimrBlobStorage.TDriveModelList DriveModelConfig = 31; optional NKikimrBlobStorage.TIncrHugeConfig IncrHugeConfig = 18; optional string UDFsDir = 15; repeated string UDFsPaths = 16; - optional TKQPConfig KQPConfig = 17; + optional TKQPConfig KQPConfig = 17; optional TMemoryLogConfig MemoryLogConfig = 19; optional TGRpcConfig GRpcConfig = 20; optional TDynamicNameserviceConfig DynamicNameserviceConfig = 22; @@ -1384,4 +1384,4 @@ message TAppConfig { optional string ClusterYamlConfig = 101; optional TConfigVersion Version = 102; -} +} diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index 34b3fd44a92..b96713ca298 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -21,11 +21,11 @@ enum EFlag { Red = 4; } -message TCustomTabletAttribute { - optional uint32 Key = 1; // implementation-dependent - optional string Value = 2; -} - +message TCustomTabletAttribute { + optional uint32 Key = 1; // implementation-dependent + optional string Value = 2; +} + message TTabletStateInfo { enum ETabletState { Created = 0; @@ -51,12 +51,12 @@ message TTabletStateInfo { optional uint64 CreateTime = 2; optional uint64 ChangeTime = 3; optional ETabletState State = 4; - optional uint32 UserState = 5; // implementation-dependent - optional uint32 Generation = 6; + optional uint32 UserState = 5; // implementation-dependent + optional uint32 Generation = 6; optional NKikimrTabletBase.TTabletTypes.EType Type = 7; - optional string Host = 8; - repeated uint32 ChannelGroupIDs = 9; // BS Group per channel - repeated TCustomTabletAttribute Attributes = 10; + optional string Host = 8; + repeated uint32 ChannelGroupIDs = 9; // BS Group per channel + repeated TCustomTabletAttribute Attributes = 10; optional uint32 NodeId = 11; // filled during merge optional bool Leader = 12; // leader or follower optional uint32 Count = 13; // filled during group count diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 3991ec161f3..c17c8a7dc37 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -107,7 +107,7 @@ enum EServiceKikimr { MINIKQL_ENGINE = 340; KQP_TASKS_RUNNER = 341; - // JOBRUNNER section - deprecated, left for compatibility with existing logger configuration + // JOBRUNNER section - deprecated, left for compatibility with existing logger configuration JOBRUNNER = 350; JOBRUNNER_PROXY = 351; JOBRUNNER_CONTRACTOR = 352; @@ -115,10 +115,10 @@ enum EServiceKikimr { FLAT_JOBRUNNER_POOL_MANAGER = 354; // EXECUTION_SERVICE section - EXECUTION_SERVICE = 360; // deprecated, left for compatibility reasons - EXECUTION_SERVICE_PROXY = 361; // deprecated, left for compatibility reasons - KWCALC_PROXY = 362; // deprecated, left for compatibility reasons - KWCALC_PROXY_WORKER = 363; // deprecated, left for compatibility reasons + EXECUTION_SERVICE = 360; // deprecated, left for compatibility reasons + EXECUTION_SERVICE_PROXY = 361; // deprecated, left for compatibility reasons + KWCALC_PROXY = 362; // deprecated, left for compatibility reasons + KWCALC_PROXY_WORKER = 363; // deprecated, left for compatibility reasons YQL_EXECUTION = 364; YAMR_PROXY = 365; YQL_KIKIMR_PROXY = 366; diff --git a/ydb/core/tablet/node_tablet_monitor.cpp b/ydb/core/tablet/node_tablet_monitor.cpp index c8ec76e3948..8bb34e7ea0b 100644 --- a/ydb/core/tablet/node_tablet_monitor.cpp +++ b/ydb/core/tablet/node_tablet_monitor.cpp @@ -137,14 +137,14 @@ public: } TTabletList(const TActorId &sender, ui32 filterNodeId, - const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, - const TIntrusivePtr<ITabletListRenderer>& renderer) + const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, + const TIntrusivePtr<ITabletListRenderer>& renderer) : Sender(sender) , NodesRequested(0) , NodesReceived(0) , FilterNodeId(filterNodeId) - , StateClassifier(stateClassifier) - , Renderer(renderer) + , StateClassifier(stateClassifier) + , Renderer(renderer) {} void Bootstrap(const TActorContext& ctx) { @@ -208,11 +208,11 @@ public: } } - void BuildTabletList(std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> filter, + void BuildTabletList(std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> filter, TVector<TTabletListElement>& tabletsToRender) { TVector<ui64> tabletIdIndex; - tabletsToRender.clear(); + tabletsToRender.clear(); for (const auto& ni : PerNodeTabletInfo) { if (FilterNodeId != 0 && FilterNodeId != ni.first) continue; @@ -228,42 +228,42 @@ public: Sort(tabletIdIndex); - for (const auto& ni : PerNodeTabletInfo) { - if (FilterNodeId != 0 && FilterNodeId != ni.first) - continue; - auto eq_it = EqualRange(NodesInfo->Nodes.begin(), NodesInfo->Nodes.end(), ni.first); - if (eq_it.first != NodesInfo->Nodes.end() && ni.second) { + for (const auto& ni : PerNodeTabletInfo) { + if (FilterNodeId != 0 && FilterNodeId != ni.first) + continue; + auto eq_it = EqualRange(NodesInfo->Nodes.begin(), NodesInfo->Nodes.end(), ni.first); + if (eq_it.first != NodesInfo->Nodes.end() && ni.second) { const TEvInterconnect::TNodeInfo& nodeInfo = *eq_it.first; - for (const auto& ti : ni.second->Record.GetTabletStateInfo()) { - if (filter(ti)) { - ui64 index = EqualRange(tabletIdIndex.begin(), tabletIdIndex.end(), ti.GetTabletId()).first - tabletIdIndex.begin(); - tabletsToRender.push_back({ &nodeInfo, index, &ti }); + for (const auto& ti : ni.second->Record.GetTabletStateInfo()) { + if (filter(ti)) { + ui64 index = EqualRange(tabletIdIndex.begin(), tabletIdIndex.end(), ti.GetTabletId()).first - tabletIdIndex.begin(); + tabletsToRender.push_back({ &nodeInfo, index, &ti }); } } } - } + } } void RenderResponse(const TActorContext &ctx) { Sort(NodesInfo->Nodes.begin(), NodesInfo->Nodes.end()); TString filterNodeHost; - if (FilterNodeId != 0) { - auto eq_it = EqualRange(NodesInfo->Nodes.begin(), NodesInfo->Nodes.end(), FilterNodeId); - if (eq_it.first != NodesInfo->Nodes.end()) { - filterNodeHost = eq_it.first->Host; - } - } + if (FilterNodeId != 0) { + auto eq_it = EqualRange(NodesInfo->Nodes.begin(), NodesInfo->Nodes.end(), FilterNodeId); + if (eq_it.first != NodesInfo->Nodes.end()) { + filterNodeHost = eq_it.first->Host; + } + } TStringStream str; HTML(str) { - Renderer->RenderPageHeader(str); - for(ui32 cls = 0; cls < StateClassifier->GetMaxTabletStateClass(); cls++) { - auto filter = StateClassifier->GetTabletStateClassFilter(cls); + Renderer->RenderPageHeader(str); + for(ui32 cls = 0; cls < StateClassifier->GetMaxTabletStateClass(); cls++) { + auto filter = StateClassifier->GetTabletStateClassFilter(cls); TVector<TTabletListElement> tablets; - BuildTabletList(filter, tablets); - auto listName = StateClassifier->GetTabletStateClassName(cls); - Renderer->RenderTabletList(str, listName, tablets, {FilterNodeId, filterNodeHost}); - } - Renderer->RenderPageFooter(str); + BuildTabletList(filter, tablets); + auto listName = StateClassifier->GetTabletStateClassName(cls); + Renderer->RenderTabletList(str, listName, tablets, {FilterNodeId, filterNodeHost}); + } + Renderer->RenderPageFooter(str); } ctx.Send(Sender, new NMon::TEvHttpInfoRes(str.Str())); Die(ctx); @@ -290,9 +290,9 @@ protected: size_t NodesRequested; size_t NodesReceived; ui32 FilterNodeId; - TIntrusivePtr<ITabletStateClassifier> StateClassifier; - TIntrusivePtr<ITabletListRenderer> Renderer; - + TIntrusivePtr<ITabletStateClassifier> StateClassifier; + TIntrusivePtr<ITabletListRenderer> Renderer; + }; class TStateStorageTabletList : public TActorBootstrapped<TStateStorageTabletList> { @@ -410,21 +410,21 @@ protected: }; class TNodeTabletMonitor : public TActorBootstrapped<TNodeTabletMonitor> { -private: - TIntrusivePtr<ITabletStateClassifier> StateClassifier; - TIntrusivePtr<ITabletListRenderer> TableRenderer; - +private: + TIntrusivePtr<ITabletStateClassifier> StateClassifier; + TIntrusivePtr<ITabletListRenderer> TableRenderer; + public: - + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TABLET_MONITORING_PROXY; } - TNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, - const TIntrusivePtr<ITabletListRenderer>& renderer) - : StateClassifier(stateClassifier) - , TableRenderer(renderer) - { + TNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, + const TIntrusivePtr<ITabletListRenderer>& renderer) + : StateClassifier(stateClassifier) + , TableRenderer(renderer) + { } void Bootstrap(const TActorContext &ctx) { @@ -465,7 +465,7 @@ private: ui32 filterNodeId = 0; if (cgi.Has("node_id")) filterNodeId = FromStringWithDefault<ui32>(cgi.Get("node_id")); - ctx.ExecutorThread.RegisterActor(new TTabletList(ev->Sender, filterNodeId, StateClassifier, TableRenderer)); + ctx.ExecutorThread.RegisterActor(new TTabletList(ev->Sender, filterNodeId, StateClassifier, TableRenderer)); return; } else if (actionParam == "browse_ss" && cgi.Has("ss_id")) { ctx.ExecutorThread.RegisterActor(new TStateStorageTabletList(ev->Sender, FromStringWithDefault<ui32>(cgi.Get("ss_id")))); @@ -477,10 +477,10 @@ private: }; -IActor* CreateNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, - const TIntrusivePtr<ITabletListRenderer>& renderer) -{ - return new TNodeTabletMonitor(stateClassifier, renderer); +IActor* CreateNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, + const TIntrusivePtr<ITabletListRenderer>& renderer) +{ + return new TNodeTabletMonitor(stateClassifier, renderer); } } // NNodeTabletMonitor diff --git a/ydb/core/tablet/node_tablet_monitor.h b/ydb/core/tablet/node_tablet_monitor.h index b98d8f23966..0fe88912003 100644 --- a/ydb/core/tablet/node_tablet_monitor.h +++ b/ydb/core/tablet/node_tablet_monitor.h @@ -7,8 +7,8 @@ #include <library/cpp/actors/core/interconnect.h> #include <ydb/core/base/events.h> #include <ydb/core/protos/node_whiteboard.pb.h> -#include <util/generic/ptr.h> -#include <util/stream/str.h> +#include <util/generic/ptr.h> +#include <util/stream/str.h> namespace NKikimr { namespace NNodeTabletMonitor { @@ -19,42 +19,42 @@ inline TActorId MakeNodeTabletMonitorID(ui32 node = 0) { } -struct ITabletStateClassifier : public TAtomicRefCount<ITabletStateClassifier>{ - virtual ui32 GetMaxTabletStateClass() const = 0; - virtual std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> GetTabletStateClassFilter(ui32 cls) const = 0; +struct ITabletStateClassifier : public TAtomicRefCount<ITabletStateClassifier>{ + virtual ui32 GetMaxTabletStateClass() const = 0; + virtual std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> GetTabletStateClassFilter(ui32 cls) const = 0; virtual TString GetTabletStateClassName(ui32 cls) = 0; - virtual ~ITabletStateClassifier() {} - -}; - - -struct TTabletListElement { + virtual ~ITabletStateClassifier() {} + +}; + + +struct TTabletListElement { const TEvInterconnect::TNodeInfo* NodeInfo; - ui64 TabletIndex; - const NKikimrWhiteboard::TTabletStateInfo* TabletStateInfo; -}; - - -struct TTabletFilterInfo { - ui32 FilterNodeId; + ui64 TabletIndex; + const NKikimrWhiteboard::TTabletStateInfo* TabletStateInfo; +}; + + +struct TTabletFilterInfo { + ui32 FilterNodeId; TString FilterNodeHost; -}; - - -struct ITabletListRenderer : public TAtomicRefCount<ITabletListRenderer> { - virtual void RenderPageHeader(TStringStream& str) = 0; - virtual void RenderPageFooter(TStringStream& str) = 0; - virtual void RenderTabletList(TStringStream& str, +}; + + +struct ITabletListRenderer : public TAtomicRefCount<ITabletListRenderer> { + virtual void RenderPageHeader(TStringStream& str) = 0; + virtual void RenderPageFooter(TStringStream& str) = 0; + virtual void RenderTabletList(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) = 0; - virtual ~ITabletListRenderer() {} - -}; - - -IActor* CreateNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, - const TIntrusivePtr<ITabletListRenderer>& renderer); - + const TTabletFilterInfo& filterInfo) = 0; + virtual ~ITabletListRenderer() {} + +}; + + +IActor* CreateNodeTabletMonitor(const TIntrusivePtr<ITabletStateClassifier>& stateClassifier, + const TIntrusivePtr<ITabletListRenderer>& renderer); + } // NNodeTabletMonitor } // NKikimr diff --git a/ydb/core/tablet/tablet_list_renderer.cpp b/ydb/core/tablet/tablet_list_renderer.cpp index 78324e5907d..ad36f12a291 100644 --- a/ydb/core/tablet/tablet_list_renderer.cpp +++ b/ydb/core/tablet/tablet_list_renderer.cpp @@ -1,91 +1,91 @@ -#include "tablet_list_renderer.h" - +#include "tablet_list_renderer.h" + #include <ydb/core/mon/mon.h> #include <library/cpp/actors/core/mon.h> #include <ydb/core/tablet/tablet_sys.h> #include <library/cpp/monlib/service/pages/templates.h> -#include <util/generic/algorithm.h> - - -namespace NKikimr { -namespace NNodeTabletMonitor { - -/////////////////////////// TTabletStateClassifier //////////////////////////// - -bool TTabletStateClassifier::IsActiveTablet(const NKikimrWhiteboard::TTabletStateInfo& state) -{ +#include <util/generic/algorithm.h> + + +namespace NKikimr { +namespace NNodeTabletMonitor { + +/////////////////////////// TTabletStateClassifier //////////////////////////// + +bool TTabletStateClassifier::IsActiveTablet(const NKikimrWhiteboard::TTabletStateInfo& state) +{ return state.GetState() != NKikimrWhiteboard::TTabletStateInfo::Dead; -} - -bool TTabletStateClassifier::IsDeadTablet(const NKikimrWhiteboard::TTabletStateInfo& state) -{ - return !IsActiveTablet(state); -} - -ui32 TTabletStateClassifier::GetMaxTabletStateClass() const -{ - return TABLET_STATE_CLASS_MAX; -} - -std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> TTabletStateClassifier::GetTabletStateClassFilter(ui32 cls) const -{ - switch(cls) { - case ACTIVE_TABLETS: - return &IsActiveTablet; - case DEAD_TABLETS: - return &IsDeadTablet; - default: - Y_FAIL("Bad tablet state class"); - return nullptr; - } -} - +} + +bool TTabletStateClassifier::IsDeadTablet(const NKikimrWhiteboard::TTabletStateInfo& state) +{ + return !IsActiveTablet(state); +} + +ui32 TTabletStateClassifier::GetMaxTabletStateClass() const +{ + return TABLET_STATE_CLASS_MAX; +} + +std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> TTabletStateClassifier::GetTabletStateClassFilter(ui32 cls) const +{ + switch(cls) { + case ACTIVE_TABLETS: + return &IsActiveTablet; + case DEAD_TABLETS: + return &IsDeadTablet; + default: + Y_FAIL("Bad tablet state class"); + return nullptr; + } +} + TString TTabletStateClassifier::GetTabletStateClassName(ui32 cls) -{ - switch(cls) { - case ACTIVE_TABLETS: - return "Active tablets"; - case DEAD_TABLETS: - return "Dead tablets"; - default: - Y_FAIL("Bad tablet state class"); - return ""; - } -} - -///////////////////////////// TTabletListRenderer ///////////////////////////// - -void TTabletListRenderer::RenderHeader(TStringStream& str, +{ + switch(cls) { + case ACTIVE_TABLETS: + return "Active tablets"; + case DEAD_TABLETS: + return "Dead tablets"; + default: + Y_FAIL("Bad tablet state class"); + return ""; + } +} + +///////////////////////////// TTabletListRenderer ///////////////////////////// + +void TTabletListRenderer::RenderHeader(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) -{ - Y_UNUSED(tabletsToRender); + const TTabletFilterInfo& filterInfo) +{ + Y_UNUSED(tabletsToRender); IOutputStream &__stream(str); H3() { - str << listName; - if (filterInfo.FilterNodeId != 0) { - str << " of Node " << filterInfo.FilterNodeId; - if (!filterInfo.FilterNodeHost.empty()) - str << " (" << filterInfo.FilterNodeHost << ")"; - } + str << listName; + if (filterInfo.FilterNodeId != 0) { + str << " of Node " << filterInfo.FilterNodeId; + if (!filterInfo.FilterNodeHost.empty()) + str << " (" << filterInfo.FilterNodeHost << ")"; + } } -} - -void TTabletListRenderer::RenderTableHeader(TStringStream& str, +} + +void TTabletListRenderer::RenderTableHeader(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) -{ - Y_UNUSED(listName); - Y_UNUSED(tabletsToRender); + const TTabletFilterInfo& filterInfo) +{ + Y_UNUSED(listName); + Y_UNUSED(tabletsToRender); IOutputStream &__stream(str); TABLEHEAD() { TABLER() { - if (filterInfo.FilterNodeId == 0) { + if (filterInfo.FilterNodeId == 0) { TABLEH() {str << "<span data-toggle='tooltip' title='NodeId'>#</span>";} TABLEH() {str << "NodeName";} - } + } TABLEH() {str << "TabletType";} TABLEH() {str << "TabletID";} TABLEH_CLASS("sorter-text") {str << "CreateTime";} @@ -96,99 +96,99 @@ void TTabletListRenderer::RenderTableHeader(TStringStream& str, TABLEH_CLASS("sorter-false") {str << "Kill";} } } -} - +} + TString TTabletListRenderer::MakeTabletMonURL(const TTabletListElement& elem, - const TTabletFilterInfo& filterInfo) -{ - Y_UNUSED(filterInfo); - TStringStream str; - str << "tablets?TabletID="<< elem.TabletStateInfo->GetTabletId(); - return str.Str(); -} - + const TTabletFilterInfo& filterInfo) +{ + Y_UNUSED(filterInfo); + TStringStream str; + str << "tablets?TabletID="<< elem.TabletStateInfo->GetTabletId(); + return str.Str(); +} + TString TTabletListRenderer::GetUserStateName(const TTabletListElement& elem) -{ - if (elem.TabletStateInfo->HasUserState()) { - return ToString(elem.TabletStateInfo->GetUserState()); - } else { - // user part has not not reported its state - return "unknown"; - } -} - -void TTabletListRenderer::RenderTableBody(TStringStream& str, +{ + if (elem.TabletStateInfo->HasUserState()) { + return ToString(elem.TabletStateInfo->GetUserState()); + } else { + // user part has not not reported its state + return "unknown"; + } +} + +void TTabletListRenderer::RenderTableBody(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) -{ - Y_UNUSED(listName); + const TTabletFilterInfo& filterInfo) +{ + Y_UNUSED(listName); IOutputStream &__stream(str); TABLEBODY() { - for (const auto& elem : tabletsToRender) { - const auto& ti = *elem.TabletStateInfo; - const auto& nodeInfo = *elem.NodeInfo; - auto index = elem.TabletIndex; - TTabletTypes::EType tabletType = (TTabletTypes::EType)ti.GetType(); + for (const auto& elem : tabletsToRender) { + const auto& ti = *elem.TabletStateInfo; + const auto& nodeInfo = *elem.NodeInfo; + auto index = elem.TabletIndex; + TTabletTypes::EType tabletType = (TTabletTypes::EType)ti.GetType(); TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType); TString nodeName = ToString(nodeInfo.NodeId); - + TABLER() { - if (filterInfo.FilterNodeId == 0) { + if (filterInfo.FilterNodeId == 0) { TString nodeName = nodeInfo.Host.empty() ? nodeInfo.Address : nodeInfo.Host; TString tooltip = nodeInfo.Address; TABLED() {str << ToString(nodeInfo.NodeId);} TABLED() {str << "<span data-html='true' data-toggle='tooltip' title='" + tooltip + "'>" << nodeName << "</span>";} - } + } TABLED() {str << tabletTypeStr;} - str << "<td data-text='" << index << "'>" - << "<a href=\"" << MakeTabletMonURL(elem, filterInfo) << "\">" << ti.tabletid() << "</a>" - << "</td>"; + str << "<td data-text='" << index << "'>" + << "<a href=\"" << MakeTabletMonURL(elem, filterInfo) << "\">" << ti.tabletid() << "</a>" + << "</td>"; TABLED() {str << TInstant::MilliSeconds(ti.GetCreateTime()).ToStringUpToSeconds();} TABLED() {str << TInstant::MilliSeconds(ti.GetChangeTime()).ToStringUpToSeconds();} TABLED() {str << GetStateName((ETabletState)ti.GetState());} TABLED() {str << GetUserStateName(elem);} TABLED() {str << ti.generation();} TABLED() { - str << "<a href=\"nodetabmon?action=kill_tablet&tablet_id=" << ti.tabletid() << "&node_id=" << nodeInfo.NodeId; - if (filterInfo.FilterNodeId != 0) - str << "&filter_node_id=" << filterInfo.FilterNodeId; - str << "\">" - << "<span class=\"glyphicon glyphicon-remove\" title=\"Kill Tablet\"/>" - << "</span></a>"; + str << "<a href=\"nodetabmon?action=kill_tablet&tablet_id=" << ti.tabletid() << "&node_id=" << nodeInfo.NodeId; + if (filterInfo.FilterNodeId != 0) + str << "&filter_node_id=" << filterInfo.FilterNodeId; + str << "\">" + << "<span class=\"glyphicon glyphicon-remove\" title=\"Kill Tablet\"/>" + << "</span></a>"; } } - } } -} - + } +} + TString TTabletListRenderer::GetStateName(ETabletState state) { return NKikimrWhiteboard::TTabletStateInfo::ETabletState_Name(state); -} - -void TTabletListRenderer::RenderPageHeader(TStringStream& str) -{ - Y_UNUSED(str); -} - -void TTabletListRenderer::RenderPageFooter(TStringStream& str) -{ - Y_UNUSED(str); -} - -void TTabletListRenderer::RenderTabletList(TStringStream& str, +} + +void TTabletListRenderer::RenderPageHeader(TStringStream& str) +{ + Y_UNUSED(str); +} + +void TTabletListRenderer::RenderPageFooter(TStringStream& str) +{ + Y_UNUSED(str); +} + +void TTabletListRenderer::RenderTabletList(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) -{ + const TTabletFilterInfo& filterInfo) +{ IOutputStream &__stream(str); - RenderHeader(str, listName, tabletsToRender, filterInfo); - + RenderHeader(str, listName, tabletsToRender, filterInfo); + TABLE_SORTABLE_CLASS("table") { - RenderTableHeader(str, listName, tabletsToRender, filterInfo); - RenderTableBody(str, listName, tabletsToRender, filterInfo); + RenderTableHeader(str, listName, tabletsToRender, filterInfo); + RenderTableBody(str, listName, tabletsToRender, filterInfo); } -} - -} // namespace NNodeTabletMonitor -} // namespace NKikimr +} + +} // namespace NNodeTabletMonitor +} // namespace NKikimr diff --git a/ydb/core/tablet/tablet_list_renderer.h b/ydb/core/tablet/tablet_list_renderer.h index abc381840b0..41de78016f5 100644 --- a/ydb/core/tablet/tablet_list_renderer.h +++ b/ydb/core/tablet/tablet_list_renderer.h @@ -1,65 +1,65 @@ -#pragma once - -#include "node_tablet_monitor.h" +#pragma once + +#include "node_tablet_monitor.h" #include <ydb/core/tablet/tablet_sys.h> - -namespace NKikimr { -namespace NNodeTabletMonitor { - -class TTabletStateClassifier : public ITabletStateClassifier { -private: - enum ETabletStateClass : ui32 { - ACTIVE_TABLETS = 0, - DEAD_TABLETS = 1, - TABLET_STATE_CLASS_MAX - }; - - static bool IsActiveTablet(const NKikimrWhiteboard::TTabletStateInfo& state); - static bool IsDeadTablet(const NKikimrWhiteboard::TTabletStateInfo& state); - -public: - virtual ui32 GetMaxTabletStateClass() const override; - virtual std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> GetTabletStateClassFilter(ui32 cls) const override; + +namespace NKikimr { +namespace NNodeTabletMonitor { + +class TTabletStateClassifier : public ITabletStateClassifier { +private: + enum ETabletStateClass : ui32 { + ACTIVE_TABLETS = 0, + DEAD_TABLETS = 1, + TABLET_STATE_CLASS_MAX + }; + + static bool IsActiveTablet(const NKikimrWhiteboard::TTabletStateInfo& state); + static bool IsDeadTablet(const NKikimrWhiteboard::TTabletStateInfo& state); + +public: + virtual ui32 GetMaxTabletStateClass() const override; + virtual std::function<bool(const NKikimrWhiteboard::TTabletStateInfo&)> GetTabletStateClassFilter(ui32 cls) const override; virtual TString GetTabletStateClassName(ui32 cls) override; - -}; - - -class TTabletListRenderer : public ITabletListRenderer { -protected: + +}; + + +class TTabletListRenderer : public ITabletListRenderer { +protected: using ETabletState = NKikimrWhiteboard::TTabletStateInfo::ETabletState; static TString GetStateName(ETabletState state); - + virtual TString MakeTabletMonURL(const TTabletListElement& elem, - const TTabletFilterInfo& filterInfo); - + const TTabletFilterInfo& filterInfo); + virtual TString GetUserStateName(const TTabletListElement& elem); - - virtual void RenderHeader(TStringStream& str, + + virtual void RenderHeader(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo); - - virtual void RenderTableHeader(TStringStream& str, + const TTabletFilterInfo& filterInfo); + + virtual void RenderTableHeader(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo); - - virtual void RenderTableBody(TStringStream& str, + const TTabletFilterInfo& filterInfo); + + virtual void RenderTableBody(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo); - -public: - void RenderPageHeader(TStringStream& str) override; - - void RenderPageFooter(TStringStream& str) override; - - void RenderTabletList(TStringStream& str, + const TTabletFilterInfo& filterInfo); + +public: + void RenderPageHeader(TStringStream& str) override; + + void RenderPageFooter(TStringStream& str) override; + + void RenderTabletList(TStringStream& str, const TString& listName, const TVector<TTabletListElement>& tabletsToRender, - const TTabletFilterInfo& filterInfo) override; -}; - -} // namespace NNodeTabletMonitor -} // namespace NKikimr + const TTabletFilterInfo& filterInfo) override; +}; + +} // namespace NNodeTabletMonitor +} // namespace NKikimr diff --git a/ydb/core/tablet/tablet_monitoring_proxy.cpp b/ydb/core/tablet/tablet_monitoring_proxy.cpp index 57d4730df4e..a61a43b7e5c 100644 --- a/ydb/core/tablet/tablet_monitoring_proxy.cpp +++ b/ydb/core/tablet/tablet_monitoring_proxy.cpp @@ -56,7 +56,7 @@ public: HFunc(TEvTabletPipe::TEvClientConnected, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(NMon::TEvRemoteHttpInfoRes, Handle); - HFunc(NMon::TEvRemoteBinaryInfoRes, Handle); + HFunc(NMon::TEvRemoteBinaryInfoRes, Handle); HFunc(NMon::TEvRemoteJsonInfoRes, Handle); CFunc(TEvents::TSystem::Wakeup, Wakeup); } @@ -85,11 +85,11 @@ public: Detach(ctx); } - void Handle(NMon::TEvRemoteBinaryInfoRes::TPtr& ev, const TActorContext& ctx) { - ctx.Send(Sender, new NMon::TEvHttpInfoRes(ev->Get()->Blob, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); - Detach(ctx); - } - + void Handle(NMon::TEvRemoteBinaryInfoRes::TPtr& ev, const TActorContext& ctx) { + ctx.Send(Sender, new NMon::TEvHttpInfoRes(ev->Get()->Blob, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Detach(ctx); + } + void Handle(NMon::TEvRemoteJsonInfoRes::TPtr &ev, const TActorContext &ctx) { ctx.Send(Sender, new NMon::TEvHttpInfoRes(NMonitoring::HTTPOKJSON + ev->Get()->Json, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Detach(ctx); diff --git a/ydb/core/tablet/tablet_setup.h b/ydb/core/tablet/tablet_setup.h index 5683f489057..57ea7d6edba 100644 --- a/ydb/core/tablet/tablet_setup.h +++ b/ydb/core/tablet/tablet_setup.h @@ -10,17 +10,17 @@ namespace NKikimr { class TTabletStorageInfo; class TTabletSetupInfo : public TThrRefBase { -public: +public: typedef std::function<IActor* (const TActorId &, TTabletStorageInfo*)> TTabletCreationFunc; using TPtr = TIntrusivePtr<TTabletSetupInfo>; - -private: + +private: const TTabletCreationFunc Op; const TMailboxType::EType MailboxType; const TMailboxType::EType TabletMailboxType; const ui32 PoolId; const ui32 TabletPoolId; - + public: TTabletSetupInfo(TTabletCreationFunc op, TMailboxType::EType mailboxType, ui32 poolId, TMailboxType::EType tabletMailboxType, ui32 tabletPoolId) : Op(op) @@ -47,12 +47,12 @@ IActor* CreateTabletFollower(const TActorId &launcher, TTabletStorageInfo *info, ui32 followerID, TResourceProfilesPtr profiles = nullptr, TSharedQuotaPtr txCacheQuota = nullptr); - -struct ITabletFactory: public virtual TThrRefBase { - virtual TIntrusivePtr<TTabletSetupInfo> CreateTablet( + +struct ITabletFactory: public virtual TThrRefBase { + virtual TIntrusivePtr<TTabletSetupInfo> CreateTablet( const TString& typeName, - const TIntrusivePtr<TTabletStorageInfo>& tabletInfo, - const TAppData& appData) = 0; -}; - + const TIntrusivePtr<TTabletStorageInfo>& tabletInfo, + const TAppData& appData) = 0; +}; + } diff --git a/ydb/core/tablet/tablet_sys.h b/ydb/core/tablet/tablet_sys.h index 4cf087f0234..661d75a4e0a 100644 --- a/ydb/core/tablet/tablet_sys.h +++ b/ydb/core/tablet/tablet_sys.h @@ -14,7 +14,7 @@ namespace NKikimr { class TTablet : public TActor<TTablet> { using TTabletStateInfo = NKikimrWhiteboard::TTabletStateInfo; using ETabletState = TTabletStateInfo::ETabletState; - + struct TStateStorageInfo { TActorId ProxyID; diff --git a/ydb/core/tablet/ya.make b/ydb/core/tablet/ya.make index 428940409df..25e611b4085 100644 --- a/ydb/core/tablet/ya.make +++ b/ydb/core/tablet/ya.make @@ -27,8 +27,8 @@ SRCS( tablet_impl.h tablet_tracing_signals.cpp tablet_tracing_signals.h - tablet_list_renderer.cpp - tablet_list_renderer.h + tablet_list_renderer.cpp + tablet_list_renderer.h tablet_metrics.h tablet_metrics.cpp tablet_monitoring_proxy.cpp diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 12a89559249..27a260bcb27 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -764,8 +764,8 @@ struct Schema { auto Key(const KeyValuesType& keyValues) { return KeyOperations<TableType, KeyValuesType>(*Database, keyValues); - } - + } + template <typename... Keys> auto Range(Keys... keyValues) { return KeyPrefixOperations<IteratorType, TableType, typename first_n_of<sizeof...(Keys), KeyValuesType>::type>(*Database, keyValues...); @@ -876,8 +876,8 @@ struct Schema { template <std::size_t...I> static TArray Tuple2Array(const TConvertTuple& tuple, std::index_sequence<I...>) { return {{static_cast<const TRawTypeValue&>(std::get<I>(tuple))...}}; - } - + } + static TArray Tuple2Array(const TConvertTuple& tuple) { return Tuple2Array(tuple, std::make_index_sequence<sizeof...(ValuesTypes)>()); } @@ -1471,8 +1471,8 @@ struct Schema { KeyPrefixOperations(TToughDb& database, KeyValuesTypes... keyValues) : Database(&database) , KeyValues(keyValues...) - {} - + {} + explicit KeyPrefixOperations(const ReverseOperations& rhs) : Database(rhs.Database) , KeyValues(rhs.KeyValues) diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index e87f2dde4cc..a853040b932 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -1074,88 +1074,88 @@ namespace NKikimr { TAutoPtr<IEventHandle> handle; runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); } - + class TFakeHive : public TActor<TFakeHive>, public NTabletFlatExecutor::TTabletExecutedFlat { - public: + public: static std::function<IActor* (const TActorId &, TTabletStorageInfo*)> DefaultGetTabletCreationFunc(ui32 type) { Y_UNUSED(type); - return nullptr; - } - - using TTabletInfo = TFakeHiveTabletInfo; - using TState = TFakeHiveState; - + return nullptr; + } + + using TTabletInfo = TFakeHiveTabletInfo; + using TState = TFakeHiveState; + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::HIVE_ACTOR; } TFakeHive(const TActorId &tablet, TTabletStorageInfo *info, TState::TPtr state, - TGetTabletCreationFunc getTabletCreationFunc) - : TActor<TFakeHive>(&TFakeHive::StateInit) + TGetTabletCreationFunc getTabletCreationFunc) + : TActor<TFakeHive>(&TFakeHive::StateInit) , NTabletFlatExecutor::TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) - , State(state) - , GetTabletCreationFunc(getTabletCreationFunc) - { - } - - void OnActivateExecutor(const TActorContext &ctx) final { - Become(&TFakeHive::StateWork); + , State(state) + , GetTabletCreationFunc(getTabletCreationFunc) + { + } + + void OnActivateExecutor(const TActorContext &ctx) final { + Become(&TFakeHive::StateWork); while (!InitialEventsQueue.empty()) { TAutoPtr<IEventHandle> &ev = InitialEventsQueue.front(); ctx.ExecutorThread.Send(ev.Release()); InitialEventsQueue.pop_front(); } - } - - void OnDetach(const TActorContext &ctx) override { - Die(ctx); - } - - void OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActorContext &ctx) override { + } + + void OnDetach(const TActorContext &ctx) override { + Die(ctx); + } + + void OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActorContext &ctx) override { Y_UNUSED(ev); - Die(ctx); - } - + Die(ctx); + } + void Enqueue(STFUNC_SIG) override { Y_UNUSED(ctx); InitialEventsQueue.push_back(ev); } - void StateInit(STFUNC_SIG) { - StateInitImpl(ev, ctx); - } - - void StateWork(STFUNC_SIG) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTablet::TEvTabletDead, HandleTabletDead); - HFunc(TEvHive::TEvCreateTablet, Handle); + void StateInit(STFUNC_SIG) { + StateInitImpl(ev, ctx); + } + + void StateWork(STFUNC_SIG) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTablet::TEvTabletDead, HandleTabletDead); + HFunc(TEvHive::TEvCreateTablet, Handle); HFunc(TEvHive::TEvAdoptTablet, Handle); HFunc(TEvHive::TEvDeleteTablet, Handle); HFunc(TEvHive::TEvDeleteOwnerTablets, Handle); HFunc(TEvHive::TEvRequestHiveInfo, Handle); HFunc(TEvHive::TEvInitiateTabletExternalBoot, Handle); HFunc(TEvFakeHive::TEvSubscribeToTabletDeletion, Handle); - HFunc(TEvents::TEvPoisonPill, Handle); - } - } - - void BrokenState(STFUNC_SIG) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTablet::TEvTabletDead, HandleTabletDead); - } - } - - void Handle(TEvHive::TEvCreateTablet::TPtr& ev, const TActorContext& ctx) { + HFunc(TEvents::TEvPoisonPill, Handle); + } + } + + void BrokenState(STFUNC_SIG) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTablet::TEvTabletDead, HandleTabletDead); + } + } + + void Handle(TEvHive::TEvCreateTablet::TPtr& ev, const TActorContext& ctx) { Cout << "FAKEHIVE " << TabletID() << " TEvCreateTablet " << ev->Get()->Record.ShortDebugString() << Endl; - NKikimrProto::EReplyStatus status = NKikimrProto::OK; + NKikimrProto::EReplyStatus status = NKikimrProto::OK; const std::pair<ui64, ui64> key(ev->Get()->Record.GetOwner(), ev->Get()->Record.GetOwnerIdx()); const auto type = ev->Get()->Record.GetTabletType(); const auto bootMode = ev->Get()->Record.GetTabletBootMode(); - auto it = State->Tablets.find(key); - const auto& defaultTabletTypes = AppData(ctx)->DefaultTabletTypes; + auto it = State->Tablets.find(key); + const auto& defaultTabletTypes = AppData(ctx)->DefaultTabletTypes; TActorId bootstrapperActorId; - if (it == State->Tablets.end()) { + if (it == State->Tablets.end()) { if (bootMode == NKikimrHive::TABLET_BOOT_MODE_EXTERNAL) { // don't boot anything } else if (auto x = GetTabletCreationFunc(type)) { @@ -1191,19 +1191,19 @@ namespace NKikimr { bootstrapperActorId = Boot(ctx, type, &NReplication::CreateController, DataGroupErasure); } else { status = NKikimrProto::ERROR; - } - - if (status == NKikimrProto::OK) { + } + + if (status == NKikimrProto::OK) { ui64 tabletId = State->AllocateTabletId(); it = State->Tablets.insert(std::make_pair(key, TTabletInfo(type, tabletId, bootstrapperActorId))).first; State->TabletIdToOwner[tabletId] = key; - } - } else { - if (it->second.Type != type) { - status = NKikimrProto::ERROR; - } - } - + } + } else { + if (it->second.Type != type) { + status = NKikimrProto::ERROR; + } + } + if (status == NKikimrProto::OK) { auto& boundChannels = ev->Get()->Record.GetBindedChannels(); it->second.BoundChannels.assign(boundChannels.begin(), boundChannels.end()); @@ -1212,8 +1212,8 @@ namespace NKikimr { ctx.Send(ev->Sender, new TEvHive::TEvCreateTabletReply(status, key.first, key.second, it->second.TabletId, TabletID()), 0, ev->Cookie); - } - + } + void TraceAdoptingCases(const std::pair<ui64, ui64> prevKey, const std::pair<ui64, ui64> newKey, const TTabletTypes::EType type, @@ -1399,21 +1399,21 @@ namespace NKikimr { ctx.Send(waiter, response.Release()); } - void Handle(TEvents::TEvPoisonPill::TPtr &ev, const TActorContext &ctx) { + void Handle(TEvents::TEvPoisonPill::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ev); - Become(&TThis::BrokenState); - ctx.Send(Tablet(), new TEvents::TEvPoisonPill); - } - - private: + Become(&TThis::BrokenState); + ctx.Send(Tablet(), new TEvents::TEvPoisonPill); + } + + private: TActorId Boot(const TActorContext& ctx, TTabletTypes::EType tabletType, std::function<IActor* (const TActorId &, TTabletStorageInfo *)> op, - TBlobStorageGroupType::EErasureSpecies erasure) { - TIntrusivePtr<TBootstrapperInfo> bi(new TBootstrapperInfo(new TTabletSetupInfo(op, TMailboxType::Simple, 0, - TMailboxType::Simple, 0))); + TBlobStorageGroupType::EErasureSpecies erasure) { + TIntrusivePtr<TBootstrapperInfo> bi(new TBootstrapperInfo(new TTabletSetupInfo(op, TMailboxType::Simple, 0, + TMailboxType::Simple, 0))); return ctx.ExecutorThread.RegisterActor(CreateBootstrapper( CreateTestTabletInfo(State->NextTabletId, tabletType, erasure), bi.Get())); - } - + } + void FillTabletInfo(NKikimrHive::TEvResponseHiveInfo& response, ui64 tabletId, const TFakeHiveTabletInfo *info) { auto& tabletInfo = *response.AddTablets(); tabletInfo.SetTabletID(tabletId); @@ -1425,25 +1425,25 @@ namespace NKikimr { } } - private: - TState::TPtr State; - TGetTabletCreationFunc GetTabletCreationFunc; + private: + TState::TPtr State; + TGetTabletCreationFunc GetTabletCreationFunc; TDeque<TAutoPtr<IEventHandle>> InitialEventsQueue; - }; - - void BootFakeHive(TTestActorRuntime& runtime, ui64 tabletId, TFakeHiveState::TPtr state, + }; + + void BootFakeHive(TTestActorRuntime& runtime, ui64 tabletId, TFakeHiveState::TPtr state, TGetTabletCreationFunc getTabletCreationFunc) { CreateTestBootstrapper(runtime, CreateTestTabletInfo(tabletId, TTabletTypes::Hive), [=](const TActorId & tablet, TTabletStorageInfo* info) { - return new TFakeHive(tablet, info, state, - (getTabletCreationFunc == nullptr) ? &TFakeHive::DefaultGetTabletCreationFunc : getTabletCreationFunc); - }); - - { - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot, 1)); - runtime.DispatchEvents(options); - } - } - + return new TFakeHive(tablet, info, state, + (getTabletCreationFunc == nullptr) ? &TFakeHive::DefaultGetTabletCreationFunc : getTabletCreationFunc); + }); + + { + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot, 1)); + runtime.DispatchEvents(options); + } + } + } diff --git a/ydb/core/testlib/tablet_helpers.h b/ydb/core/testlib/tablet_helpers.h index 7f7b717a9db..81a7c614aaa 100644 --- a/ydb/core/testlib/tablet_helpers.h +++ b/ydb/core/testlib/tablet_helpers.h @@ -84,7 +84,7 @@ namespace NKikimr { IActor* CreateFlatDummyTablet(const TActorId &tablet, TTabletStorageInfo *info); void WaitScheduledEvents(TTestActorRuntime &runtime, TDuration delay, const TActorId &sender, ui32 nodeIndex = 0); - + struct TEvFakeHive { enum EEv { @@ -109,40 +109,40 @@ namespace NKikimr { }; }; - struct TFakeHiveTabletInfo { + struct TFakeHiveTabletInfo { const TTabletTypes::EType Type; - const ui64 TabletId; + const ui64 TabletId; TActorId BootstrapperActorId; - + TChannelsBindings BoundChannels; ui32 ChannelsProfile; THashSet<TActorId> DeletionWaiters; TFakeHiveTabletInfo(TTabletTypes::EType type, ui64 tabletId, TActorId bootstrapperActorId) - : Type(type) - , TabletId(tabletId) + : Type(type) + , TabletId(tabletId) , BootstrapperActorId(bootstrapperActorId) - {} + {} TFakeHiveTabletInfo(const TFakeHiveTabletInfo& info) = default; - }; - - struct TFakeHiveState : TThrRefBase { + }; + + struct TFakeHiveState : TThrRefBase { TMap<std::pair<ui64, ui64>, TFakeHiveTabletInfo> Tablets; TMap<ui64, std::pair<ui64, ui64>> TabletIdToOwner; TMap<ui64, ui64> TabletIdToHive; - ui64 NextTabletId; + ui64 NextTabletId; ui64 NextHiveNextTabletId; - + static constexpr ui64 TABLETS_PER_CHILD_HIVE = 1000000; // amount of tablet ids we reserve for child hive - typedef TIntrusivePtr<TFakeHiveState> TPtr; - - TFakeHiveState() + typedef TIntrusivePtr<TFakeHiveState> TPtr; + + TFakeHiveState() : NextTabletId(TTestTxConfig::FakeHiveTablets) , NextHiveNextTabletId(NextTabletId + TABLETS_PER_CHILD_HIVE) - {} + {} TPtr AllocateSubHive() { if (NextHiveNextTabletId == 0) { @@ -160,11 +160,11 @@ namespace NKikimr { ui64 AllocateTabletId() { return NextTabletId++; } - }; - + }; + typedef std::function<std::function<IActor* (const TActorId &, TTabletStorageInfo*)>(ui32 type)> TGetTabletCreationFunc; - - void BootFakeHive(TTestActorRuntime& runtime, ui64 tabletId, TFakeHiveState::TPtr state, - TGetTabletCreationFunc getTabletCreationFunc = nullptr); - + + void BootFakeHive(TTestActorRuntime& runtime, ui64 tabletId, TFakeHiveState::TPtr state, + TGetTabletCreationFunc getTabletCreationFunc = nullptr); + } diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index f9c734d3726..d4907c26f1e 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -100,7 +100,7 @@ namespace NKikimr { namespace Tests { - + TServerSettings& TServerSettings::SetDomainName(const TString& value) { @@ -143,7 +143,7 @@ namespace Tests { { if (Settings->SupportsRedirect && IsServerRedirected()) return; - + if (init) Initialize(); } @@ -207,7 +207,7 @@ namespace Tests { SetupTabletServices(*Runtime, &app, (StaticNodes() + DynamicNodes()) == 1 && Settings->EnableMockOnSingleNode, Settings->CustomDiskParams); CreateBootstrapTablets(); - SetupStorage(); + SetupStorage(); for (ui32 nodeIdx = 0; nodeIdx < StaticNodes() + DynamicNodes(); ++nodeIdx) { SetupDomainLocalService(nodeIdx); @@ -220,7 +220,7 @@ namespace Tests { SetupConfigurators(nodeIdx); SetupProxies(nodeIdx); } - } + } void TServer::SetupMessageBus(ui16 port, const TString &tracePath) { if (port) { @@ -242,7 +242,7 @@ namespace Tests { )); } } - } + } void TServer::EnableGRpc(const NGrpc::TServerOptions& options) { GRpcServer.reset(new NGrpc::TGRpcServer(options)); @@ -356,7 +356,7 @@ namespace Tests { TVector<ui64>{TDomainsInfo::MakeTxAllocatorIDFixed(domainId, 1)}, Settings->StoragePoolTypes); app.AddDomain(domain.Release()); - } + } void TServer::CreateBootstrapTablets() { const ui32 domainId = Settings->Domain; @@ -549,7 +549,7 @@ namespace Tests { TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( &NReplication::CreateController, TMailboxType::Revolving, appData.UserPoolId, TMailboxType::Revolving, appData.SystemPoolId)); - } + } void TServer::SetupLocalService(ui32 nodeIdx, const TString &domainName) { TLocalConfig::TPtr localConfig = new TLocalConfig(); @@ -818,7 +818,7 @@ namespace Tests { ); NYq::InitTest(Runtime.Get(), port, Settings->GrpcPort, yqSharedResources); } - } + } void TServer::SetupLogging() { Runtime->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NLog::PRI_WARN); @@ -859,8 +859,8 @@ namespace Tests { } TTestActorRuntime* TServer::GetRuntime() const { - return Runtime.Get(); - } + return Runtime.Get(); + } const TServerSettings &TServer::GetSettings() const { return *Settings; diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index a4c94b70256..2064752ab45 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -24,7 +24,7 @@ #include <ydb/core/kesus/tablet/events.h> #include <ydb/core/security/ticket_parser.h> #include <ydb/core/base/grpc_service_factory.h> - + #include <google/protobuf/text_format.h> #include <functional> @@ -74,7 +74,7 @@ namespace Tests { ui64 ChangeDomain(ui64 tabletId, ui32 domainUid); ui64 ChangeStateStorage(ui64 tabletId, ui32 ssUid); NMiniKQL::IFunctionRegistry* DefaultFrFactory(const NScheme::TTypeRegistry& typeRegistry); - + struct TServerSettings: public TThrRefBase, public TTestFeatureFlagsHolder<TServerSettings> { static constexpr ui64 BOX_ID = 999; @@ -206,10 +206,10 @@ namespace Tests { void Initialize(); - public: + public: using TPtr = TIntrusivePtr<TServer>; using TMapStoragePool = TDomainsInfo::TDomain::TStoragePoolKinds; - + TServer(const TServerSettings& settings, bool defaultInit = true); TServer(TServerSettings::TConstPtr settings, bool defaultInit = true); @@ -237,7 +237,7 @@ namespace Tests { const TServerSettings& GetSettings() const; const NScheme::TTypeRegistry* GetTypeRegistry(); const NMiniKQL::IFunctionRegistry* GetFunctionRegistry(); - + ui32 StaticNodes() const { return Settings->NodeCount; } @@ -250,7 +250,7 @@ namespace Tests { protected: const TServerSettings::TConstPtr Settings; const bool UseStoragePools; - + THolder<TTestActorRuntime> Runtime; THolder<NYdb::TDriver> Driver; TIntrusivePtr<NBus::TBusMessageQueue> Bus; @@ -258,8 +258,8 @@ namespace Tests { TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer; std::unique_ptr<NGrpc::TGRpcServer> GRpcServer; TIntrusivePtr<NMonitoring::TDynamicCounters> GRpcServerRootCounters; - }; - + }; + class TClient { public: struct TFlatQueryOptions { diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h index 8372e51d361..c5716f46a76 100644 --- a/ydb/core/viewer/browse.h +++ b/ydb/core/viewer/browse.h @@ -9,7 +9,7 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/viewer/protos/viewer.pb.h> #include <ydb/core/viewer/json/json.h> -#include "browse_events.h" +#include "browse_events.h" #include "viewer.h" #include "wb_aggregate.h" @@ -98,25 +98,25 @@ public: return NKikimrViewer::EObjectType::Unknown; } - TString GetNextName() const { - TString nextName; + TString GetNextName() const { + TString nextName; if (CurrentPath.size() < Path.size()) { auto pos = CurrentPath.size(); if (Path[pos] == '/') { ++pos; } - if (CurrentType == NKikimrViewer::RtmrTables) { - nextName = Path.substr(pos); - } else { - auto end_pos = pos; - while (end_pos < Path.size() && Path[end_pos] != '/') { - ++end_pos; - } - - nextName = Path.substr(pos, end_pos - pos); + if (CurrentType == NKikimrViewer::RtmrTables) { + nextName = Path.substr(pos); + } else { + auto end_pos = pos; + while (end_pos < Path.size() && Path[end_pos] != '/') { + ++end_pos; + } + + nextName = Path.substr(pos, end_pos - pos); } } - return nextName; + return nextName; } void FillCommonData(NKikimrViewer::TBrowseInfo& browseInfo, NKikimrViewer::TMetaInfo& metaInfo) { @@ -197,34 +197,34 @@ public: } } - void StepOnPath(const TActorContext& ctx, const TString& currentName) { + void StepOnPath(const TActorContext& ctx, const TString& currentName) { switch (CurrentType) { case NKikimrViewer::EObjectType::Unknown: case NKikimrViewer::EObjectType::Root: case NKikimrViewer::EObjectType::Directory: case NKikimrViewer::EObjectType::Table: { - const auto* domainsInfo = AppData(ctx)->DomainsInfo.Get(); - if (!domainsInfo || !TxProxy) { - break; - } - bool hasTxAllocators = false; - for (const auto& pair: domainsInfo->Domains) { - if (!pair.second->TxAllocators.empty()) { - hasTxAllocators = true; - break; - } - } - if (!hasTxAllocators) { - break; - } + const auto* domainsInfo = AppData(ctx)->DomainsInfo.Get(); + if (!domainsInfo || !TxProxy) { + break; + } + bool hasTxAllocators = false; + for (const auto& pair: domainsInfo->Domains) { + if (!pair.second->TxAllocators.empty()) { + hasTxAllocators = true; + break; + } + } + if (!hasTxAllocators) { + break; + } THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate()); if (!BrowseContext.UserToken.empty()) { request->Record.SetUserToken(BrowseContext.UserToken); } NKikimrSchemeOp::TDescribePath* record = request->Record.MutableDescribePath(); record->SetPath(CurrentPath); - ctx.Send(TxProxy, request.Release(), IEventHandle::FlagTrackDelivery); + ctx.Send(TxProxy, request.Release(), IEventHandle::FlagTrackDelivery); ++Requests; ctx.Send(BrowseContext.Owner, new NViewerEvents::TEvBrowseRequestSent(TxProxy, TEvTxUserProxy::EvNavigate)); } @@ -251,7 +251,7 @@ public: } void NextStep(const TActorContext& ctx) { - TString currentName = GetNextName(); + TString currentName = GetNextName(); if (currentName.empty()) { return ReplyAndDie(ctx); } @@ -266,7 +266,7 @@ public: BrowseInfo.Clear(); MetaInfo.Clear(); DescribeResult.Reset(); - return StepOnPath(ctx, currentName); + return StepOnPath(ctx, currentName); } } return HandleBadRequest(ctx, "The path is not found"); @@ -297,7 +297,7 @@ public: } } - void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { + void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { BrowseInfo.MergeFrom(ev->Get()->BrowseInfo); MetaInfo.MergeFrom(ev->Get()->MetaInfo); ++Responses; @@ -322,7 +322,7 @@ public: } CurrentPath = Path.substr(0, 1); CurrentType = NKikimrViewer::EObjectType::Root; - StepOnPath(ctx, "/"); + StepOnPath(ctx, "/"); Become(&TThis::StateWork); } @@ -336,15 +336,15 @@ public: STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); - HFunc(NViewerEvents::TEvBrowseResponse, Handle); + HFunc(NViewerEvents::TEvBrowseResponse, Handle); CFunc(TEvents::TSystem::PoisonPill, HandlePoisonPill); - HFunc(TEvents::TEvUndelivered, HandleUndelivered); + HFunc(TEvents::TEvUndelivered, HandleUndelivered); } } void ReplyAndDie(const TActorContext &ctx) { FillCommonData(BrowseInfo, MetaInfo); - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(BrowseInfo), std::move(MetaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(BrowseInfo), std::move(MetaInfo))); Die(ctx); } @@ -356,17 +356,17 @@ public: void HandlePoisonPill(const TActorContext& ctx) { Die(ctx); } - - void HandleUndelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { - if (ev->Get()->SourceType == TEvTxUserProxy::EvNavigate) { - ++Responses; - ctx.Send(BrowseContext.Owner, new NViewerEvents::TEvBrowseRequestCompleted(TxProxy, TEvTxUserProxy::EvNavigate)); + + void HandleUndelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { + if (ev->Get()->SourceType == TEvTxUserProxy::EvNavigate) { + ++Responses; + ctx.Send(BrowseContext.Owner, new NViewerEvents::TEvBrowseRequestCompleted(TxProxy, TEvTxUserProxy::EvNavigate)); TxProxy = TActorId(); // do not query this TxProxy any more if (Responses == Requests) { - NextStep(ctx); - } - } - } + NextStep(ctx); + } + } + } }; class TBrowseTabletsCommon : public TActorBootstrapped<TBrowseTabletsCommon> { diff --git a/ydb/core/viewer/browse_db.h b/ydb/core/viewer/browse_db.h index 915348e1b26..fe7b5d31408 100644 --- a/ydb/core/viewer/browse_db.h +++ b/ydb/core/viewer/browse_db.h @@ -194,7 +194,7 @@ public: } } - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); Die(ctx); } }; diff --git a/ydb/core/viewer/browse_events.h b/ydb/core/viewer/browse_events.h index b7096be4621..96ced773938 100644 --- a/ydb/core/viewer/browse_events.h +++ b/ydb/core/viewer/browse_events.h @@ -1,43 +1,43 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/event_local.h> #include <ydb/core/viewer/protos/viewer.pb.h> - -#include <util/generic/string.h> - -namespace NKikimr { -namespace NViewer { - -namespace NViewerEvents { - enum EEv { - EvBrowseResponse = EventSpaceBegin(TKikimrEvents::ES_VIEWER), + +#include <util/generic/string.h> + +namespace NKikimr { +namespace NViewer { + +namespace NViewerEvents { + enum EEv { + EvBrowseResponse = EventSpaceBegin(TKikimrEvents::ES_VIEWER), EvBrowseRequestSent, EvBrowseRequestCompleted, - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_VIEWER), "expected EvEnd < EventSpaceEnd"); - + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_VIEWER), "expected EvEnd < EventSpaceEnd"); + using TTabletId = ui64; - struct TEvBrowseResponse : TEventLocal<TEvBrowseResponse, EvBrowseResponse> { - TEvBrowseResponse( - NKikimrViewer::TBrowseInfo&& browseInfo, - NKikimrViewer::TMetaInfo&& metaInfo - ) - : BrowseInfo(std::move(browseInfo)) - , MetaInfo(std::move(metaInfo)) - {} - - TEvBrowseResponse(const TString& error) - : Error(error) - {} - - NKikimrViewer::TBrowseInfo BrowseInfo; - NKikimrViewer::TMetaInfo MetaInfo; - TString Error; - }; + struct TEvBrowseResponse : TEventLocal<TEvBrowseResponse, EvBrowseResponse> { + TEvBrowseResponse( + NKikimrViewer::TBrowseInfo&& browseInfo, + NKikimrViewer::TMetaInfo&& metaInfo + ) + : BrowseInfo(std::move(browseInfo)) + , MetaInfo(std::move(metaInfo)) + {} + + TEvBrowseResponse(const TString& error) + : Error(error) + {} + + NKikimrViewer::TBrowseInfo BrowseInfo; + NKikimrViewer::TMetaInfo MetaInfo; + TString Error; + }; struct TEvBrowseRequestSent : TEventLocal<TEvBrowseRequestSent, EvBrowseRequestSent> { TActorId Actor; @@ -74,7 +74,7 @@ namespace NViewerEvents { , Event(event) {} }; -} // namespace NViewerEvents - -} // namespace NViewer -} // namespace NKikimr +} // namespace NViewerEvents + +} // namespace NViewer +} // namespace NKikimr diff --git a/ydb/core/viewer/browse_pq.h b/ydb/core/viewer/browse_pq.h index 7eaa66511cf..3e3b32a21e3 100644 --- a/ydb/core/viewer/browse_pq.h +++ b/ydb/core/viewer/browse_pq.h @@ -38,7 +38,7 @@ public: NKikimrViewer::TBrowseInfo& pbConsumers = *browseInfo.AddChildren(); pbConsumers.SetType(NKikimrViewer::EObjectType::Consumers); pbConsumers.SetName("Consumers"); - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); Die(ctx); } }; @@ -301,7 +301,7 @@ public: pbConsumer.SetType(NKikimrViewer::EObjectType::Consumer); pbConsumer.SetName(pr.first); } - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); Die(ctx); } }; @@ -349,7 +349,7 @@ public: } } } - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); Die(ctx); } }; @@ -414,7 +414,7 @@ public: } } - ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); + ctx.Send(Owner, new NViewerEvents::TEvBrowseResponse(std::move(browseInfo), std::move(metaInfo))); Die(ctx); } }; diff --git a/ydb/core/viewer/json_browse.h b/ydb/core/viewer/json_browse.h index c01bd7991c9..cba672826ee 100644 --- a/ydb/core/viewer/json_browse.h +++ b/ydb/core/viewer/json_browse.h @@ -117,7 +117,7 @@ public: STFUNC(StateWait) { switch (ev->GetTypeRewrite()) { - HFunc(NViewerEvents::TEvBrowseResponse, Handle); + HFunc(NViewerEvents::TEvBrowseResponse, Handle); HFunc(NViewerEvents::TEvBrowseRequestSent, Handle); HFunc(NViewerEvents::TEvBrowseRequestCompleted, Handle); HFunc(NMon::TEvHttpInfoRes, Handle); @@ -125,8 +125,8 @@ public: } } - void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { - NViewerEvents::TEvBrowseResponse& event(*ev->Get()); + void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { + NViewerEvents::TEvBrowseResponse& event(*ev->Get()); if (!event.Error.empty()) { ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(event.Error, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); return Die(ctx); diff --git a/ydb/core/viewer/json_content.h b/ydb/core/viewer/json_content.h index 7ca58b296fa..79b230f1288 100644 --- a/ydb/core/viewer/json_content.h +++ b/ydb/core/viewer/json_content.h @@ -16,123 +16,123 @@ using ::google::protobuf::FieldDescriptor; class TJsonContent : public TActorBootstrapped<TJsonContent> { using TThis = TJsonContent; using TBase = TActorBootstrapped<TJsonContent>; - - IViewer* Viewer; + + IViewer* Viewer; TActorId Initiator; NMon::TEvHttpInfo::TPtr Event; - IViewer::TContentRequestContext ContentRequestContext; - TInstant BrowseStarted; - + IViewer::TContentRequestContext ContentRequestContext; + TInstant BrowseStarted; + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::VIEWER_HANDLER; } - TJsonContent(IViewer* viewer, NMon::TEvHttpInfo::TPtr& ev) - : Viewer(viewer) - , Initiator(ev->Sender) + TJsonContent(IViewer* viewer, NMon::TEvHttpInfo::TPtr& ev) + : Viewer(viewer) + , Initiator(ev->Sender) , Event(ev) {} - STFUNC(StateWaitingBrowse) { + STFUNC(StateWaitingBrowse) { switch (ev->GetTypeRewrite()) { - HFunc(NViewerEvents::TEvBrowseResponse, HandleBrowseResponse); - CFunc(TEvents::TSystem::Wakeup, HandleBrowseTimeout); + HFunc(NViewerEvents::TEvBrowseResponse, HandleBrowseResponse); + CFunc(TEvents::TSystem::Wakeup, HandleBrowseTimeout); } } -public: - void Bootstrap(const TActorContext& ctx) { +public: + void Bootstrap(const TActorContext& ctx) { BuildRequestContext(&Event->Get()->Request, ContentRequestContext); if (!Event->Get()->UserToken.empty()) { ContentRequestContext.UserToken = Event->Get()->UserToken; } - BrowseStarted = ctx.Now(); + BrowseStarted = ctx.Now(); ctx.RegisterWithSameMailbox(new TBrowse(Viewer, ctx.SelfID, ContentRequestContext.Path, Event->Get()->UserToken)); - - TBase::Become( - &TThis::StateWaitingBrowse, - ctx, - ContentRequestContext.Timeout, - new TEvents::TEvWakeup()); - } - + + TBase::Become( + &TThis::StateWaitingBrowse, + ctx, + ContentRequestContext.Timeout, + new TEvents::TEvWakeup()); + } + private: - static void BuildRequestContext( + static void BuildRequestContext( const NMonitoring::IMonHttpRequest* httpRequest, - IViewer::TContentRequestContext& reqCtx) { - if (!httpRequest) { - return; + IViewer::TContentRequestContext& reqCtx) { + if (!httpRequest) { + return; } - const auto& params = httpRequest->GetParams(); - auto post = httpRequest->GetPostContent(); - - reqCtx.JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>( - params.Get("enums"), - !reqCtx.JsonSettings.EnumAsNumbers); // defaults to false - reqCtx.JsonSettings.UI64AsString = !FromStringWithDefault<bool>( - params.Get("ui64"), - !reqCtx.JsonSettings.UI64AsString); // defaults to false - - ui32 timeoutMillis = FromStringWithDefault<ui32>( - params.Get("timeout"), - (ui32)reqCtx.Timeout.MilliSeconds()); - reqCtx.Timeout = TDuration::MilliSeconds(timeoutMillis); - - reqCtx.Limit = FromStringWithDefault<ui32>(params.Get("limit"), reqCtx.Limit); - reqCtx.Offset = FromStringWithDefault<ui32>(params.Get("offset"), reqCtx.Offset); - reqCtx.Key = post; - + const auto& params = httpRequest->GetParams(); + auto post = httpRequest->GetPostContent(); + + reqCtx.JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>( + params.Get("enums"), + !reqCtx.JsonSettings.EnumAsNumbers); // defaults to false + reqCtx.JsonSettings.UI64AsString = !FromStringWithDefault<bool>( + params.Get("ui64"), + !reqCtx.JsonSettings.UI64AsString); // defaults to false + + ui32 timeoutMillis = FromStringWithDefault<ui32>( + params.Get("timeout"), + (ui32)reqCtx.Timeout.MilliSeconds()); + reqCtx.Timeout = TDuration::MilliSeconds(timeoutMillis); + + reqCtx.Limit = FromStringWithDefault<ui32>(params.Get("limit"), reqCtx.Limit); + reqCtx.Offset = FromStringWithDefault<ui32>(params.Get("offset"), reqCtx.Offset); + reqCtx.Key = post; + if (params.Has("key")) { - reqCtx.Key = params.Get("key"); + reqCtx.Key = params.Get("key"); } - - reqCtx.Path = params.Get("path"); + + reqCtx.Path = params.Get("path"); } - void HandleBrowseResponse(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { - NViewerEvents::TEvBrowseResponse& event = *ev->Get(); + void HandleBrowseResponse(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { + NViewerEvents::TEvBrowseResponse& event = *ev->Get(); + + if (!event.Error.empty()) { + return SendErrorReplyAndDie(event.Error, ctx); + } - if (!event.Error.empty()) { - return SendErrorReplyAndDie(event.Error, ctx); + auto type = event.BrowseInfo.GetType(); + auto contentHandler = Viewer->GetContentHandler(type); + if (!contentHandler) { + return SendErrorReplyAndDie(TStringBuilder() + << "HTTP/1.1 500 Internal Server Error\r\n" + "Connection: Close\r\n" + "\r\n" + "No content can be retrieved from " + << (NKikimrViewer::EObjectType_IsValid((int)type) ? NKikimrViewer::EObjectType_Name(type) : TString("unknown")) + << " object\r\n", + ctx); } - auto type = event.BrowseInfo.GetType(); - auto contentHandler = Viewer->GetContentHandler(type); - if (!contentHandler) { - return SendErrorReplyAndDie(TStringBuilder() - << "HTTP/1.1 500 Internal Server Error\r\n" - "Connection: Close\r\n" - "\r\n" - "No content can be retrieved from " - << (NKikimrViewer::EObjectType_IsValid((int)type) ? NKikimrViewer::EObjectType_Name(type) : TString("unknown")) - << " object\r\n", - ctx); - } - - ContentRequestContext.Type = event.BrowseInfo.GetType(); - ContentRequestContext.ObjectName = event.BrowseInfo.GetName(); - ContentRequestContext.Timeout -= (ctx.Now() - BrowseStarted); - - // spawn content retrieval actor + ContentRequestContext.Type = event.BrowseInfo.GetType(); + ContentRequestContext.ObjectName = event.BrowseInfo.GetName(); + ContentRequestContext.Timeout -= (ctx.Now() - BrowseStarted); + + // spawn content retrieval actor ctx.RegisterWithSameMailbox(contentHandler(Initiator, ContentRequestContext)); - Die(ctx); + Die(ctx); } - void HandleBrowseTimeout(const TActorContext& ctx) { + void HandleBrowseTimeout(const TActorContext& ctx) { return SendErrorReplyAndDie(Viewer->GetHTTPGATEWAYTIMEOUT(), ctx); } - void SendErrorReplyAndDie(const TString& error, const TActorContext& ctx) { - ctx.Send( - Initiator, - new NMon::TEvHttpInfoRes( - error, - 0, - NMon::IEvHttpInfoRes::EContentType::Custom)); - + void SendErrorReplyAndDie(const TString& error, const TActorContext& ctx) { + ctx.Send( + Initiator, + new NMon::TEvHttpInfoRes( + error, + 0, + NMon::IEvHttpInfoRes::EContentType::Custom)); + Die(ctx); } }; diff --git a/ydb/core/viewer/json_metainfo.h b/ydb/core/viewer/json_metainfo.h index 15468e92991..54064d74037 100644 --- a/ydb/core/viewer/json_metainfo.h +++ b/ydb/core/viewer/json_metainfo.h @@ -64,7 +64,7 @@ public: STFUNC(StateWait) { switch (ev->GetTypeRewrite()) { - HFunc(NViewerEvents::TEvBrowseResponse, Handle); + HFunc(NViewerEvents::TEvBrowseResponse, Handle); HFunc(NViewerEvents::TEvBrowseRequestSent, Handle); HFunc(NViewerEvents::TEvBrowseRequestCompleted, Handle); HFunc(NMon::TEvHttpInfoRes, Handle); @@ -77,8 +77,8 @@ public: Die(ctx); } - void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { - NViewerEvents::TEvBrowseResponse& event(*ev->Get()); + void Handle(NViewerEvents::TEvBrowseResponse::TPtr &ev, const TActorContext &ctx) { + NViewerEvents::TEvBrowseResponse& event(*ev->Get()); if (!event.Error.empty()) { ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(event.Error, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); return Die(ctx); diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 647f8003810..9f6c822a89f 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -19,12 +19,12 @@ enum EObjectType { Consumers = 5; Operations = 6; Consumer = 7; - RtmrTables = 8; - RtmrOperations = 9; // flat list of operations - RtmrTable = 10; - RtmrOperation = 11; - RtmrTasks = 12; // flat list of tasks - RtmrTask = 13; // task is the group of operations, associated tables and some metadata + RtmrTables = 8; + RtmrOperations = 9; // flat list of operations + RtmrTable = 10; + RtmrOperation = 11; + RtmrTasks = 12; // flat list of tasks + RtmrTask = 13; // task is the group of operations, associated tables and some metadata RtmrVolume = 14; BlockStoreVolume = 15; Kesus = 16; @@ -86,16 +86,16 @@ message TMetaCommonInfo { repeated TACE ACL = 17; TBackupInfo Backup = 18; TResources Resources = 19; - - // RTMR-specific table stats - uint64 DistinctKeysCount = 20; - uint64 UpdatesCount = 21; - string LastKey = 22; - string TopKey = 23; - uint64 TopKeyBytesCount = 24; - string MinSubKey = 25; - string MaxSubKey = 26; - uint64 MinTimestamp = 27; // timestamp of the oldest entry in the table, milliseconds since epoch + + // RTMR-specific table stats + uint64 DistinctKeysCount = 20; + uint64 UpdatesCount = 21; + string LastKey = 22; + string TopKey = 23; + uint64 TopKeyBytesCount = 24; + string MinSubKey = 25; + string MaxSubKey = 26; + uint64 MinTimestamp = 27; // timestamp of the oldest entry in the table, milliseconds since epoch // Datashard-specific stats uint64 ImmediateTxCompleted = 30; @@ -118,23 +118,23 @@ message TMetaColumnInfo { message TMetaTableInfo { repeated TMetaColumnInfo Schema = 1; - - // RTMR-specific table information - /// Lifetime in seconds - uint32 Lifetime = 2; - /// Min absolute timestamp allowed - uint64 MinTimestamp = 3; - /// Keep last n records per key - uint32 MaxRecords = 4; - /// Keep only one last record per (key, subkey) pair - bool Unique = 5; - /// Specifies if table belongs to specific task; read-only - string TaskId = 6; - /// Arbitrary data associated with the table - map<string, string> Attrs = 7; - - // RTMR-specific view information - repeated string SrcTables = 8; + + // RTMR-specific table information + /// Lifetime in seconds + uint32 Lifetime = 2; + /// Min absolute timestamp allowed + uint64 MinTimestamp = 3; + /// Keep last n records per key + uint32 MaxRecords = 4; + /// Keep only one last record per (key, subkey) pair + bool Unique = 5; + /// Specifies if table belongs to specific task; read-only + string TaskId = 6; + /// Arbitrary data associated with the table + map<string, string> Attrs = 7; + + // RTMR-specific view information + repeated string SrcTables = 8; } message TMetaTopicLagInfo { @@ -176,8 +176,8 @@ message TMetaTopicInfo { repeated TMetaTopicPartitionInfo Partitions = 10; } -message TMetaRtmrOperationInfo { - message TSettings { +message TMetaRtmrOperationInfo { + message TSettings { // Options passed to the operation instance string Options = 1; // Force pause between operation executions on one key; seconds @@ -204,80 +204,80 @@ message TMetaRtmrOperationInfo { string LibraryName = 10; // Tolerance to failures during operation execution uint32 SuspendThreshold = 12; - }; - - message TStats { - uint64 In = 1; - uint64 Count = 2; - uint64 Out = 3; - double ExecuteTime = 4; - double ReadTime = 5; - double LoadStateTime = 6; - double SaveStateTime = 7; - }; - - message TSystemTableSummary { - string Type = 1; - string Name = 2; - uint32 Lifetime = 3; // Lifetime in seconds - uint64 DataSize = 4; // in bytes + }; + + message TStats { + uint64 In = 1; + uint64 Count = 2; + uint64 Out = 3; + double ExecuteTime = 4; + double ReadTime = 5; + double LoadStateTime = 6; + double SaveStateTime = 7; + }; + + message TSystemTableSummary { + string Type = 1; + string Name = 2; + uint32 Lifetime = 3; // Lifetime in seconds + uint64 DataSize = 4; // in bytes uint64 Entries = 5; uint64 UpdatesCount = 6; uint64 UpdateTime = 7; // milliseconds since epoch - }; - - bool Enabled = 1; - - // Unique operation name - string Name = 2; - // Operation instance name - string ClassName = 3; - // Species task to which this operation belongs; read-only - string TaskId = 4; - - // List of tables/views to read data from; mandatory upon creation - repeated string SrcTables = 5; - // List of tables to write to - repeated string DstTables = 6; - - // operation settings - string Mode = 7; - bool Incremental = 8; - bool Stateful = 9; - bool NoPreview = 10; - repeated string Flags = 11; - - // Lifetime in seconds for operation state - uint32 StateLifetime = 12; - // Lifetime in seconds for operation results - uint32 ResultLifetime = 13; - - TSettings Settings = 14; - - TStats Stats = 15; - - repeated TSystemTableSummary SystemTables = 16; -} - -message TMetaRtmrTaskInfo { - string TaskId = 1; - // indicates that whole group of operations is either enabled or disabed - bool Enabled = 2; - - // associated tables - repeated string OwnedTables = 3; - // associated operations - repeated string Operations = 4; -} - + }; + + bool Enabled = 1; + + // Unique operation name + string Name = 2; + // Operation instance name + string ClassName = 3; + // Species task to which this operation belongs; read-only + string TaskId = 4; + + // List of tables/views to read data from; mandatory upon creation + repeated string SrcTables = 5; + // List of tables to write to + repeated string DstTables = 6; + + // operation settings + string Mode = 7; + bool Incremental = 8; + bool Stateful = 9; + bool NoPreview = 10; + repeated string Flags = 11; + + // Lifetime in seconds for operation state + uint32 StateLifetime = 12; + // Lifetime in seconds for operation results + uint32 ResultLifetime = 13; + + TSettings Settings = 14; + + TStats Stats = 15; + + repeated TSystemTableSummary SystemTables = 16; +} + +message TMetaRtmrTaskInfo { + string TaskId = 1; + // indicates that whole group of operations is either enabled or disabed + bool Enabled = 2; + + // associated tables + repeated string OwnedTables = 3; + // associated operations + repeated string Operations = 4; +} + message TMetaInfo { TMetaCommonInfo Common = 1; TMetaTableInfo Table = 2; TMetaTopicInfo Topic = 3; TMetaTopicConsumerInfo Consumer = 4; NKikimrTabletBase.TTabletCounters Counters = 9; - TMetaRtmrOperationInfo RtmrOperation = 10; - TMetaRtmrTaskInfo RtmrTask = 11; + TMetaRtmrOperationInfo RtmrOperation = 10; + TMetaRtmrTaskInfo RtmrTask = 11; } message TTenantResource { diff --git a/ydb/core/viewer/viewer.cpp b/ydb/core/viewer/viewer.cpp index e4f6e80ed88..f1124115bf7 100644 --- a/ydb/core/viewer/viewer.cpp +++ b/ydb/core/viewer/viewer.cpp @@ -92,37 +92,37 @@ public: } }; -void SetupPQVirtualHandlers(IViewer* viewer) { - viewer->RegisterVirtualHandler( - NKikimrViewer::EObjectType::Root, +void SetupPQVirtualHandlers(IViewer* viewer) { + viewer->RegisterVirtualHandler( + NKikimrViewer::EObjectType::Root, [] (const TActorId& owner, const IViewer::TBrowseContext& browseContext) -> IActor* { - return new NViewerPQ::TBrowseRoot(owner, browseContext); - }); - viewer->RegisterVirtualHandler( - NKikimrViewer::EObjectType::Consumers, + return new NViewerPQ::TBrowseRoot(owner, browseContext); + }); + viewer->RegisterVirtualHandler( + NKikimrViewer::EObjectType::Consumers, [] (const TActorId& owner, const IViewer::TBrowseContext& browseContext) -> IActor* { - return new NViewerPQ::TBrowseConsumers(owner, browseContext); - }); - viewer->RegisterVirtualHandler( - NKikimrViewer::EObjectType::Consumer, + return new NViewerPQ::TBrowseConsumers(owner, browseContext); + }); + viewer->RegisterVirtualHandler( + NKikimrViewer::EObjectType::Consumer, [] (const TActorId& owner, const IViewer::TBrowseContext& browseContext) -> IActor* { - return new NViewerPQ::TBrowseConsumer(owner, browseContext); - }); - viewer->RegisterVirtualHandler( - NKikimrViewer::EObjectType::Topic, + return new NViewerPQ::TBrowseConsumer(owner, browseContext); + }); + viewer->RegisterVirtualHandler( + NKikimrViewer::EObjectType::Topic, [] (const TActorId& owner, const IViewer::TBrowseContext& browseContext) -> IActor* { - return new NViewerPQ::TBrowseTopic(owner, browseContext); - }); -} - -void SetupDBVirtualHandlers(IViewer* viewer) { - viewer->RegisterVirtualHandler( - NKikimrViewer::EObjectType::Table, + return new NViewerPQ::TBrowseTopic(owner, browseContext); + }); +} + +void SetupDBVirtualHandlers(IViewer* viewer) { + viewer->RegisterVirtualHandler( + NKikimrViewer::EObjectType::Table, [] (const TActorId& owner, const IViewer::TBrowseContext& browseContext) -> IActor* { - return new NViewerDB::TBrowseTable(owner, browseContext); - }); -} - + return new NViewerDB::TBrowseTable(owner, browseContext); + }); +} + class TViewer : public TActorBootstrapped<TViewer>, public IViewer { public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -213,7 +213,7 @@ public: return KikimrRunConfig; } - void RegisterVirtualHandler( + void RegisterVirtualHandler( NKikimrViewer::EObjectType parentObjectType, TVirtualHandlerType handler) override { VirtualHandlersByParentType.insert(std::make_pair(parentObjectType, TVirtualHandler(handler))); @@ -228,24 +228,24 @@ public: return handlers; } - TContentHandler GetContentHandler(NKikimrViewer::EObjectType objectType) const override { - auto rec = ContentHandlers.find(objectType); - return (rec != ContentHandlers.end()) ? rec->second : (TContentHandler)nullptr; - } - - void RegisterContentHandler( - NKikimrViewer::EObjectType objectType, - const TContentHandler& handler) override { - if (handler) { - ContentHandlers.emplace(objectType, handler); - } - } - + TContentHandler GetContentHandler(NKikimrViewer::EObjectType objectType) const override { + auto rec = ContentHandlers.find(objectType); + return (rec != ContentHandlers.end()) ? rec->second : (TContentHandler)nullptr; + } + + void RegisterContentHandler( + NKikimrViewer::EObjectType objectType, + const TContentHandler& handler) override { + if (handler) { + ContentHandlers.emplace(objectType, handler); + } + } + private: THashMap<TString, TAutoPtr<TJsonHandlerBase>> JsonHandlers; const TKikimrRunConfig KikimrRunConfig; std::unordered_multimap<NKikimrViewer::EObjectType, TVirtualHandler> VirtualHandlersByParentType; - std::unordered_map<NKikimrViewer::EObjectType, TContentHandler> ContentHandlers; + std::unordered_map<NKikimrViewer::EObjectType, TContentHandler> ContentHandlers; STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { @@ -369,7 +369,7 @@ private: response.Write(blob.data(), blob.size()); ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(response.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); } - return true; + return true; } return false; } @@ -459,33 +459,33 @@ private: } }; -TString IViewer::TContentRequestContext::Dump() const -{ - auto typeToString = [] (int type) -> TString { - using namespace NKikimrViewer; - if (EObjectType_IsValid(type)) { - return EObjectType_Name((EObjectType)type); - } else { - return TStringBuilder() << "unknown (" << type << ")"; - } - }; - - TStringBuilder result; - result << "Path = " << Path << Endl; - result << "Name = " << ObjectName << Endl; - result << "Type = " << typeToString(Type) << Endl; - - result << "Limit = " << Limit << Endl; - result << "Offset = " << Offset << Endl; - result << "Key = " << Key << Endl; - - result << "JsonSettings.EnumAsNumbers = " << JsonSettings.EnumAsNumbers << Endl; - result << "JsonSettings.UI64AsString = " << JsonSettings.UI64AsString << Endl; - result << "Timeout = " << Timeout.MilliSeconds() << " ms" << Endl; - - return result; -} - +TString IViewer::TContentRequestContext::Dump() const +{ + auto typeToString = [] (int type) -> TString { + using namespace NKikimrViewer; + if (EObjectType_IsValid(type)) { + return EObjectType_Name((EObjectType)type); + } else { + return TStringBuilder() << "unknown (" << type << ")"; + } + }; + + TStringBuilder result; + result << "Path = " << Path << Endl; + result << "Name = " << ObjectName << Endl; + result << "Type = " << typeToString(Type) << Endl; + + result << "Limit = " << Limit << Endl; + result << "Offset = " << Offset << Endl; + result << "Key = " << Key << Endl; + + result << "JsonSettings.EnumAsNumbers = " << JsonSettings.EnumAsNumbers << Endl; + result << "JsonSettings.UI64AsString = " << JsonSettings.UI64AsString << Endl; + result << "Timeout = " << Timeout.MilliSeconds() << " ms" << Endl; + + return result; +} + ui32 CurrentMonitoringPort = 8765; IActor* CreateViewer(const TKikimrRunConfig &kikimrRunConfig) { diff --git a/ydb/core/viewer/viewer.h b/ydb/core/viewer/viewer.h index 9c2a89a8211..ab34762e3ed 100644 --- a/ydb/core/viewer/viewer.h +++ b/ydb/core/viewer/viewer.h @@ -1,7 +1,7 @@ #pragma once #include <ydb/core/viewer/json/json.h> - + #include <ydb/core/tablet/defs.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/actor.h> @@ -79,48 +79,48 @@ public: {} }; - struct TContentRequestContext { - // request settings - TJsonSettings JsonSettings; - TDuration Timeout = TDuration::MilliSeconds(10000); + struct TContentRequestContext { + // request settings + TJsonSettings JsonSettings; + TDuration Timeout = TDuration::MilliSeconds(10000); TString UserToken; - - // filter - ui32 Limit = 50; - ui32 Offset = 0; - TString Key; - - // path to object - TString Path; - - // object type and location - NKikimrViewer::EObjectType Type = NKikimrViewer::Unknown; - TString ObjectName; - - TString Dump() const; - }; - + + // filter + ui32 Limit = 50; + ui32 Offset = 0; + TString Key; + + // path to object + TString Path; + + // object type and location + NKikimrViewer::EObjectType Type = NKikimrViewer::Unknown; + TString ObjectName; + + TString Dump() const; + }; + using TContentHandler = std::function<IActor*(const TActorId&, const TContentRequestContext&)>; - + virtual const TKikimrRunConfig& GetKikimrRunConfig() const = 0; virtual TVector<const TVirtualHandler*> GetVirtualHandlers(NKikimrViewer::EObjectType type, const TString& path) const = 0; virtual void RegisterVirtualHandler(NKikimrViewer::EObjectType parentObjectType, TVirtualHandlerType handler) = 0; - - // returns nullptr if no such handler - virtual TContentHandler GetContentHandler(NKikimrViewer::EObjectType objectType) const = 0; - - virtual void RegisterContentHandler( - NKikimrViewer::EObjectType objectType, - const TContentHandler& handler) = 0; + + // returns nullptr if no such handler + virtual TContentHandler GetContentHandler(NKikimrViewer::EObjectType objectType) const = 0; + + virtual void RegisterContentHandler( + NKikimrViewer::EObjectType objectType, + const TContentHandler& handler) = 0; TString GetHTTPOKJSON(); TString GetHTTPGATEWAYTIMEOUT(); }; -void SetupPQVirtualHandlers(IViewer* viewer); -void SetupDBVirtualHandlers(IViewer* viewer); -void SetupKqpContentHandler(IViewer* viewer); - +void SetupPQVirtualHandlers(IViewer* viewer); +void SetupDBVirtualHandlers(IViewer* viewer); +void SetupKqpContentHandler(IViewer* viewer); + template <typename RequestType> struct TJsonRequestSchema { static TString GetSchema() { return TString(); } diff --git a/ydb/core/yql_testlib/ya.make b/ydb/core/yql_testlib/ya.make index cfe401d0d34..a68cca63d3f 100644 --- a/ydb/core/yql_testlib/ya.make +++ b/ydb/core/yql_testlib/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(spuchin) - -SRCS( - yql_testlib.cpp - yql_testlib.h -) - -PEERDIR( + +SRCS( + yql_testlib.cpp + yql_testlib.h +) + +PEERDIR( library/cpp/grpc/client library/cpp/regex/pcre library/cpp/testing/unittest @@ -28,8 +28,8 @@ PEERDIR( ydb/library/yql/core ydb/library/yql/providers/common/provider ydb/library/yql/providers/common/udf_resolve -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/yql_testlib/yql_testlib.cpp b/ydb/core/yql_testlib/yql_testlib.cpp index 000ce6b9368..35838c40f6a 100644 --- a/ydb/core/yql_testlib/yql_testlib.cpp +++ b/ydb/core/yql_testlib/yql_testlib.cpp @@ -1,5 +1,5 @@ -#include "yql_testlib.h" - +#include "yql_testlib.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/hive.h> #include <ydb/public/lib/base/msgbus.h> @@ -9,32 +9,32 @@ #include <ydb/core/engine/mkql_engine_flat.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - + #include <ydb/core/testlib/actors/test_runtime.h> #include <ydb/core/testlib/basics/appdata.h> #include <ydb/core/testlib/tablet_helpers.h> - + #include <ydb/library/yql/core/facade/yql_facade.h> #include <ydb/library/yql/public/udf/udf_helpers.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> #include <ydb/core/keyvalue/keyvalue.h> - + #include <ydb/core/client/server/msgbus_server_tracer.h> - -using namespace NKikimr::NUdf; - -namespace { - -class TRepeat: public TBoxedValue -{ -public: - static TStringRef Name() { - static auto name = TStringRef::Of("Repeat"); - return name; - } - + +using namespace NKikimr::NUdf; + +namespace { + +class TRepeat: public TBoxedValue +{ +public: + static TStringRef Name() { + static auto name = TStringRef::Of("Repeat"); + return name; + } + private: TUnboxedValue Run( const NKikimr::NUdf::IValueBuilder* valueBuilder, @@ -48,26 +48,26 @@ private: } return valueBuilder->NewString(res); } -}; - - -class TSendPoisonPill: public TBoxedValue -{ - mutable NKikimr::TTestActorRuntime* Runtime; - NThreading::TFuture<void> ResumeYqlExecution; - -public: - TSendPoisonPill(NKikimr::TTestActorRuntime* runtime, const NThreading::TFuture<void> resumeYqlExecution) - : Runtime(runtime) - , ResumeYqlExecution(resumeYqlExecution) { - } - - static TStringRef Name() { - static auto name = TStringRef::Of("SendPoisonPill"); - return name; - } - -private: +}; + + +class TSendPoisonPill: public TBoxedValue +{ + mutable NKikimr::TTestActorRuntime* Runtime; + NThreading::TFuture<void> ResumeYqlExecution; + +public: + TSendPoisonPill(NKikimr::TTestActorRuntime* runtime, const NThreading::TFuture<void> resumeYqlExecution) + : Runtime(runtime) + , ResumeYqlExecution(resumeYqlExecution) { + } + + static TStringRef Name() { + static auto name = TStringRef::Of("SendPoisonPill"); + return name; + } + +private: NKikimr::NUdf::TUnboxedValue Run( const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override @@ -86,64 +86,64 @@ private: ui64 result = 0; return TUnboxedValuePod(result); } -}; - -class TTestUDFs: public IUdfModule -{ - mutable NKikimr::TTestActorRuntime* Runtime; - NThreading::TFuture<void> ResumeYqlExecution; - -public: - TTestUDFs(NKikimr::TTestActorRuntime* runtime, const NThreading::TFuture<void>& resumeYqlExecution) - : Runtime(runtime) - , ResumeYqlExecution(resumeYqlExecution) { - } - - TStringRef Name() const { - return TStringRef::Of("TestUDFs"); - } - +}; + +class TTestUDFs: public IUdfModule +{ + mutable NKikimr::TTestActorRuntime* Runtime; + NThreading::TFuture<void> ResumeYqlExecution; + +public: + TTestUDFs(NKikimr::TTestActorRuntime* runtime, const NThreading::TFuture<void>& resumeYqlExecution) + : Runtime(runtime) + , ResumeYqlExecution(resumeYqlExecution) { + } + + TStringRef Name() const { + return TStringRef::Of("TestUDFs"); + } + void CleanupOnTerminate() const final {} - + void GetAllFunctions(IFunctionsSink& sink) const final { sink.Add(TRepeat::Name()); sink.Add(TSendPoisonPill::Name()); - } - - void BuildFunctionTypeInfo( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, + } + + void BuildFunctionTypeInfo( + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, IFunctionTypeInfoBuilder& builder) const final - { - try { - Y_UNUSED(userType); - Y_UNUSED(typeConfig); - - bool typesOnly = (flags & TFlags::TypesOnly); - - if (TRepeat::Name() == name) { - - builder.SimpleSignature<char*(char*, ui64)>(); - - if (!typesOnly) { - builder.Implementation(new TRepeat); - } - } - else if (TSendPoisonPill::Name() == name) { - builder.SimpleSignature<ui64(ui64)>(); - - if (!typesOnly) { - builder.Implementation(new TSendPoisonPill(Runtime, ResumeYqlExecution)); - } - } - } catch (...) { - builder.SetError(CurrentExceptionMessage()); - } - } -}; - + { + try { + Y_UNUSED(userType); + Y_UNUSED(typeConfig); + + bool typesOnly = (flags & TFlags::TypesOnly); + + if (TRepeat::Name() == name) { + + builder.SimpleSignature<char*(char*, ui64)>(); + + if (!typesOnly) { + builder.Implementation(new TRepeat); + } + } + else if (TSendPoisonPill::Name() == name) { + builder.SimpleSignature<ui64(ui64)>(); + + if (!typesOnly) { + builder.Implementation(new TSendPoisonPill(Runtime, ResumeYqlExecution)); + } + } + } catch (...) { + builder.SetError(CurrentExceptionMessage()); + } + } +}; + } // unnamed namespace @@ -152,7 +152,7 @@ namespace NKikimr { namespace Tests { void TYqlServer::Initialize() { - ResumeYqlExecutionPromise = NThreading::NewPromise<void>(); + ResumeYqlExecutionPromise = NThreading::NewPromise<void>(); Runtime.Reset(new TTestActorRuntime(StaticNodes() + DynamicNodes(), true)); @@ -167,28 +167,28 @@ void TYqlServer::Initialize() { app.AddHive(Settings->Domain, ChangeStateStorage(Hive, Settings->Domain)); app.SetFnRegistry([this](const NKikimr::NScheme::TTypeRegistry& typeRegistry) -> NKikimr::NMiniKQL::IFunctionRegistry* { Y_UNUSED(typeRegistry); - // register test UDFs + // register test UDFs auto freg = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); - freg->AddModule("", "TestUDFs", new TTestUDFs(GetRuntime(), ResumeYqlExecutionPromise.GetFuture())); - return freg.Release(); - } - ); + freg->AddModule("", "TestUDFs", new TTestUDFs(GetRuntime(), ResumeYqlExecutionPromise.GetFuture())); + return freg.Release(); + } + ); SetupMessageBus(GetSettings().Port, GetSettings().TracePath); SetupTabletServices(*Runtime, &app, StaticNodes() == 1 && Settings->EnableMockOnSingleNode, Settings->CustomDiskParams); CreateBootstrapTablets(); - SetupStorage(); + SetupStorage(); for (ui32 nodeIdx = 0; nodeIdx < GetSettings().NodeCount; ++nodeIdx) { SetupDomainLocalService(nodeIdx); SetupProxies(nodeIdx); } - SetupLogging(); -} - - + SetupLogging(); +} + + void MakeGatewaysConfig(const THashMap<TString, TString>& clusterMapping, NYql::TGatewaysConfig& gatewaysConfig) { for (auto& x : clusterMapping) { if (x.second == NYql::YtProviderName) { @@ -206,9 +206,9 @@ void MakeGatewaysConfig(const THashMap<TString, TString>& clusterMapping, NYql:: } void TYqlServer::ResumeYqlExecutionActor() { - ResumeYqlExecutionPromise.SetValue(); -} - -} // namespace NTests - -} // namespace NKikimr + ResumeYqlExecutionPromise.SetValue(); +} + +} // namespace NTests + +} // namespace NKikimr diff --git a/ydb/core/yql_testlib/yql_testlib.h b/ydb/core/yql_testlib/yql_testlib.h index 86e46a6cd4b..39176df699b 100644 --- a/ydb/core/yql_testlib/yql_testlib.h +++ b/ydb/core/yql_testlib/yql_testlib.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> #include <ydb/library/yql/minikql/mkql_program_builder.h> @@ -7,11 +7,11 @@ #include <ydb/core/testlib/test_client.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h> - -namespace NKikimr { - -namespace Tests { - + +namespace NKikimr { + +namespace Tests { + class TYqlServer : public TServer { public: TYqlServer(const TServerSettings& settings) @@ -27,17 +27,17 @@ public: } TYqlServer& operator=(TYqlServer&& server) = default; - - void ResumeYqlExecutionActor(); - + + void ResumeYqlExecutionActor(); + protected: void Initialize(); - + TAutoPtr<IThreadPool> YqlQueue; - NThreading::TPromise<void> ResumeYqlExecutionPromise; -}; - + NThreading::TPromise<void> ResumeYqlExecutionPromise; +}; + void MakeGatewaysConfig(const THashMap<TString, TString>& clusterMapping, NYql::TGatewaysConfig& gatewaysConfig); -} -} +} +} diff --git a/ydb/library/yql/core/yql_csv.cpp b/ydb/library/yql/core/yql_csv.cpp index 3228d844e94..259b6144f1b 100644 --- a/ydb/library/yql/core/yql_csv.cpp +++ b/ydb/library/yql/core/yql_csv.cpp @@ -7,15 +7,15 @@ namespace { const char QUOTE_CH = '"'; const char ESCAPE_CH = '\\'; -const char NULL_CH = '\0'; +const char NULL_CH = '\0'; class TCsvLineParser { public: - TCsvLineParser(const TStringBuf& line, char delimiter) + TCsvLineParser(const TStringBuf& line, char delimiter) : Cur_(line.begin()) , End_(line.end()) - , Delim_(delimiter) - , Prev_(NULL_CH) /* must differ from ESCAPE_CH */ + , Delim_(delimiter) + , Prev_(NULL_CH) /* must differ from ESCAPE_CH */ { } @@ -23,38 +23,38 @@ public: if (Cur_ > End_) return false; const char* tokenStart = Cur_; - while (Cur_ < End_ && (Prev_ == ESCAPE_CH || *Cur_ != QUOTE_CH) && *Cur_ != Delim_) Prev_ = *Cur_++; + while (Cur_ < End_ && (Prev_ == ESCAPE_CH || *Cur_ != QUOTE_CH) && *Cur_ != Delim_) Prev_ = *Cur_++; if (Cur_ == End_) { token = { tokenStart, Cur_ }; - Prev_ = NULL_CH; + Prev_ = NULL_CH; Cur_++; return true; } - if (Prev_ != ESCAPE_CH && *Cur_ == QUOTE_CH) { - Prev_ = *Cur_++; - tokenStart = Cur_; + if (Prev_ != ESCAPE_CH && *Cur_ == QUOTE_CH) { + Prev_ = *Cur_++; + tokenStart = Cur_; while (Cur_ < End_) { // find non escaped quote char - if (*Cur_ == QUOTE_CH && Prev_ != ESCAPE_CH) break; - Prev_ = *Cur_++; + if (*Cur_ == QUOTE_CH && Prev_ != ESCAPE_CH) break; + Prev_ = *Cur_++; } if (Cur_ == End_) ythrow yexception() << "expected closing \""; token = { tokenStart, Cur_ }; - Prev_ = *Cur_++; // skip closing quote char + Prev_ = *Cur_++; // skip closing quote char - if (Cur_ != End_ && *Cur_ != Delim_) + if (Cur_ != End_ && *Cur_ != Delim_) ythrow yexception() << "expected end of line or delimiter"; - Prev_ = *Cur_++; // move out of buffer or skip delimiter + Prev_ = *Cur_++; // move out of buffer or skip delimiter return true; - } else if (*Cur_ == Delim_) { + } else if (*Cur_ == Delim_) { token = { tokenStart, Cur_ }; - Prev_ = *Cur_++; + Prev_ = *Cur_++; return true; } @@ -64,8 +64,8 @@ public: private: const char* Cur_; const char* End_; - const char Delim_; - char Prev_; + const char Delim_; + char Prev_; }; } // namspace @@ -78,7 +78,7 @@ namespace NUtils { /////////////////////////////////////////////////////////////////////////////// TCsvInputStream::TCsvInputStream(IInputStream& slave, char delimiter) : Slave_(slave) - , Delim_(delimiter) + , Delim_(delimiter) { } @@ -88,7 +88,7 @@ TVector<TString> TCsvInputStream::ReadLine() TString line; if (Slave_.ReadLine(line)) { - TCsvLineParser lineParser(line, Delim_); + TCsvLineParser lineParser(line, Delim_); TStringBuf token; while (lineParser.Next(token)) { parts.push_back(UnescapeC(token.data(), token.size())); @@ -104,7 +104,7 @@ TVector<TString> TCsvInputStream::ReadLineWithEscaping() TString line; if (Slave_.ReadLine(line)) { - TCsvLineParser lineParser(line, Delim_); + TCsvLineParser lineParser(line, Delim_); TStringBuf token; while (lineParser.Next(token)) { parts.push_back(TString(token)); @@ -115,55 +115,55 @@ TVector<TString> TCsvInputStream::ReadLineWithEscaping() } /////////////////////////////////////////////////////////////////////////////// -// TCsvInputStream -/////////////////////////////////////////////////////////////////////////////// -TCsvInputBuffer::TCsvInputBuffer(const TStringBuf& buffer, char delimiter) - : Buffer_(buffer) - , Delim_(delimiter) -{ -} - +// TCsvInputStream +/////////////////////////////////////////////////////////////////////////////// +TCsvInputBuffer::TCsvInputBuffer(const TStringBuf& buffer, char delimiter) + : Buffer_(buffer) + , Delim_(delimiter) +{ +} + TVector<TString> TCsvInputBuffer::ReadLine() -{ +{ TVector<TString> parts; - TStringBuf line; - - if (Buffer_.ReadLine(line)) { - TCsvLineParser lineParser(line, Delim_); - TStringBuf token; - while (lineParser.Next(token)) { + TStringBuf line; + + if (Buffer_.ReadLine(line)) { + TCsvLineParser lineParser(line, Delim_); + TStringBuf token; + while (lineParser.Next(token)) { parts.push_back(UnescapeC(token.data(), token.size())); - } - } - - return parts; -} - + } + } + + return parts; +} + TVector<TString> TCsvInputBuffer::ReadLineWithEscaping() -{ +{ TVector<TString> parts; - TStringBuf line; - - if (Buffer_.ReadLine(line)) { - TCsvLineParser lineParser(line, Delim_); - TStringBuf token; - while (lineParser.Next(token)) { + TStringBuf line; + + if (Buffer_.ReadLine(line)) { + TCsvLineParser lineParser(line, Delim_); + TStringBuf token; + while (lineParser.Next(token)) { parts.push_back(TString(token.data(), token.size())); - } - } - - return parts; -} - - -/////////////////////////////////////////////////////////////////////////////// + } + } + + return parts; +} + + +/////////////////////////////////////////////////////////////////////////////// // TCsvOutputStream /////////////////////////////////////////////////////////////////////////////// TCsvOutputStream::TCsvOutputStream(IOutputStream& slave, char delimiter, bool quoteItems) : Slave_(slave) , WasNL_(true) - , Delim_(delimiter) - , QuoteItems_(quoteItems) + , Delim_(delimiter) + , QuoteItems_(quoteItems) { } @@ -174,15 +174,15 @@ void TCsvOutputStream::DoWrite(const void* buf, size_t len) WasNL_ = true; Slave_.Write(buf, len); } else { - if (!WasNL_) Slave_.Write(Delim_); + if (!WasNL_) Slave_.Write(Delim_); WasNL_ = false; - if (QuoteItems_) { - Slave_.Write(QUOTE_CH); - } + if (QuoteItems_) { + Slave_.Write(QUOTE_CH); + } Slave_.Write(EscapeC(charBuf)); - if (QuoteItems_) { - Slave_.Write(QUOTE_CH); - } + if (QuoteItems_) { + Slave_.Write(QUOTE_CH); + } } } diff --git a/ydb/library/yql/core/yql_csv.h b/ydb/library/yql/core/yql_csv.h index 2e92ed4f169..6e3e48b816f 100644 --- a/ydb/library/yql/core/yql_csv.h +++ b/ydb/library/yql/core/yql_csv.h @@ -19,27 +19,27 @@ public: private: IInputStream& Slave_; - const char Delim_; + const char Delim_; }; - + +/////////////////////////////////////////////////////////////////////////////// +// TCsvInputBuffer /////////////////////////////////////////////////////////////////////////////// -// TCsvInputBuffer -/////////////////////////////////////////////////////////////////////////////// -class TCsvInputBuffer { -public: - TCsvInputBuffer(const TStringBuf& buffer, char delimiter = ';'); - +class TCsvInputBuffer { +public: + TCsvInputBuffer(const TStringBuf& buffer, char delimiter = ';'); + TVector<TString> ReadLine(); TVector<TString> ReadLineWithEscaping(); - -private: - TStringBuf Buffer_; - const char Delim_; -}; - - -/////////////////////////////////////////////////////////////////////////////// + +private: + TStringBuf Buffer_; + const char Delim_; +}; + + +/////////////////////////////////////////////////////////////////////////////// // TCsvOutputStream /////////////////////////////////////////////////////////////////////////////// class TCsvOutputStream: public IOutputStream { @@ -60,12 +60,12 @@ public: inline TCsvOutputStream& operator<<(const TVector<TString>& values) { for(const TString& value: values) { - (*this) << value; - } - (*this) << Endl; - return *this; - } - + (*this) << value; + } + (*this) << Endl; + return *this; + } + private: void DoWrite(const void* buf, size_t len) override final; void DoFlush() override; @@ -73,8 +73,8 @@ private: private: IOutputStream& Slave_; bool WasNL_; - const char Delim_; - const bool QuoteItems_; + const char Delim_; + const bool QuoteItems_; }; } // namspace NUtils diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp index 32305da7ada..19f82aae4ed 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp @@ -4490,15 +4490,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLComputationNodeTest) { const auto failgraph = setup.BuildGraph(fail, {}); UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception); } - + { const auto wrongData = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); const auto fail = pb.StrictFromString(wrongData, pb.NewDataType(NUdf::TDataType<double>::Id)); const auto failgraph = setup.BuildGraph(fail, {}); UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception); - } + } } - + Y_UNIT_TEST_LLVM(TestFromBytes) { TSetup<LLVM> setup; TProgramBuilder& pb = *setup.PgmBuilder; diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index df97fdbd464..47ae6e4c8fc 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -3969,9 +3969,9 @@ TRuntimeNode TProgramBuilder::StrictFromString(TRuntimeNode data, TType* type) { callableBuilder.Add(NewDataLiteral(params.first)); callableBuilder.Add(NewDataLiteral(params.second)); } - return TRuntimeNode(callableBuilder.Build(), false); -} - + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::ToBytes(TRuntimeNode data) { return UnaryDataFunction(data, __func__, TDataFunctionFlags::HasStringResult | TDataFunctionFlags::AllowOptionalArgs | TDataFunctionFlags::CommonOptionalResult); } diff --git a/ydb/library/yql/protos/ya.make b/ydb/library/yql/protos/ya.make index e89de2cea14..c9200505fb4 100644 --- a/ydb/library/yql/protos/ya.make +++ b/ydb/library/yql/protos/ya.make @@ -1,13 +1,13 @@ PROTO_LIBRARY() - + OWNER(g:yql) - -SRCS( + +SRCS( common.proto yql_mount.proto clickhouse.proto -) - +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h index 60f47987005..f11bd9771f1 100644 --- a/ydb/public/lib/base/msgbus.h +++ b/ydb/public/lib/base/msgbus.h @@ -22,16 +22,16 @@ enum { MTYPE_CLIENT_SCHEME_NAVIGATE = 10407, MTYPE_CLIENT_TYPES_REQUEST = 10408, MTYPE_CLIENT_TYPES_RESPONSE = 10409, - MTYPE_CLIENT_CREATE_TABLET_POOL = 10410, // deprecated - MTYPE_CLIENT_RESOLVE_JOB_RUNNER = 10411, // deprecated - MTYPE_CLIENT_RUN_JOB = 10412, // deprecated - MTYPE_CLIENT_JOB_RESPONSE = 10413, // deprecated - MTYPE_CLIENT_REQUEST_JOB_EXECUTION = 10414, // deprecated - MTYPE_CLIENT_DELAYED_JOB_LAUNCH = 10415, // deprecated - MTYPE_CLIENT_JOB_EXECUTION_STATUS = 10416, // deprecated - MTYPE_CLIENT_REQUEST_JOB_INFORMATION = 10417, // deprecated + MTYPE_CLIENT_CREATE_TABLET_POOL = 10410, // deprecated + MTYPE_CLIENT_RESOLVE_JOB_RUNNER = 10411, // deprecated + MTYPE_CLIENT_RUN_JOB = 10412, // deprecated + MTYPE_CLIENT_JOB_RESPONSE = 10413, // deprecated + MTYPE_CLIENT_REQUEST_JOB_EXECUTION = 10414, // deprecated + MTYPE_CLIENT_DELAYED_JOB_LAUNCH = 10415, // deprecated + MTYPE_CLIENT_JOB_EXECUTION_STATUS = 10416, // deprecated + MTYPE_CLIENT_REQUEST_JOB_INFORMATION = 10417, // deprecated MTYPE_CLIENT_DATASHARD_SET_CONFIG = 10418, // deprecated - MTYPE_CLIENT_DESTROY_JOB = 10419, // deprecated + MTYPE_CLIENT_DESTROY_JOB = 10419, // deprecated MTYPE_CLIENT_COMPACTION_BROKER_SET_CONFIG = 10420, // deprecated MTYPE_CLIENT_OLD_HIVE_CREATE_TABLET = 10421, // deprecated MTYPE_CLIENT_HIVE_CREATE_TABLET_RESULT = 10422, // deprecated @@ -51,7 +51,7 @@ enum { MTYPE_CLIENT_CREATE_TABLET = 10437, MTYPE_CLIENT_LOAD_REQUEST = 10438, MTYPE_CLIENT_LOAD_RESPONSE = 10439, - MTYPE_CLIENT_DIRECT_REQUEST_JOB_EXECUTION_STATUS = 10440, // deprecated + MTYPE_CLIENT_DIRECT_REQUEST_JOB_EXECUTION_STATUS = 10440, // deprecated MTYPE_CLIENT_PERSQUEUE = 10441, MTYPE_CLIENT_DB_SCHEMA = 10443, MTYPE_CLIENT_DB_OPERATION = 10444, |