diff options
author | innokentii <innokentii@yandex-team.com> | 2023-04-12 10:39:17 +0300 |
---|---|---|
committer | innokentii <innokentii@yandex-team.com> | 2023-04-12 10:39:17 +0300 |
commit | ab4f3739bf6cc8c2d24e96cf089b347012612f8e (patch) | |
tree | 1630623d94f3b600311555fa482f6b1208fb0d4f | |
parent | afc1c6677a837d9edc09358990e2ac9b7394636d (diff) | |
download | ydb-ab4f3739bf6cc8c2d24e96cf089b347012612f8e.tar.gz |
Revert commit rXXXXXX
34 files changed, 822 insertions, 1458 deletions
diff --git a/ydb/core/cms/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/CMakeLists.darwin-x86_64.txt index cf839e87ff..6bcec6f550 100644 --- a/ydb/core/cms/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/cms/CMakeLists.darwin-x86_64.txt @@ -117,15 +117,14 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -155,7 +154,6 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -171,7 +169,6 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js - cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -201,7 +198,6 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js - cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/CMakeLists.linux-aarch64.txt b/ydb/core/cms/CMakeLists.linux-aarch64.txt index e1dcbee662..1c7e6bbe64 100644 --- a/ydb/core/cms/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/CMakeLists.linux-aarch64.txt @@ -119,15 +119,14 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -157,7 +156,6 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -173,7 +171,6 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js - cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -203,7 +200,6 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js - cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/CMakeLists.linux-x86_64.txt b/ydb/core/cms/CMakeLists.linux-x86_64.txt index e1dcbee662..1c7e6bbe64 100644 --- a/ydb/core/cms/CMakeLists.linux-x86_64.txt +++ b/ydb/core/cms/CMakeLists.linux-x86_64.txt @@ -119,15 +119,14 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -157,7 +156,6 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -173,7 +171,6 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js - cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -203,7 +200,6 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js - cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/CMakeLists.windows-x86_64.txt b/ydb/core/cms/CMakeLists.windows-x86_64.txt index cf839e87ff..6bcec6f550 100644 --- a/ydb/core/cms/CMakeLists.windows-x86_64.txt +++ b/ydb/core/cms/CMakeLists.windows-x86_64.txt @@ -117,15 +117,14 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -155,7 +154,6 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js - ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -171,7 +169,6 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js - cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -201,7 +198,6 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js - cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/cms.cpp b/ydb/core/cms/cms.cpp index 8eff72d580..a5beb840e9 100644 --- a/ydb/core/cms/cms.cpp +++ b/ydb/core/cms/cms.cpp @@ -84,7 +84,10 @@ void TCms::ProcessInitQueue(const TActorContext &ctx) void TCms::SubscribeForConfig(const TActorContext &ctx) { - NConsole::SubscribeViaConfigDispatcher(ctx, {(ui32)NKikimrConsole::TConfigItem::CmsConfigItem}, ctx.SelfID); + ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), + {(ui32)NKikimrConsole::TConfigItem::CmsConfigItem}, + "", + ctx.SelfID)); } void TCms::AdjustInfo(TClusterInfoPtr &info, const TActorContext &ctx) const @@ -976,8 +979,6 @@ void TCms::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::CMS, "TCms::Cleanup"); - NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); - if (State->Sentinel) ctx.Send(State->Sentinel, new TEvents::TEvPoisonPill); } diff --git a/ydb/core/cms/cms_impl.h b/ydb/core/cms/cms_impl.h index e0390d5060..bade5e96de 100644 --- a/ydb/core/cms/cms_impl.h +++ b/ydb/core/cms/cms_impl.h @@ -11,7 +11,6 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/cms/console/console.h> -#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/protos/counters_cms.pb.h> #include <ydb/core/tablet/tablet_counters_protobuf.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -203,8 +202,6 @@ private: TEvCms::TEvManageNotificationResponse>)); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { @@ -252,8 +249,6 @@ private: HFunc(TEvTabletPipe::TEvClientConnected, Handle); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { diff --git a/ydb/core/cms/cms_state.h b/ydb/core/cms/cms_state.h index 9aaecf4f97..1c604f3445 100644 --- a/ydb/core/cms/cms_state.h +++ b/ydb/core/cms/cms_state.h @@ -41,9 +41,6 @@ struct TCmsState : public TAtomicRefCount<TCmsState> { // CMS config. TCmsConfig Config; - // CMS config proto cache - NKikimrCms::TCmsConfig ConfigProto; - // Cluster info. It's not initialized on state creation. // Updated by event from info collector by rewritting // pointer. Therefore pointer shouldnt be preserved diff --git a/ydb/core/cms/cms_tx_load_state.cpp b/ydb/core/cms/cms_tx_load_state.cpp index b4f416e87f..eee95a79c5 100644 --- a/ydb/core/cms/cms_tx_load_state.cpp +++ b/ydb/core/cms/cms_tx_load_state.cpp @@ -57,7 +57,6 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::CMS, "Using default config"); } - state->ConfigProto = config; state->Config.Deserialize(config); if (!logRowset.EndOfSet()) diff --git a/ydb/core/cms/cms_tx_update_config.cpp b/ydb/core/cms/cms_tx_update_config.cpp index ebd4ae51bb..a1d7aa6150 100644 --- a/ydb/core/cms/cms_tx_update_config.cpp +++ b/ydb/core/cms/cms_tx_update_config.cpp @@ -9,10 +9,12 @@ public: TTxUpdateConfig( TCms *self, const NKikimrCms::TCmsConfig &config, - TAutoPtr<IEventHandle> response) + TAutoPtr<IEventHandle> response, + ui64 subscriptionId = 0) : TBase(self) , Config(config) , Response(response) + , SubscriptionId(subscriptionId) , Modify(false) { } @@ -22,14 +24,19 @@ public: bool Execute(TTransactionContext &txc, const TActorContext &ctx) override { LOG_DEBUG_S(ctx, NKikimrServices::CMS, "TTxUpdateConfig Execute"); - if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->State->ConfigProto)) { - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Param>().Key(1) - .Update<Schema::Param::Config>(Config); - - Modify = true; + if (SubscriptionId != Self->ConfigSubscriptionId) { + LOG_ERROR_S(ctx, NKikimrServices::CMS, + "Config subscription id mismatch (" << SubscriptionId + << " vs expected " << Self->ConfigSubscriptionId << ")"); + return true; } + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::Param>().Key(1) + .Update<Schema::Param::Config>(Config); + + Modify = true; + return true; } @@ -37,14 +44,13 @@ public: LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxUpdateConfig Complete"); if (Modify) { - Self->State->ConfigProto = Config; Self->State->Config.Deserialize(Config); LOG_DEBUG_S(ctx, NKikimrServices::CMS, "Updated config: " << Config.ShortDebugString()); - } - ctx.Send(Response.Release()); + ctx.Send(Response.Release()); + } if (Self->State->Config.SentinelConfig.Enable) { if (!Self->State->Sentinel) { @@ -61,6 +67,7 @@ public: private: NKikimrCms::TCmsConfig Config; TAutoPtr<IEventHandle> Response; + ui64 SubscriptionId; bool Modify; }; @@ -68,10 +75,12 @@ ITransaction *TCms::CreateTxUpdateConfig(TEvConsole::TEvConfigNotificationReques auto &rec = ev->Get()->Record; auto response = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(); + response->Record.SetSubscriptionId(rec.GetSubscriptionId()); response->Record.MutableConfigId()->CopyFrom(rec.GetConfigId()); return new TTxUpdateConfig(this, rec.GetConfig().GetCmsConfig(), - new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie) + new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie), + rec.GetSubscriptionId() ); } @@ -79,7 +88,8 @@ ITransaction *TCms::CreateTxUpdateConfig(TEvCms::TEvSetConfigRequest::TPtr &ev) TAutoPtr<TEvCms::TEvSetConfigResponse> response = new TEvCms::TEvSetConfigResponse; response->Record.MutableStatus()->SetCode(NKikimrCms::TStatus::OK); return new TTxUpdateConfig(this, ev->Get()->Record.GetConfig(), - new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie) + new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie), + ConfigSubscriptionId ); } diff --git a/ydb/core/cms/console/config_helpers.cpp b/ydb/core/cms/console/config_helpers.cpp index 91d3ca1e71..a66dfe3b37 100644 --- a/ydb/core/cms/console/config_helpers.cpp +++ b/ydb/core/cms/console/config_helpers.cpp @@ -1,5 +1,4 @@ #include "config_helpers.h" -#include "configs_dispatcher.h" #include "console.h" #include "util.h" @@ -447,25 +446,4 @@ IActor *CreateSubscriptionEraser(ui64 subscriptionId, return new TConfigHelper(subscriptionId, owner, cookie); } -void SubscribeViaConfigDispatcher(const TActorContext &ctx, - const TVector<ui32> &configItemKinds, - TActorId owner, - ui64 cookie) -{ - ctx.Send( - MakeConfigsDispatcherID(ctx.SelfID.NodeId()), - new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(configItemKinds, owner), - cookie); -} - -void UnsubscribeViaConfigDispatcher(const TActorContext &ctx, - TActorId owner, - ui64 cookie) -{ - ctx.Send( - MakeConfigsDispatcherID(ctx.SelfID.NodeId()), - new TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest(owner), - cookie); -} - } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/config_helpers.h b/ydb/core/cms/console/config_helpers.h index e02a7bea79..0ff594779a 100644 --- a/ydb/core/cms/console/config_helpers.h +++ b/ydb/core/cms/console/config_helpers.h @@ -71,20 +71,6 @@ IActor *CreateConfigSubscriber(TActorId serviceId, ui64 cookie = 0); /** - * These functions will make subscription through configs dispatcher - * Those subscriptions handle both yaml and non-yaml configs (not in same subscriprion) - * handle all deduplication, and reconnects - * internally new configs dispatcher uses InMemorySubscriprion's - */ -void SubscribeViaConfigDispatcher(const TActorContext &ctx, - const TVector<ui32> &configItemKinds, - TActorId owner, - ui64 cookie = 0); -void UnsubscribeViaConfigDispatcher(const TActorContext &ctx, - TActorId owner, - ui64 cookie = 0); - -/** * Subscription eraser is used to remove config subscriptions by ID. If owner is * specified then TEvConsole::TEvRemoveConfigSubscriptionRepsonse event is * forwared to it. diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 75748b1e88..770fd56c18 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -1,20 +1,14 @@ #include "config_helpers.h" #include "config_index.h" #include "configs_dispatcher.h" -#include "console_configs_subscriber.h" #include "console.h" #include "http.h" -#include "util.h" -#include <ydb/core/cms/console/yaml_config/yaml_config.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/mon/mon.h> #include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/mon.h> -#include <library/cpp/actors/interconnect/interconnect.h> -#include <library/cpp/json/json_reader.h> #include <util/generic/bitmap.h> #include <util/generic/ptr.h> @@ -33,45 +27,17 @@ namespace NKikimr::NConsole { -const THashSet<ui32> DYNAMIC_KINDS({ - (ui32)NKikimrConsole::TConfigItem::ActorSystemConfigItem, - (ui32)NKikimrConsole::TConfigItem::BootstrapConfigItem, - (ui32)NKikimrConsole::TConfigItem::CmsConfigItem, - (ui32)NKikimrConsole::TConfigItem::CompactionConfigItem, - (ui32)NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem, - (ui32)NKikimrConsole::TConfigItem::FeatureFlagsItem, - (ui32)NKikimrConsole::TConfigItem::HiveConfigItem, - (ui32)NKikimrConsole::TConfigItem::ImmediateControlsConfigItem, - (ui32)NKikimrConsole::TConfigItem::LogConfigItem, - (ui32)NKikimrConsole::TConfigItem::MonitoringConfigItem, - (ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, - (ui32)NKikimrConsole::TConfigItem::NodeBrokerConfigItem, - (ui32)NKikimrConsole::TConfigItem::SchemeShardConfigItem, - (ui32)NKikimrConsole::TConfigItem::SharedCacheConfigItem, - (ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem, - (ui32)NKikimrConsole::TConfigItem::TableServiceConfigItem, - (ui32)NKikimrConsole::TConfigItem::TenantPoolConfigItem, - (ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem, -}); - -const THashSet<ui32> NON_YAML_KINDS({ - (ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, -}); +namespace { class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> { private: using TBase = TActorBootstrapped<TConfigsDispatcher>; struct TConfig { - NKikimrConfig::TConfigVersion Version; + TConfigId ConfigId; NKikimrConfig::TAppConfig Config; }; - struct TYamlVersion { - ui64 Version; - TMap<ui64, ui64> VolatileVersions; - }; - /** * Structure to describe configs subscription shared by multiple * dispatcher subscribers. @@ -79,32 +45,20 @@ private: struct TSubscription : public TThrRefBase { using TPtr = TIntrusivePtr<TSubscription>; + // ID of corresponding subscription in CMS. Zero value means + // we haven't received subscription confirmation from CMS yet. + ui64 SubscriptionId = 0; TDynBitMap Kinds; THashSet<TActorId> Subscribers; - - // Set to true for all yaml kinds. - // Some 'legacy' kinds, which is usually managed by some automation e.g. NetClassifierDistributableConfigItem - // Left this field false and consume old console configs - bool Yaml = false; - // Set to true if there were no config update notifications + // processed for this subscription. + bool FirstUpdate = true; // Last config which was delivered to all subscribers. TConfig CurrentConfig; - - // If any yaml config delivered to all subscribers and acknowleged by them - // This field is set to version from this yaml config - std::optional<TYamlVersion> YamlVersion; - // Config update which is currently delivered to subscribers. - THolder<TEvConsole::TEvConfigNotificationRequest> UpdateInProcess = nullptr; - NKikimrConfig::TConfigVersion UpdateInProcessConfigVersion; - ui64 UpdateInProcessCookie; - std::optional<TYamlVersion> UpdateInProcessYamlVersion; - + TEvConsole::TEvConfigNotificationRequest::TPtr UpdateInProcess; // Subscribers who didn't respond yet to the latest config update. THashSet<TActorId> SubscribersToUpdate; - - bool FirstUpdate = false; }; /** @@ -118,7 +72,7 @@ private: TActorId Subscriber; THashSet<TSubscription::TPtr> Subscriptions; - NKikimrConfig::TConfigVersion CurrentConfigVersion; + TConfigId CurrentConfigId; }; public: @@ -134,61 +88,132 @@ public: void EnqueueEvent(TAutoPtr<IEventHandle> &ev); void ProcessEnqueuedEvents(); - void SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber); + TDynBitMap KindsToBitMap(const TVector<ui32> &kinds) const; + TString KindsToString(const TDynBitMap &kinds) const; + TVector<ui32> KindsToVector(const TDynBitMap &kinds) const; - TSubscription::TPtr FindSubscription(const TActorId &subscriber); + /** + * Overwrite specified protobuf fields with values from + * another protobuf. It's assumed that source config doesn't + * have fields not listed in kinds. + */ + void ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds) const; + bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); + + TSubscription::TPtr FindSubscription(ui64 id); TSubscription::TPtr FindSubscription(const TDynBitMap &kinds); TSubscriber::TPtr FindSubscriber(TActorId aid); - void UpdateYamlVersion(const TSubscription::TPtr &kinds) const; + void SendNotificationResponse(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); + + void MaybeSendNotificationResponse(TSubscription::TPtr subscription); + + void CreateSubscriberActor(ui32 kind, bool replace); + void CreateSubscriberActor(const TDynBitMap &kinds, bool replace); + /** + * Send config notification to a subscriber. Called for subscriptions + * having config update being processed. + */ + + void SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber); + /** + * Remove subscriber and all his subscriptions. + */ + void RemoveSubscriber(TSubscriber::TPtr subscriber); + + /** + * Remove subscription from own data and CMS. It should be called + * for confirmed CMS subscriptions which have no more local + * subscribers. + */ + void RemoveSubscription(TSubscription::TPtr subscription); + + /** + * Create subscription for subscriber. If subscription with similar + * config kinds already exists then just re-use it. Otherwise + * create a new one. If existing subscription has some config received + * then deliver it to the new subscriber. + */ + void AddSubscription(TActorId subscriber, const TDynBitMap &kinds, bool replace); + + /** + * This is called on start and on tenant change to clean up old config + * subscriptions. It also adds subscription for own config. + */ + void CleanUpSubscriptions(); + /** + * Process successfull subscription registration in CMS. Send + * corresponsing notifications to subscribers. If no more subscribers + * left for this subscription then remove it. + */ + + void ProcessAddedSubscription(TSubscription::TPtr subscription, ui64 id); + + /** + * This method is used to process notifications sent to self. + */ + void ProcessLocalCacheUpdate(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); - NKikimrConfig::TAppConfig ParseYamlProtoConfig(); - void Handle(NMon::TEvHttpInfo::TPtr &ev); - void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev); - void Handle(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev); - void Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev); - void Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); void Handle(TEvConfigsDispatcher::TEvGetConfigRequest::TPtr &ev); void Handle(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest::TPtr &ev); - void Handle(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest::TPtr &ev); + void Handle(TEvConsole::TEvAddConfigSubscriptionResponse::TPtr &ev); + void Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); + void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); + void Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev); + void Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev); + void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev); - STATEFN(StateInit) - { + /** + * Initial state when we just get status from tenant pool to collect assigned + * tenants. It's possible we start before tenant pool and therefore might have + * to retry request. + */ + STATEFN(StateInit) { TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); switch (ev->GetTypeRewrite()) { - // Monitoring page hFuncTraced(NMon::TEvHttpInfo, Handle); - hFuncTraced(TEvInterconnect::TEvNodesInfo, Handle); - // Updates from console - hFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); - hFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); - // Events from clients - hFuncTraced(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest, Handle); - hFuncTraced(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest, Handle); + hFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); + default: EnqueueEvent(ev); break; } } - STATEFN(StateWork) - { + /** + * In this state we remove all old service subscriptions and install a new + * one for own config. + */ + STATEFN(StateConfigure) { + TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); + switch (ev->GetTypeRewrite()) { + hFuncTraced(NMon::TEvHttpInfo, Handle); + hFuncTraced(TEvConsole::TEvReplaceConfigSubscriptionsResponse, Handle); + + default: + EnqueueEvent(ev); + break; + } + } + + /** + * Primary state for subscriptions and notifications processing. + */ + STATEFN(StateWork) { TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); switch (ev->GetTypeRewrite()) { - // Monitoring page hFuncTraced(NMon::TEvHttpInfo, Handle); - hFuncTraced(TEvInterconnect::TEvNodesInfo, Handle); - // Updates from console - hFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); - hFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); - // Events from clients hFuncTraced(TEvConfigsDispatcher::TEvGetConfigRequest, Handle); hFuncTraced(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest, Handle); - hFuncTraced(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest, Handle); + hFuncTraced(TEvConsole::TEvAddConfigSubscriptionResponse, Handle); hFuncTraced(TEvConsole::TEvConfigNotificationResponse, Handle); + hFuncTraced(TEvConsole::TEvConfigNotificationRequest, Handle); + hFuncTraced(TEvConsole::TEvGetNodeConfigResponse, Handle); + hFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); IgnoreFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + default: Y_FAIL("unexpected event type: %" PRIx32 " event: %s", ev->GetTypeRewrite(), ev->ToString().data()); @@ -196,27 +221,27 @@ public: } } - private: TMap<TString, TString> Labels; + TDeque<TAutoPtr<IEventHandle>> EventsQueue; NKikimrConfig::TAppConfig InitialConfig; NKikimrConfig::TAppConfig CurrentConfig; - ui64 NextRequestCookie; - TVector<TActorId> HttpRequests; - TActorId CommonSubscriptionClient; - TDeque<TAutoPtr<IEventHandle>> EventsQueue; - - THashMap<TActorId, TSubscription::TPtr> SubscriptionsBySubscriber; + THashSet<TSubscription::TPtr> Subscriptions; + THashMap<ui64, TSubscription::TPtr> SubscriptionsById; THashMap<TDynBitMap, TSubscription::TPtr> SubscriptionsByKinds; THashMap<TActorId, TSubscriber::TPtr> Subscribers; - TString YamlConfig; - TMap<ui64, TString> VolatileYamlConfigs; - TMap<ui64, size_t> VolatileYamlConfigHashes; - TString ResolvedYamlConfig; - TString ResolvedJsonConfig; - NKikimrConfig::TAppConfig YamlProtoConfig; - bool YamlConfigEnabled = false; + // Messages that had an unknown subscription id at the time they are received + THashMap<ui64, TEvConsole::TEvConfigNotificationRequest::TPtr> OutOfOrderConfigNotifications; + + // Cookies are used to tie CMS requests to kinds they were generated for. + THashMap<ui64, TDynBitMap> RequestCookies; + ui64 NextRequestCookie; + THashSet<TString> CurrentTenants; + + // Structures to process config requests. + THashMap<ui64, THolder<IEventHandle>> ConfigRequests; + THashMap<TDynBitMap, std::shared_ptr<NKikimrConfig::TAppConfig>> ConfigsCache; }; TConfigsDispatcher::TConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels) @@ -237,15 +262,8 @@ void TConfigsDispatcher::Bootstrap() mon->RegisterActorPage(actorsMonPage, "configs_dispatcher", "Configs Dispatcher", false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); } - auto commonClient = CreateConfigsSubscriber( - SelfId(), - TVector<ui32>(DYNAMIC_KINDS.begin(), DYNAMIC_KINDS.end()), - CurrentConfig, - 0, - true); - CommonSubscriptionClient = RegisterWithSameMailbox(commonClient); - Become(&TThis::StateInit); + Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe); } void TConfigsDispatcher::EnqueueEvent(TAutoPtr<IEventHandle> &ev) @@ -264,571 +282,671 @@ void TConfigsDispatcher::ProcessEnqueuedEvents() } } -void TConfigsDispatcher::SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber) +TDynBitMap TConfigsDispatcher::KindsToBitMap(const TVector<ui32> &kinds) const { - Y_VERIFY(subscription->UpdateInProcess); + TDynBitMap result; + for (auto &kind : kinds) + result.Set(kind); + return result; +} - subscription->SubscribersToUpdate.insert(subscriber); +TString TConfigsDispatcher::KindsToString(const TDynBitMap &kinds) const +{ + TStringStream ss; + bool first = true; + Y_FOR_EACH_BIT(kind, kinds) { + ss << (first ? "" : ", ") << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); + first = false; + } + return ss.Str(); +} - auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); - notification->Record.CopyFrom(subscription->UpdateInProcess->Record); +TVector<ui32> TConfigsDispatcher::KindsToVector(const TDynBitMap &kinds) const +{ + TVector<ui32> res; + Y_FOR_EACH_BIT(kind, kinds) { + res.push_back(kind); + } + return res; +} - BLOG_TRACE("Send TEvConsole::TEvConfigNotificationRequest to " << subscriber - << ": " << notification->Record.ShortDebugString()); +void TConfigsDispatcher::ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, + NKikimrConfig::TAppConfig &to, + const TDynBitMap &kinds) const +{ + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); - Send(subscriber, notification.Release(), 0, subscription->UpdateInProcessCookie); + Y_FOR_EACH_BIT(kind, kinds) { + auto *field = desc->FindFieldByNumber(kind); + if (field && reflection->HasField(to, field)) + reflection->ClearField(&to, field); + } + + to.MergeFrom(from); } -TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TDynBitMap &kinds) +bool TConfigsDispatcher::CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs) { - if (auto it = SubscriptionsByKinds.find(kinds); it != SubscriptionsByKinds.end()) - return it->second; - - return nullptr; + TString str1, str2; + Y_PROTOBUF_SUPPRESS_NODISCARD lhs.SerializeToString(&str1); + Y_PROTOBUF_SUPPRESS_NODISCARD rhs.SerializeToString(&str2); + return (str1 == str2); } -TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TActorId &id) +TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(ui64 id) { - if (auto it = SubscriptionsBySubscriber.find(id); it != SubscriptionsBySubscriber.end()) - return it->second; + auto it = SubscriptionsById.find(id); + if (it == SubscriptionsById.end()) + return nullptr; + return it->second; +} - return nullptr; +TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TDynBitMap &kinds) +{ + auto it = SubscriptionsByKinds.find(kinds); + if (it == SubscriptionsByKinds.end()) + return nullptr; + return it->second; } TConfigsDispatcher::TSubscriber::TPtr TConfigsDispatcher::FindSubscriber(TActorId aid) { - if (auto it = Subscribers.find(aid); it != Subscribers.end()) - return it->second; + auto it = Subscribers.find(aid); + if (it == Subscribers.end()) + return nullptr; + return it->second; +} + +void TConfigsDispatcher::SendNotificationResponse(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) +{ + const auto &rec = ev->Get()->Record; + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - return nullptr; + BLOG_TRACE("Send TEvConfigNotificationResponse: " << resp->Record.ShortDebugString()); + + Send(ev->Sender, resp.Release(), 0, ev->Cookie); } -NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() +void TConfigsDispatcher::MaybeSendNotificationResponse(TSubscription::TPtr subscription) { - NKikimrConfig::TAppConfig newYamlProtoConfig = {}; + if (!subscription->UpdateInProcess || !subscription->SubscribersToUpdate.empty()) + return; - NYamlConfig::ResolveAndParseYamlConfig( - YamlConfig, - VolatileYamlConfigs, - Labels, - newYamlProtoConfig, - &ResolvedYamlConfig, - &ResolvedJsonConfig); + auto &rec = subscription->UpdateInProcess->Get()->Record; + subscription->CurrentConfig.Config.Swap(rec.MutableConfig()); + subscription->CurrentConfig.ConfigId.Load(rec.GetConfigId()); - return newYamlProtoConfig; + ReplaceConfigItems(subscription->CurrentConfig.Config, CurrentConfig, subscription->Kinds); + + BLOG_D("Got all confirmations for config update" + << " subscriptionid=" << subscription->SubscriptionId + << " configid=" << TConfigId(rec.GetConfigId()).ToString()); + + SendNotificationResponse(subscription->UpdateInProcess); + + subscription->UpdateInProcess = nullptr; } -void TConfigsDispatcher::Handle(NMon::TEvHttpInfo::TPtr &ev) +void TConfigsDispatcher::CreateSubscriberActor(ui32 kind, bool replace) { - if (HttpRequests.empty()) - Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes); + TDynBitMap kinds; + kinds.Set(kind); + CreateSubscriberActor(kinds, replace); +} - HttpRequests.push_back(ev->Sender); +void TConfigsDispatcher::CreateSubscriberActor(const TDynBitMap &kinds, bool replace) +{ + BLOG_D("Create new subscriber kinds=" << KindsToString(kinds)); + + auto *subscriber = CreateConfigSubscriber(MakeConfigsDispatcherID(SelfId().NodeId()), + KindsToVector(kinds), + SelfId(), + replace, + NextRequestCookie); + Register(subscriber); + RequestCookies[NextRequestCookie] = kinds; + ++NextRequestCookie; } -void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) +void TConfigsDispatcher::SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber) { - Y_UNUSED(ev); - TStringStream str; - str << NMonitoring::HTTPOKHTML; - HTML(str) { - HEAD() { - str << "<link rel='stylesheet' href='../cms/ext/bootstrap.min.css'>" << Endl - << "<script language='javascript' type='text/javascript' src='../cms/ext/jquery.min.js'></script>" << Endl - << "<script language='javascript' type='text/javascript' src='../cms/ext/bootstrap.bundle.min.js'></script>" << Endl - << "<script language='javascript' type='text/javascript'>" << Endl - << "var nodeNames = ["; - - for (auto &node: ev->Get()->Nodes) { - str << "{'nodeName':'" << node.Host << "'}, "; - } + Y_VERIFY(subscription->SubscriptionId); + Y_VERIFY(subscription->UpdateInProcess); - str << "];" << Endl - << "</script>" << Endl - << "<script src='../cms/ext/fuse.min.js'></script>" << Endl - << "<script src='../cms/common.js'></script>" << Endl - << "<script src='../cms/ext/fuzzycomplete.min.js'></script>" << Endl - << "<link rel='stylesheet' href='../cms/ext/fuzzycomplete.min.css'>" << Endl - << "<link rel='stylesheet' href='../cms/cms.css'>" << Endl - << "<script data-main='../cms/configs_dispatcher_main' src='../cms/ext/require.min.js'></script>" << Endl; + subscription->SubscribersToUpdate.insert(subscriber); - } - NHttp::OutputStyles(str); + auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); + notification->Record.CopyFrom(subscription->UpdateInProcess->Get()->Record); - DIV() { - OL_CLASS("breadcrumb") { - LI_CLASS("breadcrumb-item") { - str << "<a href='..' id='host-ref'>YDB Developer UI</a>" << Endl; - } - LI_CLASS("breadcrumb-item") { - str << "<a href='.'>Actors</a>" << Endl; - } - LI_CLASS("breadcrumb-item active") { - str << "Configs Dispatcher" << Endl; - } - } - } + BLOG_TRACE("Send TEvConsole::TEvConfigNotificationRequest to " << subscriber + << ": " << notification->Record.ShortDebugString()); - DIV_CLASS("container") { - DIV_CLASS("navbar navbar-expand-lg navbar-light bg-light") { - DIV_CLASS("navbar-collapse") { - UL_CLASS("navbar-nav mr-auto") { - LI_CLASS("nav-item") { - str << "<a class='nav-link' href=\"../cms?#page=yaml-config\">Console</a>" << Endl; - } - } - FORM_CLASS("form-inline my-2 my-lg-0") { - str << "<input type='text' id='nodePicker' class='form-control mr-sm-2' name='nodes' placeholder='Nodes...'>" << Endl; - str << "<a type='button' class='btn btn-primary my-2 my-sm-0' id='nodesGo'>Go</a>" << Endl; - } - } - } - DIV_CLASS("tab-left") { - COLLAPSED_REF_CONTENT("node-labels", "Node labels") { - PRE() { - for (auto &[key, value] : Labels) { - str << key << " = " << value << Endl; - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("state", "State") { - PRE() { - str << "SelfId: " << SelfId() << Endl; - auto s = CurrentStateFunc(); - str << "State: " << ( s == &TThis::StateWork ? "StateWork" - : s == &TThis::StateInit ? "StateInit" - : "Unknown" ) << Endl; - str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; - str << "Subscriptions: " << Endl; - for (auto &[kinds, subscription] : SubscriptionsByKinds) { - str << "- Kinds: " << KindsToString(kinds) << Endl - << " Subscription: " << Endl - << " Yaml: " << subscription->Yaml << Endl - << " Subscribers: " << Endl; - for (auto &id : subscription->Subscribers) { - str << " - Actor: " << id << Endl; - } - if (subscription->YamlVersion) { - str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; - bool first = true; - for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { - str << (first ? "" : ",") << id << "." << hash; - first = false; - } - str << "]" << Endl; - } else { - str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; - } - str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; - if (subscription->UpdateInProcess) { - str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl - << " SubscribersToUpdate:"; - for (auto &id : subscription->SubscribersToUpdate) { - str << " " << id; - } - str << Endl; - str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl - << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; - if (subscription->UpdateInProcessYamlVersion) { - str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; - } - } - } - str << "Subscribers:" << Endl; - for (auto &[subscriber, _] : SubscriptionsBySubscriber) { - str << "- " << subscriber << Endl; - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { - DIV() { - TAG(TH5) { - str << "Persistent Config" << Endl; - } - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { - str << YamlConfig; - } - } - str << "<hr/>" << Endl; - for (auto &[id, config] : VolatileYamlConfigs) { - DIV() { - TAG(TH5) { - str << "Volatile Config Id: " << id << Endl; - } - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - DIV_CLASS("yaml-config-item") { - str << config; - } - } - } - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { - str << ResolvedYamlConfig; - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { - PRE() { - str << ResolvedJsonConfig << Endl; - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { - NHttp::OutputConfigHTML(str, YamlProtoConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("current-config", "Current config") { - NHttp::OutputConfigHTML(str, CurrentConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("initial-config", "Initial config") { - NHttp::OutputConfigHTML(str, InitialConfig); - } - } + Send(subscriber, notification.Release(), 0, subscription->UpdateInProcess->Cookie); +} + +void TConfigsDispatcher::RemoveSubscriber(TSubscriber::TPtr subscriber) +{ + BLOG_D("Remove subscriber " << subscriber->Subscriber); + + for (auto subscription : subscriber->Subscriptions) { + Y_VERIFY(subscription->Subscribers.contains(subscriber->Subscriber)); + subscription->Subscribers.erase(subscriber->Subscriber); + + if (subscription->UpdateInProcess) { + subscription->SubscribersToUpdate.erase(subscriber->Subscriber); + MaybeSendNotificationResponse(subscription); } - } - for (auto &actor : HttpRequests) { - Send(actor, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + // If there are no more subscribers using this subscription then + // it can be removed. Don't remove subscriptions which are not + // yet confirmed by CMS. + if (subscription->Subscribers.empty() && subscription->SubscriptionId) + RemoveSubscription(subscription); } - HttpRequests.clear(); + Subscribers.erase(subscriber->Subscriber); } -void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev) +void TConfigsDispatcher::RemoveSubscription(TSubscription::TPtr subscription) { - auto &rec = ev->Get()->Record; + Subscriptions.erase(subscription); + SubscriptionsById.erase(subscription->SubscriptionId); + SubscriptionsByKinds.erase(subscription->Kinds); - CurrentConfig = rec.GetConfig(); + BLOG_D("Remove subscription id=" << subscription->SubscriptionId + << " kinds=" << KindsToString(subscription->Kinds)); - const auto& newYamlConfig = rec.GetYamlConfig(); + Register(CreateSubscriptionEraser(subscription->SubscriptionId)); +} - bool isYamlChanged = newYamlConfig != YamlConfig; +void TConfigsDispatcher::AddSubscription(TActorId subscriber, + const TDynBitMap &kinds, + bool replace) +{ + BLOG_D("Add subscription for " << subscriber << " kinds=" << KindsToString(kinds)); - if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { - isYamlChanged = true; - } + // If there is a subscription for required config kinds then + // re-use it for new subscriber. Otherwise create a new one. + auto subscription = FindSubscription(kinds); + if (!subscription) { + subscription = new TSubscription; + subscription->Kinds = kinds; - for (auto &volatileConfig : rec.GetVolatileConfigs()) { - if (auto it = VolatileYamlConfigHashes.find(volatileConfig.GetId()); - it == VolatileYamlConfigHashes.end() || it->second != THash<TString>()(volatileConfig.GetConfig())) { - isYamlChanged = true; - } + Subscriptions.insert(subscription); + SubscriptionsByKinds.emplace(kinds, subscription); + + CreateSubscriberActor(kinds, replace); } + subscription->Subscribers.insert(subscriber); - if (isYamlChanged) { - YamlConfig = newYamlConfig; - VolatileYamlConfigs.clear(); - VolatileYamlConfigHashes.clear(); - for (auto &volatileConfig : rec.GetVolatileConfigs()) { - VolatileYamlConfigs[volatileConfig.GetId()] = volatileConfig.GetConfig(); - VolatileYamlConfigHashes[volatileConfig.GetId()] = THash<TString>()(volatileConfig.GetConfig()); - } + auto s = FindSubscriber(subscriber); + if (!s) { + s = new TSubscriber; + s->Subscriber = subscriber; + Subscribers.emplace(subscriber, s); } + s->Subscriptions.insert(subscription); - NKikimrConfig::TAppConfig newYamlProtoConfig = {}; + // Non-zero subscription ID means there is an active CMS + // subscription and therefore we can respond to the subscriber + // immediately. Otherwise we should wait until CMS + // subscription request is complete. + if (subscription->SubscriptionId) { + Y_VERIFY(!replace); + auto resp = MakeHolder<TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse>(); - bool yamlConfigTurnedOff = false; + BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " + << subscriber); - if (!YamlConfig.empty() && isYamlChanged) { - newYamlProtoConfig = ParseYamlProtoConfig(); - bool wasYamlConfigEnabled = YamlConfigEnabled; - YamlConfigEnabled = newYamlProtoConfig.HasYamlConfigEnabled() && newYamlProtoConfig.GetYamlConfigEnabled(); - yamlConfigTurnedOff = wasYamlConfigEnabled && !YamlConfigEnabled; - } else { - newYamlProtoConfig = YamlProtoConfig; + Send(subscriber, resp.Release()); } - std::swap(YamlProtoConfig, newYamlProtoConfig); - - THashSet<ui32> affectedKinds; - for (const auto& kind : ev->Get()->Record.GetAffectedKinds()) { - affectedKinds.insert(kind); + // If there is an ongoing config update then include new subscriber into + // the process. + if (subscription->UpdateInProcess) { + Y_VERIFY(!replace); + SendUpdateToSubscriber(subscription, subscriber); + } else if (!subscription->FirstUpdate) { + // If subscription already had an update notification then send corresponding + // notification to the subscriber using current config. + Y_VERIFY(!replace); + Y_VERIFY(subscription->SubscriptionId); + auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); + notification->Record.SetSubscriptionId(subscription->SubscriptionId); + subscription->CurrentConfig.ConfigId.Serialize(*notification->Record.MutableConfigId()); + notification->Record.MutableConfig()->CopyFrom(subscription->CurrentConfig.Config); + + BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to "<< subscriber); + + Send(subscriber, notification.Release()); } +} - for (auto &[kinds, subscription] : SubscriptionsByKinds) { - if (subscription->UpdateInProcess) { - subscription->UpdateInProcess = nullptr; - subscription->SubscribersToUpdate.clear(); - } - - NKikimrConfig::TAppConfig trunc; +void TConfigsDispatcher::CleanUpSubscriptions() +{ + BLOG_N("Cleaning up all current subscriptions"); + + // If there are active subscriptions then we should + // mark them as removed by reseting their IDs + // and configs. + for (auto &subscription : Subscriptions) { + subscription->SubscriptionId = 0; + subscription->SubscribersToUpdate.clear(); + subscription->UpdateInProcess = nullptr; + subscription->FirstUpdate = true; + } + SubscriptionsById.clear(); + RequestCookies.clear(); + OutOfOrderConfigNotifications.clear(); - bool hasAffectedKinds = false; + // We should invalidate configs cache to avoid its usage until + // updated configs are received. + ConfigsCache.clear(); - if (subscription->Yaml && YamlConfigEnabled) { - ReplaceConfigItems(YamlProtoConfig, trunc, subscription->Kinds); - } else { - Y_FOR_EACH_BIT(kind, kinds) { - if (affectedKinds.contains(kind)) { - hasAffectedKinds = true; - } - } + TDynBitMap kinds; + kinds.Set(NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem); + auto subscription = FindSubscription(kinds); + if (subscription) { + CreateSubscriberActor(kinds, true); + } else { + AddSubscription(SelfId(), kinds, true); + } - // we try resend all configs if yaml config was turned off - if (!hasAffectedKinds && !yamlConfigTurnedOff) { - continue; - } + Become(&TThis::StateConfigure); +} - ReplaceConfigItems(ev->Get()->Record.GetConfig(), trunc, kinds); - } +void TConfigsDispatcher::ProcessAddedSubscription(TSubscription::TPtr subscription, ui64 id) +{ + BLOG_N("Confirmed CMS subscription" + << " kinds=" << KindsToString(subscription->Kinds) + << " id=" << id); - subscription->FirstUpdate = true; + Y_VERIFY(!subscription->SubscriptionId); + subscription->SubscriptionId = id; + SubscriptionsById[id] = subscription; - if (hasAffectedKinds || !CompareConfigs(subscription->CurrentConfig.Config, trunc)) - { - subscription->UpdateInProcess = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); - subscription->UpdateInProcess->Record.MutableConfig()->CopyFrom(trunc); - Y_FOR_EACH_BIT(kind, kinds) { - subscription->UpdateInProcess->Record.AddItemKinds(kind); - } - subscription->UpdateInProcessCookie = ++NextRequestCookie; - subscription->UpdateInProcessConfigVersion = FilterVersion(ev->Get()->Record.GetConfig().GetVersion(), kinds); + for (auto &subscriber : subscription->Subscribers) { + BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << subscriber); - if (YamlConfigEnabled) { - UpdateYamlVersion(subscription); - } + Send(subscriber, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + } - for (auto &subscriber : subscription->Subscribers) { - auto k = kinds; - BLOG_TRACE("Sending for kinds: " << KindsToString(k)); - SendUpdateToSubscriber(subscription, subscriber); - } - } else if (YamlConfigEnabled && subscription->Yaml) { - UpdateYamlVersion(subscription); - } else if (!YamlConfigEnabled) { - subscription->YamlVersion = std::nullopt; - } + auto it = OutOfOrderConfigNotifications.find(id); + if (it != OutOfOrderConfigNotifications.end()) { + auto ev = std::move(it->second); + OutOfOrderConfigNotifications.erase(it); + Handle(ev); } - - if (CurrentStateFunc() == &TThis::StateInit) { - Become(&TThis::StateWork); - ProcessEnqueuedEvents(); + + while (!(SubscriptionsById.size() < Subscriptions.size()) && !OutOfOrderConfigNotifications.empty()) { + auto it = OutOfOrderConfigNotifications.begin(); + auto ev = std::move(it->second); + OutOfOrderConfigNotifications.erase(it); + SendNotificationResponse(ev); } + + // Probably there are no more subscribers for this subscription. + // In that case it should be removed. + if (subscription->Subscribers.empty()) + RemoveSubscription(subscription); } -void TConfigsDispatcher::UpdateYamlVersion(const TSubscription::TPtr &subscription) const +void TConfigsDispatcher::ProcessLocalCacheUpdate(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) { - TYamlVersion yamlVersion; - yamlVersion.Version = NYamlConfig::GetVersion(YamlConfig); - for (auto &[id, hash] : VolatileYamlConfigHashes) { - yamlVersion.VolatileVersions[id] = hash; + auto &rec = ev->Get()->Record; + BLOG_D("Got new config: " << rec.ShortDebugString()); + + auto subscription = FindSubscription(rec.GetSubscriptionId()); + if (!subscription) { + BLOG_ERROR("Cannot find subscription for configs cache update subscriptionid=" << rec.GetSubscriptionId()); + return; } - subscription->UpdateInProcessYamlVersion = yamlVersion; + + BLOG_D("Update local cache for kinds=" << KindsToString(subscription->Kinds) + << " config='" << rec.GetConfig().ShortDebugString() << "'"); + + ConfigsCache[subscription->Kinds].reset(new NKikimrConfig::TAppConfig(rec.GetConfig())); + + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); + + BLOG_TRACE("Send TEvConsole::TEvConfigNotificationResponse to self: " << resp->Record.ShortDebugString()); + + Send(ev->Sender, resp.Release(), 0, ev->Cookie); } -void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev) +void TConfigsDispatcher::Handle(NMon::TEvHttpInfo::TPtr &ev) { - // The only reason we can get this response is ambiguous domain - // So it is okay to fail here - Y_FAIL("Can't start Configs Dispatcher: %s", - ev->Get()->Record.GetReason().c_str()); + TStringStream str; + str << NMonitoring::HTTPOKHTML; + HTML(str) { + NHttp::OutputStaticPart(str); + PRE() { + str << "Maintained tenant: " << JoinSeq(", ", CurrentTenants); + } + DIV_CLASS("tab-left") { + COLLAPSED_REF_CONTENT("node-labels", "Node labels") { + PRE() { + for (auto& [key, value] : Labels) { + str << key << " = " << value << Endl; + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("current-config", "Current config") { + NHttp::OutputConfigHTML(str, CurrentConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-config", "Initial config") { + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("subscriptions", "Subscriptions") { + PRE() { + for (auto subscription : Subscriptions) { + str << " - ID: " << subscription->SubscriptionId << Endl + << " Kinds: " << KindsToString(subscription->Kinds) << Endl + << " Subscribers:"; + for (auto &id : subscription->Subscribers) + str << " " << id; + str << Endl + << " FirstUpdate: " << subscription->FirstUpdate << Endl + << " CurrentConfigId: " << subscription->CurrentConfig.ConfigId.ToString() << Endl + << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; + if (subscription->UpdateInProcess) { + str << " UpdateInProcess: " << subscription->UpdateInProcess->Get()->Record.ShortDebugString() << Endl + << " SubscribersToUpdate:"; + for (auto &id : subscription->SubscribersToUpdate) + str << " " << id; + str << Endl; + } + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("subscribers", "Subscribers") { + PRE() { + for (auto &pr : Subscribers) { + str << " - Subscriber: " << pr.second->Subscriber << Endl + << " Subscriptions:"; + for (auto subscription : pr.second->Subscriptions) + str << " " << subscription->SubscriptionId; + str << Endl + << " CurrentConfigId: " << pr.second->CurrentConfigId.ToString() << Endl; + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("cache", "Configs cache") { + DIV_CLASS("tab-left") { + ui32 id = 1; + for (auto &pr : ConfigsCache) { + TString kinds = KindsToString(pr.first); + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("cache-" + ToString(id++), kinds) { + DIV_CLASS("tab-left") { + PRE() { + str << pr.second->DebugString() << Endl; + } + } + } + } + } + } + } + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); } void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvGetConfigRequest::TPtr &ev) { - auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); + auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); - for (auto kind : ev->Get()->ConfigItemKinds) { - if (!DYNAMIC_KINDS.contains(kind)) { - TStringStream sstr; - sstr << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); - Y_FAIL("unexpected kind in GetConfigRequest: %s", sstr.Str().data()); + if (ev->Get()->Cache && !kinds.Empty()) { + auto subscription = FindSubscription(kinds); + if (!subscription || !subscription->Subscribers.contains(SelfId())) { + BLOG_D("Add subscription for local cache kinds=" << KindsToString(kinds)); + AddSubscription(SelfId(), kinds, false); } } - auto trunc = std::make_shared<NKikimrConfig::TAppConfig>(); - ReplaceConfigItems(CurrentConfig, *trunc, KindsToBitMap(ev->Get()->ConfigItemKinds)); - resp->Config = trunc; + if (ConfigsCache.contains(kinds)) { + auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); + resp->Config = ConfigsCache.at(kinds); - BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" - " to " << ev->Sender << ": " << resp->Config->ShortDebugString()); + BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" + " to " << ev->Sender << ": " << resp->Config->ShortDebugString()); - Send(ev->Sender, std::move(resp), 0, ev->Cookie); + Send(ev->Sender, std::move(resp), 0, ev->Cookie); + } else { + Register(CreateNodeConfigCourier(ev->Get()->ConfigItemKinds, SelfId(), NextRequestCookie)); + ConfigRequests[NextRequestCookie++] = THolder<IEventHandle>(ev.Release()); + } } void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest::TPtr &ev) { - bool yamlKinds = false; - bool nonYamlKinds = false; - for (auto kind : ev->Get()->ConfigItemKinds) { - if (!DYNAMIC_KINDS.contains(kind)) { - TStringStream sstr; - sstr << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); - Y_FAIL("unexpected kind in SetConfigSubscriptionRequest: %s", sstr.Str().data()); - } + auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); + auto subscriber = FindSubscriber(ev->Sender); - if (NON_YAML_KINDS.contains(kind)) { - nonYamlKinds = true; - } else { - yamlKinds = true; + if (subscriber) { + Y_VERIFY(subscriber->Subscriptions.size() == 1); + auto subscription = *subscriber->Subscriptions.begin(); + + if (subscription->Kinds == kinds) { + BLOG_D("Nothing to change for " << subscriber->Subscriber); + BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << subscriber->Subscriber); + + Send(subscriber->Subscriber, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + return; } - } - if (yamlKinds && nonYamlKinds) { - Y_FAIL("both yaml and non yaml kinds in SetConfigSubscriptionRequest"); + // something changed so refresh subscription + RemoveSubscriber(subscriber); } - auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); - auto subscriberActor = ev->Get()->Subscriber ? ev->Get()->Subscriber : ev->Sender; - - auto subscription = FindSubscription(kinds); - if (!subscription) { - subscription = new TSubscription; - subscription->Kinds = kinds; - subscription->Yaml = yamlKinds; + if (!kinds.Empty()) { + AddSubscription(ev->Sender, kinds, false); + } else { + BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << ev->Sender); - SubscriptionsByKinds.emplace(kinds, subscription); - } - subscription->Subscribers.insert(subscriberActor); - SubscriptionsBySubscriber.emplace(subscriberActor, subscription); - - auto subscriber = FindSubscriber(subscriberActor); - if (!subscriber) { - subscriber = new TSubscriber; - subscriber->Subscriber = subscriberActor; - Subscribers.emplace(subscriberActor, subscriber); - } - subscriber->Subscriptions.insert(subscription); - - // We don't care about versions and kinds here - Send(ev->Sender, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - - if (subscription->FirstUpdate) { - // first time we send even empty config - if (!subscription->UpdateInProcess) { - subscription->UpdateInProcess = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); - NKikimrConfig::TAppConfig trunc; - if (YamlConfigEnabled) { - ReplaceConfigItems(YamlProtoConfig, trunc, kinds); - } else { - ReplaceConfigItems(CurrentConfig, trunc, kinds); - } - subscription->UpdateInProcess->Record.MutableConfig()->CopyFrom(trunc); - Y_FOR_EACH_BIT(kind, kinds) { - subscription->UpdateInProcess->Record.AddItemKinds(kind); - } - subscription->UpdateInProcessCookie = ++NextRequestCookie; - subscription->UpdateInProcessConfigVersion = FilterVersion(CurrentConfig.GetVersion(), kinds); - } - BLOG_TRACE("Sending for kinds: " << KindsToString(kinds)); - SendUpdateToSubscriber(subscription, subscriber->Subscriber); + Send(ev->Sender, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); } } -void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest::TPtr &ev) +void TConfigsDispatcher::Handle(TEvConsole::TEvAddConfigSubscriptionResponse::TPtr &ev) { - auto subscriberActor = ev->Get()->Subscriber ? ev->Get()->Subscriber : ev->Sender; - auto subscriber = FindSubscriber(subscriberActor); - if (!subscriber) { + auto it = RequestCookies.find(ev->Cookie); + if (it == RequestCookies.end()) { + BLOG_I("Cookie mismatch for TEvAddConfigSubscriptionResponse"); return; } + auto kinds = it->second; + RequestCookies.erase(it); - for (auto &subscription : subscriber->Subscriptions) { - subscription->SubscribersToUpdate.erase(subscriberActor); - if (subscription->SubscribersToUpdate.empty()) { - if (subscription->UpdateInProcess) { - subscription->CurrentConfig.Version = subscription->UpdateInProcessConfigVersion; - subscription->CurrentConfig.Config = subscription->UpdateInProcess->Record.GetConfig(); - } - subscription->YamlVersion = subscription->UpdateInProcessYamlVersion; - subscription->UpdateInProcessYamlVersion = std::nullopt; - subscription->UpdateInProcess = nullptr; - } - - subscription->Subscribers.erase(subscriberActor); - if (subscription->Subscribers.empty()) { - SubscriptionsByKinds.erase(subscription->Kinds); - } + auto &rec = ev->Get()->Record; + if (rec.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CONFIGS_DISPATCHER, + "Cannot get config subscription for " << KindsToString(kinds) + << " code=" << rec.GetStatus().GetCode() + << " reason= " << rec.GetStatus().GetReason()); + CreateSubscriberActor(kinds, false); + return; } - Subscribers.erase(subscriberActor); - SubscriptionsBySubscriber.erase(subscriberActor); + auto subscription = FindSubscription(kinds); + Y_VERIFY(subscription); - Send(ev->Sender, new TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); + ProcessAddedSubscription(subscription, rec.GetSubscriptionId()); } - void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev) { auto rec = ev->Get()->Record; - auto subscription = FindSubscription(ev->Sender); + auto subscription = FindSubscription(rec.GetSubscriptionId()); // Probably subscription was cleared up due to tenant's change. if (!subscription) { - BLOG_ERROR("Got notification response for unknown subscription " << ev->Sender); + BLOG_I("Got notification response for unknown subscription " << rec.GetSubscriptionId()); return; } if (!subscription->UpdateInProcess) { - BLOG_D("Notification was ignored for subscription " << ev->Sender); + BLOG_D("Notification was ignored for subscription " + << rec.GetSubscriptionId()); + return; + } + + if (ev->Cookie != subscription->UpdateInProcess->Cookie) { + BLOG_ERROR("Notification cookie mismatch for subscription " << rec.GetSubscriptionId()); return; } - if (ev->Cookie != subscription->UpdateInProcessCookie) { - BLOG_ERROR("Notification cookie mismatch for subscription " << ev->Sender << " " << ev->Cookie << " != " << subscription->UpdateInProcessCookie); - // TODO fix clients + TConfigId id1(subscription->UpdateInProcess->Get()->Record.GetConfigId()); + TConfigId id2(rec.GetConfigId()); + // This might be outdated notification response. + if (id1 != id2) { + BLOG_I("Config id mismatch in notification response for subscription " << rec.GetSubscriptionId()); return; } if (!subscription->SubscribersToUpdate.contains(ev->Sender)) { - BLOG_ERROR("Notification from unexpected subscriber for subscription " << ev->Sender); + BLOG_ERROR("Notification from unexpected subscriber for subscription " << rec.GetSubscriptionId()); return; } - Subscribers.at(ev->Sender)->CurrentConfigVersion = subscription->UpdateInProcessConfigVersion; + Subscribers.at(ev->Sender)->CurrentConfigId = id1; // If all subscribers responded then send response to CMS. subscription->SubscribersToUpdate.erase(ev->Sender); + MaybeSendNotificationResponse(subscription); +} + +void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) +{ + // Process local update sent by own local subscription. + if (ev->Sender == SelfId()) { + ProcessLocalCacheUpdate(ev); + return; + } - if (subscription->SubscribersToUpdate.empty()) { - subscription->CurrentConfig.Config = subscription->UpdateInProcess->Record.GetConfig(); - subscription->CurrentConfig.Version = subscription->UpdateInProcessConfigVersion; - subscription->YamlVersion = subscription->UpdateInProcessYamlVersion; - subscription->UpdateInProcessYamlVersion = std::nullopt; + auto &rec = ev->Get()->Record; + auto subscription = FindSubscription(rec.GetSubscriptionId()); + if (!subscription) { + BLOG_W("Got notification for unknown subscription id=" << rec.GetSubscriptionId()); + + if (SubscriptionsById.size() < Subscriptions.size()) { + // There are subscriptions that don't have an id yet + // Delay processing until we know ids of all subscriptions + auto &prev = OutOfOrderConfigNotifications[rec.GetSubscriptionId()]; + if (prev) { + SendNotificationResponse(prev); + } + prev = ev; + return; + } + + SendNotificationResponse(ev); + return; + } + + if (subscription->UpdateInProcess) { + BLOG_D("Drop previous unfinished notification for subscription id=" + << subscription->SubscriptionId); subscription->UpdateInProcess = nullptr; + subscription->SubscribersToUpdate.clear(); + } + + subscription->UpdateInProcess = std::move(ev); + + /** + * Avoid notifications in case only config id changed and + * config body is equal to currently used one. + */ + if (subscription->FirstUpdate || !CompareConfigs(subscription->CurrentConfig.Config, rec.GetConfig())) { + for (auto &subscriber : subscription->Subscribers) + SendUpdateToSubscriber(subscription, subscriber); + } else { + MaybeSendNotificationResponse(subscription); + } + + subscription->FirstUpdate = false; +} + +void TConfigsDispatcher::Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev) +{ + auto it = ConfigRequests.find(ev->Cookie); + + if (it == ConfigRequests.end()) { + BLOG_ERROR("Node config response for unknown request cookie=" << ev->Cookie); + return; + } + + auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); + resp->Config.reset(new NKikimrConfig::TAppConfig(ev->Get()->Record.GetConfig())); + + BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" + " to " << ev->Sender + << ": " << resp->Config->ShortDebugString()); + + Send(it->second->Sender, resp.Release(), 0, it->second->Cookie); + + ConfigRequests.erase(it); +} + +void TConfigsDispatcher::Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev) +{ + auto it = RequestCookies.find(ev->Cookie); + if (it == RequestCookies.end()) { + BLOG_ERROR("Cookie mismatch for TEvReplaceConfigSubscriptionsResponse"); + return; + } + + auto &rec = ev->Get()->Record; + if (rec.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CONFIGS_DISPATCHER, + "Cannot initialize subscription: " << rec.GetStatus().GetReason()); + CleanUpSubscriptions(); + return; + } + + auto subscription = FindSubscription(it->second); + Y_VERIFY(subscription); + ProcessAddedSubscription(subscription, rec.GetSubscriptionId()); + + // Register other subscriptions in CMS. + for (auto subscription : Subscriptions) + if (!subscription->SubscriptionId) + CreateSubscriberActor(subscription->Kinds, false); + + Become(&TThis::StateWork); + ProcessEnqueuedEvents(); +} + +void TConfigsDispatcher::Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) +{ + auto &rec = ev->Get()->Record; + + THashSet<TString> tenants; + for (auto &slot : rec.GetSlots()) + tenants.insert(slot.GetAssignedTenant()); + + // If we are in initial state then subscriptions set is empty + // and we should start initialization. Otherwise we should + // re-initialize if tenants set changed. + if (CurrentTenants != tenants || Subscriptions.empty()) { + CurrentTenants = tenants; + + BLOG_N("Update list of assigned tenants: " << JoinSeq(", ", CurrentTenants)); + + CleanUpSubscriptions(); } } - -IActor *CreateConfigsDispatcher( - const NKikimrConfig::TAppConfig &config, - const TMap<TString, TString> &labels) + +} // anonymous namespace + +IActor *CreateConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels) { return new TConfigsDispatcher(config, labels); } diff --git a/ydb/core/cms/console/configs_dispatcher.h b/ydb/core/cms/console/configs_dispatcher.h index 0c470db68c..a1511eecd6 100644 --- a/ydb/core/cms/console/configs_dispatcher.h +++ b/ydb/core/cms/console/configs_dispatcher.h @@ -32,8 +32,6 @@ struct TEvConfigsDispatcher { EvSetConfigSubscriptionResponse, EvGetConfigRequest, EvGetConfigResponse, - EvRemoveConfigSubscriptionRequest, - EvRemoveConfigSubscriptionResponse, EvEnd }; @@ -42,48 +40,26 @@ struct TEvConfigsDispatcher { "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_CONFIGS_DISPATCHER)"); struct TEvSetConfigSubscriptionRequest : public TEventLocal<TEvSetConfigSubscriptionRequest, EvSetConfigSubscriptionRequest> { - TEvSetConfigSubscriptionRequest(TActorId subscriber = {}) - : Subscriber(subscriber) + TEvSetConfigSubscriptionRequest() { } - TEvSetConfigSubscriptionRequest(ui32 kind, TActorId subscriber = {}) + TEvSetConfigSubscriptionRequest(ui32 kind) : ConfigItemKinds({kind}) - , Subscriber(subscriber) { } - TEvSetConfigSubscriptionRequest(std::initializer_list<ui32> kinds, TActorId subscriber = {}) + TEvSetConfigSubscriptionRequest(std::initializer_list<ui32> kinds) : ConfigItemKinds(kinds) - , Subscriber(subscriber) - { - } - - TEvSetConfigSubscriptionRequest(const TVector<ui32> &kinds, TActorId subscriber = {}) - : ConfigItemKinds(kinds) - , Subscriber(subscriber) { } TVector<ui32> ConfigItemKinds; - const TActorId Subscriber; }; struct TEvSetConfigSubscriptionResponse : public TEventLocal<TEvSetConfigSubscriptionResponse, EvSetConfigSubscriptionResponse> { }; - struct TEvRemoveConfigSubscriptionRequest : public TEventLocal<TEvRemoveConfigSubscriptionRequest, EvRemoveConfigSubscriptionRequest> { - TEvRemoveConfigSubscriptionRequest(TActorId subscriber = {}) - : Subscriber(subscriber) - { - } - - const TActorId Subscriber; - }; - - struct TEvRemoveConfigSubscriptionResponse : public TEventLocal<TEvRemoveConfigSubscriptionResponse, EvRemoveConfigSubscriptionResponse> { - }; - struct TEvGetConfigRequest : public TEventLocal<TEvGetConfigRequest, EvGetConfigRequest> { TEvGetConfigRequest(ui32 kind, bool cache = true) : ConfigItemKinds({kind}) diff --git a/ydb/core/cms/console/configs_dispatcher_ut.cpp b/ydb/core/cms/console/configs_dispatcher_ut.cpp index 5c74928ef3..8c9a26890d 100644 --- a/ydb/core/cms/console/configs_dispatcher_ut.cpp +++ b/ydb/core/cms/console/configs_dispatcher_ut.cpp @@ -55,9 +55,6 @@ TTenantTestConfig DefaultConsoleTestConfig() NKikimrConsole::TConfigItem ITEM_DOMAIN_LOG_1; NKikimrConsole::TConfigItem ITEM_DOMAIN_LOG_2; -NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_1; -NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_2; -NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_3; TActorId InitConfigsDispatcher(TTenantTestRuntime &runtime) { @@ -70,19 +67,6 @@ TActorId InitConfigsDispatcher(TTenantTestRuntime &runtime) NKikimrConfig::TAppConfig(), {}, {}, "", "", 2, NKikimrConsole::TConfigItem::MERGE, ""); - ITEM_NET_CLASSIFIER_1 - = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, - NKikimrConfig::TAppConfig(), {}, {}, "", "", 3, - NKikimrConsole::TConfigItem::MERGE, ""); - ITEM_NET_CLASSIFIER_2 - = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, - NKikimrConfig::TAppConfig(), {}, {}, "", "", 4, - NKikimrConsole::TConfigItem::MERGE, ""); - ITEM_NET_CLASSIFIER_3 - = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, - NKikimrConfig::TAppConfig(), {}, {}, "", "", 5, - NKikimrConsole::TConfigItem::MERGE, ""); - return MakeConfigsDispatcherID(runtime.GetNodeId(0)); } @@ -147,7 +131,6 @@ struct TEvPrivate { struct TEvGotNotification : public TEventLocal<TEvGotNotification, EvGotNotification> { TConfigId ConfigId; - NKikimrConfig::TAppConfig Config; }; struct TEvComplete : public TEventLocal<TEvComplete, EvComplete> {}; @@ -204,7 +187,6 @@ public: if (Sink) { auto *event = new TEvPrivate::TEvGotNotification; event->ConfigId.Load(rec.GetConfigId()); - event->Config = rec.GetConfig(); ctx.Send(Sink, event); } @@ -260,6 +242,15 @@ TActorId AddSubscriber(TTenantTestRuntime &runtime, TVector<ui32> kinds, bool ho return aid; } +NKikimrConfig::TAppConfig GetConfig(TTenantTestRuntime &runtime, TVector<ui32> kinds, bool cache = true) +{ + TAutoPtr<IEventHandle> handle; + runtime.Send(new IEventHandle(MakeConfigsDispatcherID(runtime.GetNodeId(0)), + runtime.Sender, + new TEvConfigsDispatcher::TEvGetConfigRequest(kinds, cache))); + return *runtime.GrabEdgeEventRethrow<TEvConfigsDispatcher::TEvGetConfigResponse>(handle)->Config; +} + void HoldSubscriber(TTenantTestRuntime &runtime, TActorId aid) { TAutoPtr<IEventHandle> handle; @@ -280,6 +271,21 @@ void SetSubscriptions(TTenantTestRuntime &runtime, TActorId aid, TVector<ui32> k } // anonymous namespace Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { + Y_UNIT_TEST(TestSelfSubscription) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig()); + auto serviceId = InitConfigsDispatcher(runtime); + + ui64 id; + TDispatchOptions options; + options.FinalEvents.emplace_back(CatchReplaceConfigResult(id), 1); + runtime.DispatchEvents(options); + + CheckListConfigSubscriptions(runtime, Ydb::StatusIds::SUCCESS, 0, serviceId, + id, runtime.GetNodeId(0), FQDNHostName(), TENANT1_1_NAME, "type1", + 0, serviceId, + TVector<ui32>({(ui32)NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem})); + } + Y_UNIT_TEST(TestSubscriptionNotification) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); TAutoPtr<IEventHandle> handle; @@ -294,11 +300,11 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_1)); - // Expect two responses from subscribers and zero from dispatcher + // Expect two responses from subscribers and one from dispatcher. TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 3); runtime.DispatchEvents(options); - } + } Y_UNIT_TEST(TestSubscriptionNotificationForNewSubscriberAfterUpdate) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); @@ -317,9 +323,9 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { UnholdSubscriber(runtime, s1); - // Expect response from subscriber + // Expect response from subscriber and from dispatcher. TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); runtime.DispatchEvents(options); // New subscriber should get notification. @@ -353,9 +359,9 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { UnholdSubscriber(runtime, s1); - // Expect response from unhold subscriber + // Expect response from unhold subscriber and from dispatcher. TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); runtime.DispatchEvents(options); } @@ -371,6 +377,7 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { SetSubscriptions(runtime, s1, {}); TDispatchOptions options; + options.FinalEvents.emplace_back(TEvConsole::EvRemoveConfigSubscriptionResponse, 1); runtime.DispatchEvents(options); runtime.GrabEdgeEventRethrow<TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse>(handle); @@ -394,178 +401,81 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { options1.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options1); - // We don't track acks from config dispatcher with InMemory subscriptions + // Subscriber removal should cause config notification response. SetSubscriptions(runtime, s1, {}); TDispatchOptions options2; + options2.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options2); } - Y_UNIT_TEST(TestEmptyChangeCausesNoNotification) { + Y_UNIT_TEST(TestGetCachedConfig) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); TAutoPtr<IEventHandle> handle; InitConfigsDispatcher(runtime); - ui64 notifications = 0; - TActorId subscriber; - auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { - if (ev->Recipient == recipient && ev->Sender == subscriber) { - switch (ev->GetTypeRewrite()) { - case TEvPrivate::EvGotNotification: - ++notifications; - break; - } + ui64 nodeConfigRequests = 0; + auto observer = [&nodeConfigRequests](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + switch (ev->GetTypeRewrite()) { + case TEvConsole::EvGetNodeConfigRequest: + ++nodeConfigRequests; + break; } return TTestActorRuntime::EEventAction::PROCESS; }; ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); - ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster2"); CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, MakeAddAction(ITEM_DOMAIN_LOG_1)); - runtime.SetObserverFunc(observer); - - // Add subscriber and get config via notification. - subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); - runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - UNIT_ASSERT(notifications > 0); - - // Now add another element which doesn't change config body. - // It should cause notification to dispatcher but not test subscriber. - SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); - notifications = 0; - TDispatchOptions options1; - runtime.DispatchEvents(options1); - UNIT_ASSERT_VALUES_EQUAL(notifications, 0); - } - - Y_UNIT_TEST(TestYamlAndNonYamlCoexist) { NKikimrConfig::TAppConfig config; - auto *label = config.AddLabels(); - label->SetName("test"); - label->SetValue("true"); - - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), config); - TAutoPtr<IEventHandle> handle; - InitConfigsDispatcher(runtime); + config.MutableLogConfig()->SetClusterName("cluster1"); - ui64 notifications = 0; - TActorId subscriber; - auto observer = [¬ifications, &subscriber, recipient = runtime.Sender]( - TTestActorRuntimeBase&, - TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { - if (ev->Recipient == recipient && ev->Sender == subscriber) { - switch (ev->GetTypeRewrite()) { - case TEvPrivate::EvGotNotification: - ++notifications; - break; - } - } - return TTestActorRuntime::EEventAction::PROCESS; - }; runtime.SetObserverFunc(observer); - ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); - ITEM_NET_CLASSIFIER_1.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(1); - - CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, - MakeAddAction(ITEM_DOMAIN_LOG_1), - MakeAddAction(ITEM_NET_CLASSIFIER_1)); - - subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem}); - - auto reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - NKikimrConfig::TAppConfig expectedConfig; - auto *ncdConfig = expectedConfig.MutableNetClassifierDistributableConfig(); - ncdConfig->SetLastUpdateTimestamp(1); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); - notifications = 0; - - TString yamlConfig1 = R"( ---- -cluster: "" -version: 1 ---- -config: - log_config: - cluster_name: cluster2 - net_classifier_distributable_config: - last_update_timestamp: 3 - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: [] -)"; - - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig1); - - UNIT_ASSERT(notifications == 0); - - ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster3"); - - CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, - MakeAddAction(ITEM_DOMAIN_LOG_2)); + // Config should be requested from CMS. + auto config1 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, false); + CheckEqualsIgnoringVersion(config, config1); + UNIT_ASSERT(nodeConfigRequests > 0); - UNIT_ASSERT(notifications == 0); + // We didn't ask to cache, so config should still be requested from CMS. + // This time ask to cache config. + nodeConfigRequests = 0; + auto config2 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); + CheckEqualsIgnoringVersion(config, config2); + UNIT_ASSERT(nodeConfigRequests > 0); - ITEM_NET_CLASSIFIER_2.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(3); - - CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, - MakeAddAction(ITEM_NET_CLASSIFIER_2)); - - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - ncdConfig->SetLastUpdateTimestamp(3); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); - notifications = 0; - - TString yamlConfig2 = R"( ---- -cluster: "" -version: 2 ---- -config: {yaml_config_enabled: false} -allowed_labels: {} -selector_config: [] -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig2); - - UNIT_ASSERT(notifications == 0); + // Make sure subscription is online by using it with another subscriber. + AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); + runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - ITEM_NET_CLASSIFIER_3.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(5); + // This time we should get config with no requests to CMS. + nodeConfigRequests = 0; + auto config3 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); + CheckEqualsIgnoringVersion(config, config3); + UNIT_ASSERT_VALUES_EQUAL(nodeConfigRequests, 0); - CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, - MakeAddAction(ITEM_NET_CLASSIFIER_3)); + // Change config and expect dispatcher to process notification. + SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); + runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - ncdConfig->SetLastUpdateTimestamp(5); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + // Now we should get new config with no requests to CMS. + config.MutableLogConfig()->SetClusterName("cluster2"); + auto config4 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); + CheckEqualsIgnoringVersion(config, config4); + UNIT_ASSERT_VALUES_EQUAL(nodeConfigRequests, 0); } - Y_UNIT_TEST(TestYamlEndToEnd) { - NKikimrConfig::TAppConfig config; - auto *label = config.AddLabels(); - label->SetName("test"); - label->SetValue("true"); - - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), config); + Y_UNIT_TEST(TestEmptyChangeCausesNoNotification) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig()); TAutoPtr<IEventHandle> handle; InitConfigsDispatcher(runtime); ui64 notifications = 0; TActorId subscriber; - auto observer = [¬ifications, &subscriber, recipient = runtime.Sender]( - TTestActorRuntimeBase&, - TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { if (ev->Recipient == recipient && ev->Sender == subscriber) { switch (ev->GetTypeRewrite()) { case TEvPrivate::EvGotNotification: @@ -575,234 +485,28 @@ selector_config: [] } return TTestActorRuntime::EEventAction::PROCESS; }; - runtime.SetObserverFunc(observer); ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, MakeAddAction(ITEM_DOMAIN_LOG_1)); - subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); - auto reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - NKikimrConfig::TAppConfig expectedConfig; - auto *logConfig = expectedConfig.MutableLogConfig(); - logConfig->SetClusterName("cluster1"); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); - notifications = 0; - - TString yamlConfig1 = R"( ---- -cluster: "" -version: 1 ---- -config: - log_config: - cluster_name: cluster1 -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: [] -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig1); - UNIT_ASSERT(notifications == 0); - - TString yamlConfig2 = R"( ---- -cluster: "" -version: 2 ---- -config: - log_config: - cluster_name: cluster1 - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: [] -)"; - - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig2); - - ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster2"); - - CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, - MakeAddAction(ITEM_DOMAIN_LOG_2)); - - TString yamlConfig3 = R"( ---- -cluster: "" -version: 3 ---- -config: - log_config: - cluster_name: cluster3 - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: [] -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig3); - - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - expectedConfig = {}; - logConfig = expectedConfig.MutableLogConfig(); - logConfig->SetClusterName("cluster3"); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); - notifications = 0; + runtime.SetObserverFunc(observer); - TString yamlConfig4 = R"( ---- -cluster: "" -version: 4 ---- -config: - log_config: - cluster_name: cluster3 - cms_config: - sentinel_config: - enable: true - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: [] -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig4); - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig4); - UNIT_ASSERT(notifications == 0); - - TString yamlConfig5 = R"( ---- -cluster: "" -version: 5 ---- -config: - log_config: - cluster_name: cluster3 - cms_config: - sentinel_config: - enable: true - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: -- description: Test - selector: - test: true - config: - log_config: !inherit - entry: - - component: AUDIT_LOG_WRITER - level: 7 -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig5); - - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - expectedConfig = {}; - logConfig = expectedConfig.MutableLogConfig(); - logConfig->SetClusterName("cluster3"); - auto *entry = logConfig->AddEntry(); - entry->SetComponent("AUDIT_LOG_WRITER"); - entry->SetLevel(7); + // Add subscriber and get config via notification. + subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); + runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); - notifications = 0; - TString yamlConfig6 = R"( ---- -cluster: "" -version: 6 ---- -config: - log_config: - cluster_name: cluster3 - cms_config: - sentinel_config: - enable: true - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: -- description: Test - selector: - test: - not_in: - - true - config: - log_config: !inherit - entry: - - component: AUDIT_LOG_WRITER - level: 7 -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig6); - - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - expectedConfig = {}; - logConfig = expectedConfig.MutableLogConfig(); - logConfig->SetClusterName("cluster3"); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + // Now add another element which doesn't change config body. + // It should cause notification to dispatcher but not test subscriber. + SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); notifications = 0; - - TString yamlConfig7 = R"( ---- -cluster: "" -version: 7 ---- -config: - log_config: - cluster_name: cluster3 - yaml_config_enabled: true - -allowed_labels: - test: - type: enum - values: - ? true - -selector_config: -- description: Test - selector: - test: true - config: - yaml_config_enabled: false -)"; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig7); - - reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); - expectedConfig = {}; - logConfig = expectedConfig.MutableLogConfig(); - logConfig->SetClusterName("cluster2"); - UNIT_ASSERT(notifications > 0); - UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + TDispatchOptions options1; + options1.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); + runtime.DispatchEvents(options1); + UNIT_ASSERT_VALUES_EQUAL(notifications, 0); } } diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index 3602df1853..3928c04cfb 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -93,7 +93,6 @@ public: switch (ev->GetTypeRewrite()) { HFuncTraced(TEvPrivate::TEvRetryPoolStatus, Handle); HFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); - HFuncTraced(TEvConsole::TEvGetNodeConfigResponse, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionResponse, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); @@ -150,32 +149,6 @@ public: Generation = 0; Die(ctx); } - - if (!FirstUpdateSent) { - auto request = MakeHolder<TEvConsole::TEvGetNodeConfigRequest>(); - request->Record.MutableNode()->SetNodeId(SelfId().NodeId()); - request->Record.MutableNode()->SetHost(FQDNHostName()); - request->Record.MutableNode()->SetTenant(Tenant); - request->Record.MutableNode()->SetNodeType(NodeType); - for (auto &kind : Kinds) { - request->Record.AddItemKinds(kind); - } - - NTabletPipe::SendData(ctx, Pipe, request.Release(), Cookie); - } - } - - void Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev, const TActorContext &ctx) { - if (!FirstUpdateSent) { - ctx.ExecutorThread.Send( - new NActors::IEventHandle( - SelfId(), - ev->Sender, - new NConsole::TEvConsole::TEvConfigSubscriptionNotification( - Generation, - ev->Get()->Record.GetConfig(), - THashSet<ui32>(Kinds.begin(), Kinds.end())))); - } } void Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev, const TActorContext &ctx) { @@ -212,7 +185,6 @@ public: YamlConfig = rec.GetYamlConfig(); YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); } - notChanged = false; } diff --git a/ydb/core/cms/console/util.cpp b/ydb/core/cms/console/util.cpp index 806aeed20f..1ba525addd 100644 --- a/ydb/core/cms/console/util.cpp +++ b/ydb/core/cms/console/util.cpp @@ -42,74 +42,4 @@ TString KindsToString(const TVector<ui32> &kinds) { return ss.Str(); } -TDynBitMap KindsToBitMap(const TVector<ui32> &kinds) -{ - TDynBitMap result; - for (auto &kind : kinds) - result.Set(kind); - - return result; -} - -void ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds) -{ - NKikimrConfig::TAppConfig fromCopy = from; - - auto *desc = to.GetDescriptor(); - auto *reflection = to.GetReflection(); - - for (int i = 0; i < desc->field_count(); i++) - { - auto *field = desc->field(i); - auto tag = field->number(); - if (field && !field->is_repeated()) { - if (kinds.Test(tag)) { - if (reflection->HasField(to, field)) { - reflection->ClearField(&to, field); - } - } else { - if (reflection->HasField(fromCopy, field)) { - reflection->ClearField(&fromCopy, field); - } - } - } else { - reflection->ClearField(&to, field); - reflection->ClearField(&fromCopy, field); - } - } - - to.MergeFrom(fromCopy); -} - -bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs) -{ - return lhs.SerializeAsString() == rhs.SerializeAsString(); -} - -bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs, const TDynBitMap &kinds) -{ - NKikimrConfig::TAppConfig lhsTrunc; - ReplaceConfigItems(lhs, lhsTrunc, kinds); - - NKikimrConfig::TAppConfig rhsTrunc; - ReplaceConfigItems(rhs, rhsTrunc, kinds); - - return CompareConfigs(rhsTrunc, lhsTrunc); -} - -NKikimrConfig::TConfigVersion FilterVersion( - const NKikimrConfig::TConfigVersion &version, - const TDynBitMap &kinds) -{ - NKikimrConfig::TConfigVersion result; - - for (auto &item : version.GetItems()) { - if (kinds.Test(item.GetKind())) { - result.AddItems()->CopyFrom(item); - } - } - - return result; -} - } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/util.h b/ydb/core/cms/console/util.h index 0b992d4615..3a88cf4572 100644 --- a/ydb/core/cms/console/util.h +++ b/ydb/core/cms/console/util.h @@ -1,8 +1,6 @@ #pragma once #include "defs.h" -#include <ydb/core/protos/config.pb.h> - #include <ydb/core/base/tablet_pipe.h> namespace NKikimr::NConsole { @@ -15,25 +13,4 @@ TString KindsToString(const THashSet<ui32> &kinds); TString KindsToString(const TVector<ui32> &kinds); -TDynBitMap KindsToBitMap(const TVector<ui32> &kinds); - -/** - * Replace 'kinds' in 'to' from 'from' - * repeated items are removed - */ -void ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds); - -bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); - -/** - * Compares only fields in specified kinds - * repeated items are ignored - */ -bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs, const TDynBitMap &kinds); - -/** - * Extracts versions for specified kinds - */ -NKikimrConfig::TConfigVersion FilterVersion(const NKikimrConfig::TConfigVersion &version, const TDynBitMap &kinds); - } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/yaml_config/yaml_config.cpp b/ydb/core/cms/console/yaml_config/yaml_config.cpp index 0376f20041..4f00f34955 100644 --- a/ydb/core/cms/console/yaml_config/yaml_config.cpp +++ b/ydb/core/cms/console/yaml_config/yaml_config.cpp @@ -1,9 +1,8 @@ #include "yaml_config.h" #include "yaml_config_impl.h" -#include <ydb/core/base/appdata.h> - #include <library/cpp/protobuf/json/json2proto.h> +#include <ydb/core/base/appdata.h> template <> struct THash<NYamlConfig::TLabel> { @@ -598,77 +597,6 @@ ui64 GetVersion(const TString& config) { return version; } -/** - * Config used to convert protobuf from/to json - * changes how names are translated e.g. PDiskInfo -> pdisk_info instead of p_disk_info - */ -NProtobufJson::TJson2ProtoConfig GetJsonToProtoConfig() { - NProtobufJson::TJson2ProtoConfig config; - config.SetFieldNameMode(NProtobufJson::TJson2ProtoConfig::FieldNameSnakeCaseDense); - config.SetEnumValueMode(NProtobufJson::TJson2ProtoConfig::EnumCaseInsensetive); - config.CastRobust = true; - config.MapAsObject = true; - config.AllowUnknownFields = false; - return config; -} - -void ResolveAndParseYamlConfig( - const TString& yamlConfig, - const TMap<ui64, TString>& volatileYamlConfigs, - const TMap<TString, TString>& labels, - NKikimrConfig::TAppConfig& appConfig, - TString* resolvedYamlConfig, - TString* resolvedJsonConfig) { - - auto parser = NFyaml::TParser::Create(yamlConfig); - parser.NextDocument(); - auto tree = parser.NextDocument(); - - for (auto& [_, config] : volatileYamlConfigs) { - auto d = NFyaml::TDocument::Parse(config); - NYamlConfig::AppendVolatileConfigs(tree.value(), d); - } - - TSet<NYamlConfig::TNamedLabel> namedLabels; - for (auto& [name, label] : labels) { - namedLabels.insert(NYamlConfig::TNamedLabel{name, label}); - } - - auto config = NYamlConfig::Resolve(tree.value(), namedLabels); - - if (resolvedYamlConfig) { - TStringStream resolvedYamlConfigStream; - resolvedYamlConfigStream << config.second; - *resolvedYamlConfig = resolvedYamlConfigStream.Str(); - } - - TStringStream resolvedJsonConfigStream; - resolvedJsonConfigStream << NFyaml::TJsonEmitter(config.second); - - if (resolvedJsonConfig) { - *resolvedJsonConfig = resolvedJsonConfigStream.Str(); - } - - NJson::TJsonValue json; - Y_VERIFY(NJson::ReadJsonTree(resolvedJsonConfigStream.Str(), &json), "Got invalid config from Console"); - - NProtobufJson::MergeJson2Proto(json, appConfig, NYamlConfig::GetJsonToProtoConfig()); -} - -void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to) { - if (from.HasNameserviceConfig()) { - to.MutableNameserviceConfig()->CopyFrom(from.GetNameserviceConfig()); - } - - if (from.HasNetClassifierDistributableConfig()) { - to.MutableNetClassifierDistributableConfig()->CopyFrom(from.GetNetClassifierDistributableConfig()); - } - - if (from.NamedConfigsSize()) { - to.MutableNamedConfigs()->CopyFrom(from.GetNamedConfigs()); - } -} - } // namespace NYamlConfig template <> diff --git a/ydb/core/cms/console/yaml_config/yaml_config.h b/ydb/core/cms/console/yaml_config/yaml_config.h index 97e1b71c03..08f0c63fad 100644 --- a/ydb/core/cms/console/yaml_config/yaml_config.h +++ b/ydb/core/cms/console/yaml_config/yaml_config.h @@ -172,22 +172,4 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatil */ ui64 GetVersion(const TString& config); -/** - * Resolves config for given labels and stores result to appConfig - * Stores intermediate resolve data in resolvedYamlConfig and resolvedJsonConfig if given - */ -void ResolveAndParseYamlConfig( - const TString& yamlConfig, - const TMap<ui64, TString>& volatileYamlConfigs, - const TMap<TString, TString>& labels, - NKikimrConfig::TAppConfig& appConfig, - TString* resolvedYamlConfig = nullptr, - TString* resolvedJsonConfig = nullptr); - -/** - * Replaces kinds not managed by yaml config (e.g. NetClassifierConfig) from config 'from' in config 'to' - * if corresponding configs are presenet in 'from' - */ -void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to); - } // namespace NYamlConfig diff --git a/ydb/core/cms/ui/config_dispatcher.css b/ydb/core/cms/ui/config_dispatcher.css deleted file mode 100644 index fe0aa308cd..0000000000 --- a/ydb/core/cms/ui/config_dispatcher.css +++ /dev/null @@ -1,6 +0,0 @@ -.CodeMirror { - border: 1px solid #eee; - height: auto; - display: flex; - width: 1110px; -} diff --git a/ydb/core/cms/ui/configs_dispatcher_main.js b/ydb/core/cms/ui/configs_dispatcher_main.js deleted file mode 100644 index 7af3993f5f..0000000000 --- a/ydb/core/cms/ui/configs_dispatcher_main.js +++ /dev/null @@ -1,120 +0,0 @@ -var createEditor; - -var codeMirror; -var codeMirrorResolved; - -function replaceWithEditor(selector) { - var container = $(selector); - var value = container.text(); - container.text(""); - var editor = createEditor(container.get(0), true, 1068) - editor.setValue(value); - return editor; -} - -function main() { - $("#nodePicker").fuzzyComplete(nodeNames); - $('#nodePicker').on('keyup blur', function() { - if (window.location.pathname === "/actors/configs_dispatcher") { - $('#nodesGo').attr("href", window.location.protocol + "//" + $(this).parent().find("select").val() + ":8765/actors/configs_dispatcher"); - } else { - $('#nodesGo').attr("href", "/" + $(this).parent().find("select").val() + ":8765/actors/configs_dispatcher"); - } - }); - - codeMirror = replaceWithEditor("#yaml-config-item"); - codeMirror.trigger('fold', 'editor.foldLevel2'); - - $("#fold-yaml-config").click(function() { - codeMirror.trigger('fold', 'editor.foldLevel2'); - }); - - $("#unfold-yaml-config").click(function() { - codeMirror.trigger('fold', 'editor.unfoldAll'); - }); - - $("#copy-yaml-config").click(function() { - copyToClipboard(codeMirror.getValue()); - }); - - codeMirrorResolved = replaceWithEditor("#resolved-yaml-config-item"); - codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); - - $("#fold-resolved-yaml-config").click(function() { - codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); - }); - - $("#unfold-resolved-yaml-config").click(function() { - codeMirrorResolved.trigger('fold', 'editor.unfoldAll'); - }); - - $("#copy-resolved-yaml-config").click(function() { - copyToClipboard(codeMirrorResolved.getValue()); - }); - - $("#host-ref").text("YDB Developer UI - " + window.location.hostname); - - $(".yaml-config-item").each(function() { - let editor = replaceWithEditor(this); - - $(this).parent().find('.fold-yaml-config').click(function() { - editor.trigger('fold', 'editor.foldLevel2'); - }); - - $(this).parent().find('.unfold-yaml-config').click(function() { - editor.trigger('fold', 'editor.unfoldAll'); - }); - - $(this).parent().find('.copy-yaml-config').click(function() { - copyToClipboard(editor.getValue()); - }); - }); -} - -let run = () => { - require.config({ - paths: { vs: "https://cdn.jsdelivr.net/npm/monaco-editor@0.27.0/min/vs" } - }); - - require(["vs/editor/editor.main"], function () { - createEditor = (container, readOnly, width) => { - var editor; - container.style.border = '1px solid #eee'; - container.style.borderRadius = '8px'; - container.style.overflow = 'hidden'; - editor = monaco.editor.create(container, { - language: "yaml", - scrollBeyondLastLine: false, - wrappingStrategy: 'advanced', - minimap: { - enabled: false, - }, - overviewRulerLanes: 0, - automaticLayout: true, - readOnly: readOnly, - scrollbar: { - alwaysConsumeMouseWheel: false, - }, - }); - let ignoreEvent = false; - const updateHeight = () => { - const contentHeight = editor.getContentHeight(); - container.style.width = `100%`; - container.style.height = `${contentHeight + 2}px`; - try { - ignoreEvent = true; - editor.layout({ width, height: contentHeight }); - } finally { - ignoreEvent = false; - } - }; - editor.onDidContentSizeChange(updateHeight); - updateHeight(); - return editor; - } - - $(document).ready(main); - }); -}; - -run(); diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.darwin-x86_64.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.darwin-x86_64.txt index 2e8256f0cc..3b9d5d19d6 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.darwin-x86_64.txt @@ -23,7 +23,6 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib - cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt index e54a367187..6e5b760dbd 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt @@ -24,7 +24,6 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib - cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-x86_64.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-x86_64.txt index e54a367187..6e5b760dbd 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-x86_64.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-x86_64.txt @@ -24,7 +24,6 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib - cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.windows-x86_64.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.windows-x86_64.txt index 2e8256f0cc..3b9d5d19d6 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.windows-x86_64.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.windows-x86_64.txt @@ -23,7 +23,6 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib - cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 142942a56d..428e6beedd 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -2,7 +2,6 @@ #include "cli_cmds.h" #include <ydb/core/base/location.h> #include <ydb/core/base/path.h> -#include <ydb/core/cms/console/yaml_config/yaml_config.h> #include <ydb/core/driver_lib/run/run.h> #include <ydb/library/yaml_config/yaml_config_parser.h> #include <ydb/public/lib/deprecated/kicli/kicli.h> @@ -730,8 +729,7 @@ protected: TenantName, NodeType, DeduceNodeDomain(), - AppConfig.GetAuthConfig().GetStaffApiUserToken(), - true); + AppConfig.GetAuthConfig().GetStaffApiUserToken()); if (result.IsSuccess()) { auto appConfig = result.GetConfig(); @@ -743,22 +741,7 @@ protected: } } - NKikimrConfig::TAppConfig yamlConfig; - - if (result.HasYamlConfig() && !result.GetYamlConfig().empty()) { - NYamlConfig::ResolveAndParseYamlConfig( - result.GetYamlConfig(), - result.GetVolatileYamlConfigs(), - RunConfig.Labels, - yamlConfig); - } - - if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { - BaseConfig.Swap(&yamlConfig); - NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), BaseConfig); - } else { - BaseConfig.Swap(&appConfig); - } + BaseConfig.Swap(&appConfig); Cout << "Success." << Endl; @@ -1071,8 +1054,7 @@ protected: TenantName, NodeType, DeduceNodeDomain(), - AppConfig.GetAuthConfig().GetStaffApiUserToken(), - true); + AppConfig.GetAuthConfig().GetStaffApiUserToken()); if (!result.IsSuccess()) { error = result.GetErrorMessage(); @@ -1082,24 +1064,7 @@ protected: Cout << "Success." << Endl; - NKikimrConfig::TAppConfig appConfig; - - NKikimrConfig::TAppConfig yamlConfig; - - if (result.HasYamlConfig() && !result.GetYamlConfig().empty()) { - NYamlConfig::ResolveAndParseYamlConfig( - result.GetYamlConfig(), - result.GetVolatileYamlConfigs(), - RunConfig.Labels, - yamlConfig); - } - - if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { - appConfig = yamlConfig; - NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), appConfig); - } else { - appConfig = result.GetConfig(); - } + auto appConfig = result.GetConfig(); if (RunConfig.PathToConfigCacheFile) { Cout << "Saving config to cache file " << RunConfig.PathToConfigCacheFile << Endl; diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp index baddc2b950..4bf42243b1 100644 --- a/ydb/core/mind/node_broker.cpp +++ b/ydb/core/mind/node_broker.cpp @@ -132,8 +132,6 @@ void TNodeBroker::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TNodeBroker::Cleanup"); - NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); - TxProcessor->Clear(); } @@ -394,8 +392,14 @@ void TNodeBroker::AddNodeToEpochCache(const TNodeInfo &node) void TNodeBroker::SubscribeForConfigUpdates(const TActorContext &ctx) { + if (ConfigSubscriptionId) + return; + ui32 item = (ui32)NKikimrConsole::TConfigItem::NodeBrokerConfigItem; - NConsole::SubscribeViaConfigDispatcher(ctx, {item}, ctx.SelfID); + ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), + {item}, + "", + ctx.SelfID)); } void TNodeBroker::ProcessTx(ITransaction *tx, diff --git a/ydb/core/mind/node_broker__update_config.cpp b/ydb/core/mind/node_broker__update_config.cpp index 2211b447ac..f622985644 100644 --- a/ydb/core/mind/node_broker__update_config.cpp +++ b/ydb/core/mind/node_broker__update_config.cpp @@ -10,7 +10,7 @@ public: TEvConsole::TEvConfigNotificationRequest::TPtr notification) : TBase(self) , Notification(std::move(notification)) - , Config(Notification->Get()->GetConfig().GetNodeBrokerConfig()) + , Config(Notification->Get()->Record.GetConfig().GetNodeBrokerConfig()) , Modify(false) { } @@ -31,8 +31,12 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); - if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->Config)) - Modify = true; + if (rec.GetSubscriptionId() != Self->ConfigSubscriptionId) { + LOG_ERROR_S(ctx, NKikimrServices::NODE_BROKER, + "Config subscription id mismatch (" << rec.GetSubscriptionId() + << " vs expected " << Self->ConfigSubscriptionId << ")"); + return false; + } auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); Response = new IEventHandle(Notification->Sender, Self->SelfId(), resp.Release(), @@ -48,9 +52,6 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); - if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->Config)) - Modify = true; - auto resp = MakeHolder<TEvNodeBroker::TEvSetConfigResponse>(); resp->Record.MutableStatus()->SetCode(NKikimrNodeBroker::TStatus::OK); Response = new IEventHandle(Request->Sender, Self->SelfId(), resp.Release(), @@ -68,8 +69,9 @@ public: if (Request && !ProcessRequest(ctx)) return true; - if (Modify) - Self->DbUpdateConfig(Config, txc); + Self->DbUpdateConfig(Config, txc); + + Modify = true; return true; } diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h index 8acd46d5c7..be3f77cb77 100644 --- a/ydb/core/mind/node_broker_impl.h +++ b/ydb/core/mind/node_broker_impl.h @@ -4,7 +4,6 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/console.h> -#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/tx_processor.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -180,8 +179,6 @@ private: HFuncTraced(TEvPrivate::TEvUpdateEpoch, Handle); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { diff --git a/ydb/core/mind/tenant_slot_broker.cpp b/ydb/core/mind/tenant_slot_broker.cpp index 54b99dc209..e4a5449094 100644 --- a/ydb/core/mind/tenant_slot_broker.cpp +++ b/ydb/core/mind/tenant_slot_broker.cpp @@ -415,7 +415,10 @@ void TTenantSlotBroker::OnActivateExecutor(const TActorContext &ctx) tabletCounters->RemoveSubgroup("type", "TENANT_SLOT_BROKER"); Counters = new TCounters(tabletCounters->GetSubgroup("type", "TENANT_SLOT_BROKER")); - NConsole::SubscribeViaConfigDispatcher(ctx, {(ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem}, ctx.SelfID); + ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), + {(ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem}, + "", + ctx.SelfID)); ProcessTx(CreateTxInitScheme(), ctx); } @@ -612,8 +615,6 @@ bool TTenantSlotBroker::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, void TTenantSlotBroker::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::TENANT_SLOT_BROKER, "Cleanup"); - - NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); } void TTenantSlotBroker::Die(const TActorContext &ctx) @@ -624,7 +625,6 @@ void TTenantSlotBroker::Die(const TActorContext &ctx) void TTenantSlotBroker::LoadConfigFromProto(const NKikimrTenantSlotBroker::TConfig &config) { - Config = config; PendingTimeout = TDuration::MicroSeconds(config.GetPendingSlotTimeout()); } diff --git a/ydb/core/mind/tenant_slot_broker__update_config.cpp b/ydb/core/mind/tenant_slot_broker__update_config.cpp index c9f77a0dfc..360b28d9c1 100644 --- a/ydb/core/mind/tenant_slot_broker__update_config.cpp +++ b/ydb/core/mind/tenant_slot_broker__update_config.cpp @@ -19,21 +19,20 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); - NIceDb::TNiceDb db(txc.DB); - - const auto &config = Event->Get()->GetConfig().GetTenantSlotBrokerConfig(); - - if (!google::protobuf::util::MessageDifferencer::Equals(config, Self->Config)) { - TString serializedConfig = config.SerializeAsString(); - db.Table<Schema::Config>().Key(ConfigKey_Config) - .Update(NIceDb::TUpdate<Schema::Config::Value>(serializedConfig)); - - Modify = true; + if (rec.GetSubscriptionId() != Self->ConfigSubscriptionId) { + LOG_ERROR_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, + "Config subscription id mismatch (" << rec.GetSubscriptionId() + << " vs expected " << Self->ConfigSubscriptionId << ")"); + return true; } - auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - Response = new IEventHandle(Event->Sender, Self->SelfId(), resp.Release(), - 0, Event->Cookie); + NIceDb::TNiceDb db(txc.DB); + TString config; + Y_PROTOBUF_SUPPRESS_NODISCARD rec.GetConfig().GetTenantSlotBrokerConfig().SerializeToString(&config); + db.Table<Schema::Config>().Key(ConfigKey_Config) + .Update(NIceDb::TUpdate<Schema::Config::Value>(config)); + + Modify = true; return true; } @@ -45,17 +44,16 @@ public: if (Modify) { auto &rec = Event->Get()->Record; Self->LoadConfigFromProto(rec.GetConfig().GetTenantSlotBrokerConfig()); - } - if (Response) - ctx.Send(Response); + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); + ctx.Send(Event->Sender, resp.Release(), 0, Event->Cookie); + } Self->TxCompleted(this, ctx); } private: TEvConsole::TEvConfigNotificationRequest::TPtr Event; - TAutoPtr<IEventHandle> Response; bool Modify; }; diff --git a/ydb/core/mind/tenant_slot_broker_impl.h b/ydb/core/mind/tenant_slot_broker_impl.h index 7c3c2f0810..5edd9efcdb 100644 --- a/ydb/core/mind/tenant_slot_broker_impl.h +++ b/ydb/core/mind/tenant_slot_broker_impl.h @@ -6,7 +6,6 @@ #include <ydb/core/base/location.h> #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/console.h> -#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/protos/tenant_slot_broker.pb.h> @@ -1122,8 +1121,6 @@ private: HFuncTraced(TEvTenantSlotBroker::TEvGetTenantState, Handle); HFuncTraced(TEvTenantSlotBroker::TEvListTenants, Handle); HFuncTraced(TEvTenantSlotBroker::TEvRegisterPool, Handle); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { @@ -1154,7 +1151,6 @@ public: private: TDeque<TAutoPtr<IEventHandle>> InitQueue; - NKikimrTenantSlotBroker::TConfig Config; TDuration PendingTimeout; ui64 RequestId; ui32 DomainId; diff --git a/ydb/core/mind/tenant_ut_pool.cpp b/ydb/core/mind/tenant_ut_pool.cpp index 2abd1984cd..bd3dbc7fc0 100644 --- a/ydb/core/mind/tenant_ut_pool.cpp +++ b/ydb/core/mind/tenant_ut_pool.cpp @@ -135,7 +135,7 @@ void ChangeMonitoringConfig(TTenantTestRuntime &runtime, TDispatchOptions options; options.FinalEvents.emplace_back - (TIsConfigNotificationProcessed(2 * runtime.GetNodeCount(), + (TIsConfigNotificationProcessed(3 * runtime.GetNodeCount(), 2 * waitForPoolStatus * runtime.GetNodeCount())); runtime.DispatchEvents(options); } diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp index 1c7f1b68f1..90d1abfef5 100644 --- a/ydb/core/testlib/tenant_runtime.cpp +++ b/ydb/core/testlib/tenant_runtime.cpp @@ -1004,12 +1004,7 @@ void TTenantTestRuntime::Setup(bool createTenantPools) // Create other local services for (size_t i = 0; i < Config.Nodes.size(); ++i) { if (Config.CreateConfigsDispatcher) { - TMap<TString, TString> labels; - for (const auto &label : Extension.GetLabels()) { - labels[label.GetName()] = label.GetValue(); - } - labels.emplace("node_id", ToString(i)); - auto aid = Register(CreateConfigsDispatcher(Extension, labels)); + auto aid = Register(CreateConfigsDispatcher(Extension, {})); EnableScheduleForActor(aid, true); RegisterService(MakeConfigsDispatcherID(GetNodeId(0)), aid, 0); } |