diff options
author | radix <radix@yandex-team.ru> | 2022-02-10 16:49:44 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:44 +0300 |
commit | 24658cfd172ef5de124df383c334079783154fbf (patch) | |
tree | bccc29f3945c7245036b5973bcc7358a760fcc25 | |
parent | ceabccd128eb94ed6566936fcab2093ec62aa84a (diff) | |
download | ydb-24658cfd172ef5de124df383c334079783154fbf.tar.gz |
Restoring authorship annotation for <radix@yandex-team.ru>. Commit 1 of 2.
180 files changed, 10404 insertions, 10404 deletions
diff --git a/library/cpp/http/misc/parsed_request.cpp b/library/cpp/http/misc/parsed_request.cpp index e332a24e91..baef4221a7 100644 --- a/library/cpp/http/misc/parsed_request.cpp +++ b/library/cpp/http/misc/parsed_request.cpp @@ -16,11 +16,11 @@ static inline TStringBuf StripLeft(const TStringBuf& s) noexcept { TParsedHttpRequest::TParsedHttpRequest(const TStringBuf& str) { TStringBuf tmp; - if (!StripLeft(str).TrySplit(' ', Method, tmp)) { + if (!StripLeft(str).TrySplit(' ', Method, tmp)) { ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; } - if (!StripLeft(tmp).TrySplit(' ', Request, Proto)) { + if (!StripLeft(tmp).TrySplit(' ', Request, Proto)) { ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; } diff --git a/util/folder/pathsplit.cpp b/util/folder/pathsplit.cpp index 81d439a727..0477fbc033 100644 --- a/util/folder/pathsplit.cpp +++ b/util/folder/pathsplit.cpp @@ -36,7 +36,7 @@ void TPathSplitTraitsUnix::DoParsePart(const TStringBuf part0) { TStringBuf next(part0); TStringBuf part; - while (TStringBuf(next).TrySplit('/', part, next)) { + while (TStringBuf(next).TrySplit('/', part, next)) { AppendComponent(part); } diff --git a/util/generic/strbuf.h b/util/generic/strbuf.h index 70b9360d58..e23d257fd4 100644 --- a/util/generic/strbuf.h +++ b/util/generic/strbuf.h @@ -171,63 +171,63 @@ public: return data() != nullptr; } -public: - /** - * Tries to split string in two parts using given delimiter character. - * Searches for the delimiter, scanning string from the beginning. - * The delimiter is excluded from the result. Both out parameters are - * left unmodified if there was no delimiter character in string. - * - * @param[in] delim Delimiter character. - * @param[out] l The first part of split result. - * @param[out] r The second part of split result. - * @returns Whether the split was actually performed. - */ +public: + /** + * Tries to split string in two parts using given delimiter character. + * Searches for the delimiter, scanning string from the beginning. + * The delimiter is excluded from the result. Both out parameters are + * left unmodified if there was no delimiter character in string. + * + * @param[in] delim Delimiter character. + * @param[out] l The first part of split result. + * @param[out] r The second part of split result. + * @returns Whether the split was actually performed. + */ inline bool TrySplit(TCharType delim, TdSelf& l, TdSelf& r) const noexcept { return TrySplitOn(TBase::find(delim), l, r); } - /** - * Tries to split string in two parts using given delimiter character. - * Searches for the delimiter, scanning string from the end. - * The delimiter is excluded from the result. Both out parameters are - * left unmodified if there was no delimiter character in string. - * - * @param[in] delim Delimiter character. - * @param[out] l The first part of split result. - * @param[out] r The second part of split result. - * @returns Whether the split was actually performed. - */ + /** + * Tries to split string in two parts using given delimiter character. + * Searches for the delimiter, scanning string from the end. + * The delimiter is excluded from the result. Both out parameters are + * left unmodified if there was no delimiter character in string. + * + * @param[in] delim Delimiter character. + * @param[out] l The first part of split result. + * @param[out] r The second part of split result. + * @returns Whether the split was actually performed. + */ inline bool TryRSplit(TCharType delim, TdSelf& l, TdSelf& r) const noexcept { return TrySplitOn(TBase::rfind(delim), l, r); } - /** - * Tries to split string in two parts using given delimiter sequence. - * Searches for the delimiter, scanning string from the beginning. - * The delimiter sequence is excluded from the result. Both out parameters - * are left unmodified if there was no delimiter character in string. - * - * @param[in] delim Delimiter sequence. - * @param[out] l The first part of split result. - * @param[out] r The second part of split result. - * @returns Whether the split was actually performed. - */ + /** + * Tries to split string in two parts using given delimiter sequence. + * Searches for the delimiter, scanning string from the beginning. + * The delimiter sequence is excluded from the result. Both out parameters + * are left unmodified if there was no delimiter character in string. + * + * @param[in] delim Delimiter sequence. + * @param[out] l The first part of split result. + * @param[out] r The second part of split result. + * @returns Whether the split was actually performed. + */ inline bool TrySplit(TdSelf delim, TdSelf& l, TdSelf& r) const noexcept { return TrySplitOn(TBase::find(delim), l, r, delim.size()); } - /** - * Tries to split string in two parts using given delimiter sequence. - * Searches for the delimiter, scanning string from the end. - * The delimiter sequence is excluded from the result. Both out parameters - * are left unmodified if there was no delimiter character in string. - * - * @param[in] delim Delimiter sequence. - * @param[out] l The first part of split result. - * @param[out] r The second part of split result. - * @returns Whether the split was actually performed. - */ + /** + * Tries to split string in two parts using given delimiter sequence. + * Searches for the delimiter, scanning string from the end. + * The delimiter sequence is excluded from the result. Both out parameters + * are left unmodified if there was no delimiter character in string. + * + * @param[in] delim Delimiter sequence. + * @param[out] l The first part of split result. + * @param[out] r The second part of split result. + * @returns Whether the split was actually performed. + */ inline bool TryRSplit(TdSelf delim, TdSelf& l, TdSelf& r) const noexcept { return TrySplitOn(TBase::rfind(delim), l, r, delim.size()); } @@ -521,7 +521,7 @@ private: template <typename TDelimiterType> inline void SplitTemplate(TDelimiterType delim, TdSelf& l, TdSelf& r) const noexcept { - if (!TrySplit(delim, l, r)) { + if (!TrySplit(delim, l, r)) { l = *this; r = TdSelf(); } @@ -529,7 +529,7 @@ private: template <typename TDelimiterType> inline void RSplitTemplate(TDelimiterType delim, TdSelf& l, TdSelf& r) const noexcept { - if (!TryRSplit(delim, l, r)) { + if (!TryRSplit(delim, l, r)) { r = *this; l = TdSelf(); } diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index c666f7468c..b70bda343c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -131,13 +131,13 @@ struct TAppData { THolder<NKikimrBlobStorage::TNodeWardenServiceSet> StaticBlobStorageConfig; THolder<NKikimrCms::TCmsConfig> DefaultCmsConfig; - + NKikimrStream::TStreamingConfig StreamingConfig; NKikimrPQ::TPQConfig PQConfig; - NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig; - NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig; - NKikimrNetClassifier::TNetClassifierDistributableConfig NetClassifierDistributableConfig; - NKikimrConfig::TSqsConfig SqsConfig; + NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig; + NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig; + NKikimrNetClassifier::TNetClassifierDistributableConfig NetClassifierDistributableConfig; + NKikimrConfig::TSqsConfig SqsConfig; NKikimrProto::TAuthConfig AuthConfig; NKikimrProto::TKeyConfig KeyConfig; NKikimrProto::TKeyConfig PDiskKeyConfig; diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index f5fedfe19b..8ed9dc4396 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -109,8 +109,8 @@ struct TKikimrEvents : TEvents { ES_GRPC_REQUEST_PROXY, ES_EXPORT_SERVICE, ES_TX_ALLOCATOR_CLIENT, - ES_PQ_CLUSTER_TRACKER, - ES_NET_CLASSIFIER, + ES_PQ_CLUSTER_TRACKER, + ES_NET_CLASSIFIER, ES_SYSTEM_VIEW, ES_TENANT_NODE_ENUMERATOR, ES_SERVICE_ACCOUNT_SERVICE, diff --git a/ydb/core/cms/console/console.cpp b/ydb/core/cms/console/console.cpp index fe477202af..7747a1f157 100644 --- a/ydb/core/cms/console/console.cpp +++ b/ydb/core/cms/console/console.cpp @@ -3,8 +3,8 @@ #include "console_tenants_manager.h" #include "http.h" -#include "net_classifier_updater.h" - +#include "net_classifier_updater.h" + #include <ydb/core/base/counters.h> #include <ydb/core/cms/console/validators/registry.h> @@ -33,10 +33,10 @@ void TConsole::OnActivateExecutor(const TActorContext &ctx) AppData()->FeatureFlags); ctx.RegisterWithSameMailbox(TenantsManager); - if (AppData(ctx)->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()) { - NetClassifierUpdaterId = ctx.Register(NNetClassifierUpdater::MakeNetClassifierUpdaterActor(SelfId())); - } - + if (AppData(ctx)->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()) { + NetClassifierUpdaterId = ctx.Register(NNetClassifierUpdater::MakeNetClassifierUpdaterActor(SelfId())); + } + TxProcessor->ProcessTx(CreateTxInitScheme(), ctx); } @@ -109,11 +109,11 @@ void TConsole::Cleanup(const TActorContext &ctx) TenantsManager = nullptr; } - if (NetClassifierUpdaterId) { - Send(NetClassifierUpdaterId, new TEvents::TEvPoisonPill); - NetClassifierUpdaterId = {}; - } - + if (NetClassifierUpdaterId) { + Send(NetClassifierUpdaterId, new TEvents::TEvPoisonPill); + NetClassifierUpdaterId = {}; + } + TxProcessor->Clear(); } @@ -145,12 +145,12 @@ void TConsole::ProcessEnqueuedEvents(const TActorContext &ctx) void TConsole::ClearState() { Config.Clear(); - if (ConfigsManager) { + if (ConfigsManager) { ConfigsManager->ClearState(); - } - if (TenantsManager) { + } + if (TenantsManager) { TenantsManager->ClearState(); - } + } Counters->ResetCounters(); } diff --git a/ydb/core/cms/console/console_impl.h b/ydb/core/cms/console/console_impl.h index 0894337c3e..a42ee8ee05 100644 --- a/ydb/core/cms/console/console_impl.h +++ b/ydb/core/cms/console/console_impl.h @@ -162,9 +162,9 @@ private: TTxProcessor::TPtr TxProcessor; TDynamicCounterPtr Counters; - TConfigsManager* ConfigsManager; - TTenantsManager* TenantsManager; - + TConfigsManager* ConfigsManager; + TTenantsManager* TenantsManager; + TActorId NetClassifierUpdaterId; }; diff --git a/ydb/core/cms/console/net_classifier_updater.cpp b/ydb/core/cms/console/net_classifier_updater.cpp index 121a65efc3..4c25545f88 100644 --- a/ydb/core/cms/console/net_classifier_updater.cpp +++ b/ydb/core/cms/console/net_classifier_updater.cpp @@ -1,15 +1,15 @@ -#include "net_classifier_updater.h" - +#include "net_classifier_updater.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipe.h> - + #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/http/http_proxy.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/json/json_reader.h> - -#include <util/stream/zlib.h> - + +#include <util/stream/zlib.h> + #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR || defined BLOG_NOTICE #error log macro definition clash #endif @@ -20,159 +20,159 @@ #define BLOG_ERROR(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::CMS_CONFIGS, stream) -namespace NKikimr::NNetClassifierUpdater { - -using namespace NConsole; - -// helps to find a particular config item. DO NOT CHANGE this constant -static const TString COOKIE = "NetClassifierPackedNetDataFromExternalSource"; - -// prevents duplicates in config list. DO NOT CHANGE this constant -static constexpr ui32 CONFIG_ORDER = 10; - -static TString PackNetData(const TString& netData, size_t level = 6) { - TString result; - result.reserve(std::size(netData)); - TStringOutput stream(result); - TZLibCompress(&stream, ZLib::GZip, level).Write(netData); - return result; -} - -/* - ***NetClassifier Updater algorithm*** - - The main idea is to keep only one special config item to periodically modify it. - All discovery nodes are subscribed to config dispatcher to get updates. - - Initing stage (requires WakeUp event to start): - 1) Look for the config item via cookie - if has distributable config item: - GOTO Working stage - else: - Add distributable empty config item with cookie and order - GOTO Working stage if added successfully - Working stage (also requires WakeUp event to start): - 1) Get Networks Data from an external http(s) resource - 2) Find current config id and generation through GetConfigItem - 3) Send ModifyEvent using step 2 information. - 4) Schedule WakeUp event on success - - In case of any error algorithm falls back to Initing stage and schedules WakeUp. -*/ - -class NetClassifierUpdater : public TActorBootstrapped<NetClassifierUpdater> { -private: - using TBase = TActorBootstrapped<NetClassifierUpdater>; - -public: +namespace NKikimr::NNetClassifierUpdater { + +using namespace NConsole; + +// helps to find a particular config item. DO NOT CHANGE this constant +static const TString COOKIE = "NetClassifierPackedNetDataFromExternalSource"; + +// prevents duplicates in config list. DO NOT CHANGE this constant +static constexpr ui32 CONFIG_ORDER = 10; + +static TString PackNetData(const TString& netData, size_t level = 6) { + TString result; + result.reserve(std::size(netData)); + TStringOutput stream(result); + TZLibCompress(&stream, ZLib::GZip, level).Write(netData); + return result; +} + +/* + ***NetClassifier Updater algorithm*** + + The main idea is to keep only one special config item to periodically modify it. + All discovery nodes are subscribed to config dispatcher to get updates. + + Initing stage (requires WakeUp event to start): + 1) Look for the config item via cookie + if has distributable config item: + GOTO Working stage + else: + Add distributable empty config item with cookie and order + GOTO Working stage if added successfully + Working stage (also requires WakeUp event to start): + 1) Get Networks Data from an external http(s) resource + 2) Find current config id and generation through GetConfigItem + 3) Send ModifyEvent using step 2 information. + 4) Schedule WakeUp event on success + + In case of any error algorithm falls back to Initing stage and schedules WakeUp. +*/ + +class NetClassifierUpdater : public TActorBootstrapped<NetClassifierUpdater> { +private: + using TBase = TActorBootstrapped<NetClassifierUpdater>; + +public: NetClassifierUpdater(TActorId localConsole) - : LocalConsole(localConsole) - { - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::NET_CLASSIFIER_UPDATER; - } - + : LocalConsole(localConsole) + { + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::NET_CLASSIFIER_UPDATER; + } + void Bootstrap() { - Become(&TThis::Initing); + Become(&TThis::Initing); Send(SelfId(), new TEvents::TEvWakeup); - } - -private: - const auto& UpdaterConfig() const { + } + +private: + const auto& UpdaterConfig() const { return AppData()->NetClassifierConfig.GetUpdaterConfig(); - } - - void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) { - // start the config check - RequestCurrentConfigViaCookie(); - } - - void RequestCurrentConfigViaCookie() { + } + + void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) { + // start the config check + RequestCurrentConfigViaCookie(); + } + + void RequestCurrentConfigViaCookie() { BLOG_D("NetClassifierUpdater requested distributable config item via cookie"); - - auto event = MakeHolder<TEvConsole::TEvGetConfigItemsRequest>(); - - event->Record.AddItemKinds(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem)); - event->Record.MutableCookieFilter()->AddCookies(COOKIE); - - Send(LocalConsole, event.Release()); - } - - void InitDefaultConfiguration() { + + auto event = MakeHolder<TEvConsole::TEvGetConfigItemsRequest>(); + + event->Record.AddItemKinds(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem)); + event->Record.MutableCookieFilter()->AddCookies(COOKIE); + + Send(LocalConsole, event.Release()); + } + + void InitDefaultConfiguration() { LOG_INFO_S(*TlsActivationContext, NKikimrServices::CMS_CONFIGS, - "NetClassifierUpdate is adding distributable config item with cookie"); - - auto event = MakeHolder<TEvConsole::TEvConfigureRequest>(); - - auto& configItem = *event->Record.AddActions()->MutableAddConfigItem()->MutableConfigItem(); - configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); // just initialize the field - - configItem.SetKind(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem)); - - configItem.SetOrder(CONFIG_ORDER); // prevents config item duplicates - configItem.SetCookie(COOKIE); - - Send(LocalConsole, event.Release()); - } - - void HandleWhileIniting(TEvConsole::TEvConfigureResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { + "NetClassifierUpdate is adding distributable config item with cookie"); + + auto event = MakeHolder<TEvConsole::TEvConfigureRequest>(); + + auto& configItem = *event->Record.AddActions()->MutableAddConfigItem()->MutableConfigItem(); + configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); // just initialize the field + + configItem.SetKind(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem)); + + configItem.SetOrder(CONFIG_ORDER); // prevents config item duplicates + configItem.SetCookie(COOKIE); + + Send(LocalConsole, event.Release()); + } + + void HandleWhileIniting(TEvConsole::TEvConfigureResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { BLOG_D("NetClassifierUpdater created a new distributable config item"); - CompleteInitialization(); - } else { + CompleteInitialization(); + } else { BLOG_ERROR("NetClassifierUpdater failed to add config item: " << record.ShortDebugString()); - InitializeAgain(); - } - } - - void HandleWhileIniting(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { - if (record.ConfigItemsSize() == 0) { - // cookied config item is missing, add it - InitDefaultConfiguration(); - } else { - Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie - + InitializeAgain(); + } + } + + void HandleWhileIniting(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { + if (record.ConfigItemsSize() == 0) { + // cookied config item is missing, add it + InitDefaultConfiguration(); + } else { + Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie + BLOG_D("NetClassifierUpdater found the distributable config via cookie"); - - CompleteInitialization(); - } - } else { + + CompleteInitialization(); + } + } else { BLOG_ERROR("NetClassifierUpdater failed get current distributable config version: " << record.ShortDebugString()); - InitializeAgain(); - } - } - - STATEFN(Initing) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvConsole::TEvConfigureResponse, HandleWhileIniting); - hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileIniting); - hFunc(TEvents::TEvWakeup, HandleWhileIniting); - hFunc(TEvents::TEvPoisonPill, HandlePoison); - } - } - - void CompleteInitialization() { + InitializeAgain(); + } + } + + STATEFN(Initing) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvConsole::TEvConfigureResponse, HandleWhileIniting); + hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileIniting); + hFunc(TEvents::TEvWakeup, HandleWhileIniting); + hFunc(TEvents::TEvPoisonPill, HandlePoison); + } + } + + void CompleteInitialization() { BLOG_D("NetClassifierUpdater has been initialized"); - - Become(&TThis::Working); + + Become(&TThis::Working); Send(SelfId(), new TEvents::TEvWakeup); - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, HandleWhileWorking); - hFunc(TEvConsole::TEvConfigureResponse, HandleWhileWorking); - hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileWorking); - hFunc(TEvents::TEvWakeup, HandleWhileWorking); - hFunc(TEvents::TEvPoisonPill, HandlePoison); - } - } - + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, HandleWhileWorking); + hFunc(TEvConsole::TEvConfigureResponse, HandleWhileWorking); + hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileWorking); + hFunc(TEvents::TEvWakeup, HandleWhileWorking); + hFunc(TEvents::TEvPoisonPill, HandlePoison); + } + } + auto FormNetDataFromJson(TStringBuf jsonData) const { NKikimrNetClassifier::TNetData netData; TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end()); @@ -212,35 +212,35 @@ private: return netData; } - auto FormNetData(TStringBuf tsvData) const { - NKikimrNetClassifier::TNetData netData; - - while (auto record = tsvData.NextTok('\n')) { - auto& subnet = *netData.AddSubnets(); - subnet.SetMask(TString(record.NextTok('\t'))); - subnet.SetLabel(TString(record.NextTok('\t'))); - } - - return netData; - } - - void ScheduleNextUpdate() { - const TDuration interval = TDuration::Seconds(UpdaterConfig().GetNetDataUpdateIntervalSeconds()); - Schedule(interval, new TEvents::TEvWakeup); - } - - void InitializeAgain() { - PackedNetData = {}; - LastUpdateTimestamp = {}; - LastUpdateDatetimeUTC = {}; - - Become(&TThis::Initing); - - const TDuration interval = TDuration::Seconds(UpdaterConfig().GetRetryIntervalSeconds()); - Schedule(interval, new TEvents::TEvWakeup); - } - - void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { + auto FormNetData(TStringBuf tsvData) const { + NKikimrNetClassifier::TNetData netData; + + while (auto record = tsvData.NextTok('\n')) { + auto& subnet = *netData.AddSubnets(); + subnet.SetMask(TString(record.NextTok('\t'))); + subnet.SetLabel(TString(record.NextTok('\t'))); + } + + return netData; + } + + void ScheduleNextUpdate() { + const TDuration interval = TDuration::Seconds(UpdaterConfig().GetNetDataUpdateIntervalSeconds()); + Schedule(interval, new TEvents::TEvWakeup); + } + + void InitializeAgain() { + PackedNetData = {}; + LastUpdateTimestamp = {}; + LastUpdateDatetimeUTC = {}; + + Become(&TThis::Initing); + + const TDuration interval = TDuration::Seconds(UpdaterConfig().GetRetryIntervalSeconds()); + Schedule(interval, new TEvents::TEvWakeup); + } + + void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { if (ev->Get()->Error.empty()) { if (ev->Get()->Response->Status == "200") { const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV @@ -248,10 +248,10 @@ private: : FormNetDataFromJson(ev->Get()->Response->Body); if (netData.SubnetsSize() != 0) { PackedNetData = PackAcquiredSubnets(netData); - + LastUpdateTimestamp = TActivationContext::Now(); LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer - + // To modify the config it's essential to find the current id and generation RequestCurrentConfigViaCookie(); return; @@ -263,89 +263,89 @@ private: } } else { BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error); - } + } InitializeAgain(); - } - - void HandleWhileWorking(TEvConsole::TEvConfigureResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { - // hurray! the update is finished - ScheduleNextUpdate(); - } else { + } + + void HandleWhileWorking(TEvConsole::TEvConfigureResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { + // hurray! the update is finished + ScheduleNextUpdate(); + } else { BLOG_ERROR("NetClassifierUpdater failed to update distributable config: " << record.ShortDebugString()); - InitializeAgain(); - } - } - - void HandleWhileWorking(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { - Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie - - auto event = MakeHolder<TEvConsole::TEvConfigureRequest>(); - - auto& configItem = *event->Record.AddActions()->MutableModifyConfigItem()->MutableConfigItem(); - - // copy id, generation and cookie - configItem.CopyFrom(ev->Get()->Record.GetConfigItems(0)); - - auto& distributableConfig = *configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); - distributableConfig.SetPackedNetData(PackedNetData); - distributableConfig.SetLastUpdateDatetimeUTC(LastUpdateDatetimeUTC); - distributableConfig.SetLastUpdateTimestamp(LastUpdateTimestamp.MicroSeconds()); - - Send(LocalConsole, event.Release()); - } else { + InitializeAgain(); + } + } + + void HandleWhileWorking(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) { + Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie + + auto event = MakeHolder<TEvConsole::TEvConfigureRequest>(); + + auto& configItem = *event->Record.AddActions()->MutableModifyConfigItem()->MutableConfigItem(); + + // copy id, generation and cookie + configItem.CopyFrom(ev->Get()->Record.GetConfigItems(0)); + + auto& distributableConfig = *configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); + distributableConfig.SetPackedNetData(PackedNetData); + distributableConfig.SetLastUpdateDatetimeUTC(LastUpdateDatetimeUTC); + distributableConfig.SetLastUpdateTimestamp(LastUpdateTimestamp.MicroSeconds()); + + Send(LocalConsole, event.Release()); + } else { BLOG_ERROR("NetClassifierUpdater failed to get current distributable config version: " << record.ShortDebugString()); - InitializeAgain(); - } - } - - TString PackAcquiredSubnets(const NKikimrNetClassifier::TNetData& netData) const { - TString serializedProto; + InitializeAgain(); + } + } + + TString PackAcquiredSubnets(const NKikimrNetClassifier::TNetData& netData) const { + TString serializedProto; Y_PROTOBUF_SUPPRESS_NODISCARD netData.SerializeToString(&serializedProto); - - return PackNetData(serializedProto); - } - - void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) { - // it's high time to update the networks data! - if (!HttpProxyId) { - HttpProxyId = Register(NHttp::CreateHttpProxy(HttpSensors)); - } - - NHttp::THttpOutgoingRequestPtr httpRequest = - NHttp::THttpOutgoingRequest::CreateRequestGet(UpdaterConfig().GetNetDataSourceUrl()); - - Send(HttpProxyId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)); - } - - void HandlePoison(TEvents::TEvPoisonPill::TPtr&) { - PassAway(); - } - -private: + + return PackNetData(serializedProto); + } + + void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) { + // it's high time to update the networks data! + if (!HttpProxyId) { + HttpProxyId = Register(NHttp::CreateHttpProxy(HttpSensors)); + } + + NHttp::THttpOutgoingRequestPtr httpRequest = + NHttp::THttpOutgoingRequest::CreateRequestGet(UpdaterConfig().GetNetDataSourceUrl()); + + Send(HttpProxyId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)); + } + + void HandlePoison(TEvents::TEvPoisonPill::TPtr&) { + PassAway(); + } + +private: TActorId LocalConsole; TActorId HttpProxyId; NMonitoring::TMetricRegistry HttpSensors; - - TString PackedNetData; - TString LastUpdateDatetimeUTC; - TInstant LastUpdateTimestamp; -}; - + + TString PackedNetData; + TString LastUpdateDatetimeUTC; + TInstant LastUpdateTimestamp; +}; + IActor* MakeNetClassifierUpdaterActor(TActorId localConsole) { - return new NetClassifierUpdater(localConsole); -} - -TString UnpackNetData(const TString& packedNetData) { - TStringInput stream(packedNetData); - try { - return TZLibDecompress(&stream, ZLib::GZip).ReadAll(); - } catch (const yexception& e) { - return {}; - } -} - -} // namespace NKikimr::NNetClassifierUpdater + return new NetClassifierUpdater(localConsole); +} + +TString UnpackNetData(const TString& packedNetData) { + TStringInput stream(packedNetData); + try { + return TZLibDecompress(&stream, ZLib::GZip).ReadAll(); + } catch (const yexception& e) { + return {}; + } +} + +} // namespace NKikimr::NNetClassifierUpdater diff --git a/ydb/core/cms/console/net_classifier_updater.h b/ydb/core/cms/console/net_classifier_updater.h index b0ebcd63d9..135b33977b 100644 --- a/ydb/core/cms/console/net_classifier_updater.h +++ b/ydb/core/cms/console/net_classifier_updater.h @@ -1,12 +1,12 @@ -#pragma once - -#include "defs.h" -#include "console.h" - -namespace NKikimr::NNetClassifierUpdater { - +#pragma once + +#include "defs.h" +#include "console.h" + +namespace NKikimr::NNetClassifierUpdater { + IActor* MakeNetClassifierUpdaterActor(TActorId localConsole); - -TString UnpackNetData(const TString& packedNetData); - -} // namespace NKikimr::NNetClassifierUpdater + +TString UnpackNetData(const TString& packedNetData); + +} // namespace NKikimr::NNetClassifierUpdater diff --git a/ydb/core/cms/console/net_classifier_updater_ut.cpp b/ydb/core/cms/console/net_classifier_updater_ut.cpp index da41ee51f2..0561ab742a 100644 --- a/ydb/core/cms/console/net_classifier_updater_ut.cpp +++ b/ydb/core/cms/console/net_classifier_updater_ut.cpp @@ -3,89 +3,89 @@ #include <ydb/core/cms/console/console.h> #include <ydb/core/cms/console/net_classifier_updater.h> #include <ydb/core/testlib/test_client.h> - + #include <library/cpp/actors/http/http_proxy.cpp> - + #include <library/cpp/protobuf/util/is_equal.h> - + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/json/json_writer.h> - -#include <util/string/builder.h> - -namespace NKikimr::NNetClassifierUpdaterTests { - -using namespace NConsole; -using namespace Tests; - + +#include <util/string/builder.h> + +namespace NKikimr::NNetClassifierUpdaterTests { + +using namespace NConsole; +using namespace Tests; + using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig; const TString NETWORKS_URI = "/fancy_path/networks.tsv"; -static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) { - const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: " - << netData.size() << "\r\n\r\n" << netData; - NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content); - - return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse); -} - -template<typename TDistributableConfProto> -bool CheckDistributableConfig(const TDistributableConfProto& config, const NKikimrNetClassifier::TNetData& expectedNetData) { - if (config.GetPackedNetData()) { - UNIT_ASSERT(config.GetLastUpdateDatetimeUTC()); - UNIT_ASSERT(config.GetLastUpdateTimestamp()); - - UNIT_ASSERT_STRINGS_EQUAL( - TInstant::MicroSeconds(config.GetLastUpdateTimestamp()).ToRfc822String(), - config.GetLastUpdateDatetimeUTC() - ); - - const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(config.GetPackedNetData()); - UNIT_ASSERT(serializedNetData); - - NKikimrNetClassifier::TNetData netData; - UNIT_ASSERT(netData.ParseFromString(serializedNetData)); - - UNIT_ASSERT(NProtoBuf::IsEqual(netData, expectedNetData)); - - return true; - } - - return false; -} - -static NKikimrNetClassifier::TNetData FormNetData() { - NKikimrNetClassifier::TNetData netData; - - { - auto& subnet = *netData.AddSubnets(); - subnet.SetMask("2a02:6b8:fc0a::/48"); - subnet.SetLabel("SAS"); - } - { - auto& subnet = *netData.AddSubnets(); - subnet.SetMask("87.250.239.224/31"); - subnet.SetLabel("VLA"); - } - - return netData; -} - -static TString ConvertToTsv(const NKikimrNetClassifier::TNetData& netData) { - TStringBuilder builder; - - for (size_t i = 0; i < netData.SubnetsSize(); ++i) { - const auto& subnet = netData.GetSubnets(i); - if (i) { - builder << "\n"; - } - builder << subnet.GetMask() << "\t" << subnet.GetLabel(); - } - - return builder; -} - +static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) { + const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: " + << netData.size() << "\r\n\r\n" << netData; + NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content); + + return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse); +} + +template<typename TDistributableConfProto> +bool CheckDistributableConfig(const TDistributableConfProto& config, const NKikimrNetClassifier::TNetData& expectedNetData) { + if (config.GetPackedNetData()) { + UNIT_ASSERT(config.GetLastUpdateDatetimeUTC()); + UNIT_ASSERT(config.GetLastUpdateTimestamp()); + + UNIT_ASSERT_STRINGS_EQUAL( + TInstant::MicroSeconds(config.GetLastUpdateTimestamp()).ToRfc822String(), + config.GetLastUpdateDatetimeUTC() + ); + + const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(config.GetPackedNetData()); + UNIT_ASSERT(serializedNetData); + + NKikimrNetClassifier::TNetData netData; + UNIT_ASSERT(netData.ParseFromString(serializedNetData)); + + UNIT_ASSERT(NProtoBuf::IsEqual(netData, expectedNetData)); + + return true; + } + + return false; +} + +static NKikimrNetClassifier::TNetData FormNetData() { + NKikimrNetClassifier::TNetData netData; + + { + auto& subnet = *netData.AddSubnets(); + subnet.SetMask("2a02:6b8:fc0a::/48"); + subnet.SetLabel("SAS"); + } + { + auto& subnet = *netData.AddSubnets(); + subnet.SetMask("87.250.239.224/31"); + subnet.SetLabel("VLA"); + } + + return netData; +} + +static TString ConvertToTsv(const NKikimrNetClassifier::TNetData& netData) { + TStringBuilder builder; + + for (size_t i = 0; i < netData.SubnetsSize(); ++i) { + const auto& subnet = netData.GetSubnets(i); + if (i) { + builder << "\n"; + } + builder << subnet.GetMask() << "\t" << subnet.GetLabel(); + } + + return builder; +} + static TString ConvertToJson(const NKikimrNetClassifier::TNetData& netData) { @@ -126,7 +126,7 @@ NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig( return updaterConfig; } -Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { +Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { void TestGetUpdatesFromHttpServer( const TString& sourceResponce, const NKikimrNetClassifier::TNetData& expectedNetData, @@ -134,54 +134,54 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { const TVector<TString>& netBoxTags = {} ) { NMonitoring::TMetricRegistry sensors; - - TPortManager pm; - const ui16 port = pm.GetPort(2134); - const ui64 netDataSourcePort = pm.GetPort(13334); - TServerSettings settings(port); - auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig(); + + TPortManager pm; + const ui16 port = pm.GetPort(2134); + const ui64 netDataSourcePort = pm.GetPort(13334); + TServerSettings settings(port); + auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig(); updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags); - TServer cleverServer = TServer(settings); - auto& actorSystem = *cleverServer.GetRuntime(); - - NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors); + TServer cleverServer = TServer(settings); + auto& actorSystem = *cleverServer.GetRuntime(); + + NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors); NActors::TActorId proxyId = actorSystem.Register(proxy); - - actorSystem.Send( + + actorSystem.Send( new NActors::IEventHandle(proxyId, TActorId(), new NHttp::TEvHttpProxy::TEvAddListeningPort(netDataSourcePort)), 0, true - ); - + ); + NActors::TActorId serverId = actorSystem.AllocateEdgeActor(); - + actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true); - TAutoPtr<NActors::IEventHandle> handle; - NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); + TAutoPtr<NActors::IEventHandle> handle; + NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI); - + actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true); const TActorId sender = actorSystem.AllocateEdgeActor(); - - size_t iterations = 0; - while (true) { - UNIT_ASSERT(++iterations < 60); - - const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); - actorSystem.Send( - new IEventHandle(MakeConfigsDispatcherID(sender.NodeId()), sender, - new TEvConfigsDispatcher::TEvGetConfigRequest(kind) - )); - - const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle); - + + size_t iterations = 0; + while (true) { + UNIT_ASSERT(++iterations < 60); + + const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); + actorSystem.Send( + new IEventHandle(MakeConfigsDispatcherID(sender.NodeId()), sender, + new TEvConfigsDispatcher::TEvGetConfigRequest(kind) + )); + + const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle); + if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) { - break; - } - - // wait for the proper update - Sleep(TDuration::Seconds(1)); - } - } + break; + } + + // wait for the proper update + Sleep(TDuration::Seconds(1)); + } + } Y_UNIT_TEST(TestGetUpdatesFromHttpServer) { auto netData = FormNetData(); @@ -252,6 +252,6 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"}); } } -} - -} // namespace NKikimr::NNetClassifierUpdaterTests +} + +} // namespace NKikimr::NNetClassifierUpdaterTests diff --git a/ydb/core/cms/console/ut/ya.make b/ydb/core/cms/console/ut/ya.make index 9f42e18427..377edf66d4 100644 --- a/ydb/core/cms/console/ut/ya.make +++ b/ydb/core/cms/console/ut/ya.make @@ -34,7 +34,7 @@ SRCS( immediate_controls_configurator_ut.cpp log_settings_configurator_ut.cpp modifications_validator_ut.cpp - net_classifier_updater_ut.cpp + net_classifier_updater_ut.cpp ) END() diff --git a/ydb/core/cms/console/ya.make b/ydb/core/cms/console/ya.make index e33d3fc19a..8f8144a027 100644 --- a/ydb/core/cms/console/ya.make +++ b/ydb/core/cms/console/ya.make @@ -59,7 +59,7 @@ SRCS( log_settings_configurator.h modifications_validator.cpp modifications_validator.h - net_classifier_updater.cpp + net_classifier_updater.cpp shared_cache_configurator.cpp shared_cache_configurator.h tx_processor.cpp diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h index 099664792e..a85a38220a 100644 --- a/ydb/core/driver_lib/cli_utils/cli.h +++ b/ydb/core/driver_lib/cli_utils/cli.h @@ -34,7 +34,7 @@ namespace NDriverClient { int KeyValueRequest(TCommandConfig &cmdConf, int argc, char **argv); int PersQueueRequest(TCommandConfig &cmdConf, int argc, char **argv); int PersQueueStress(TCommandConfig &cmdConf, int argc, char **argv); - int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv); + int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv); int LoadRequest(TCommandConfig &cmdConf, int argc, char **argv); int ActorsysPerfTest(TCommandConfig &cmdConf, int argc, char **argv); void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp index 0497d4ea8f..0ba9640b2d 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp @@ -119,8 +119,8 @@ public: ConfigFiles["FeatureFlags"] = "feature_flags.txt"; ConfigFiles["SqsConfig"] = "sqs.txt"; ConfigFiles["PQConfig"] = "pq.txt"; - ConfigFiles["PQClusterDiscoveryConfig"] = "pqcd.txt"; - ConfigFiles["NetClassifierConfig"] = "netclassifier.txt"; + ConfigFiles["PQClusterDiscoveryConfig"] = "pqcd.txt"; + ConfigFiles["NetClassifierConfig"] = "netclassifier.txt"; ConfigFiles["KeyConfig"] = "key.txt"; ConfigFiles["PDiskKeyConfig"] = "pdisk_key.txt"; ConfigFiles["ClusterYamlConfig"] = "cluster.yaml"; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 6766dd3171..9c5aa29c66 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -209,8 +209,8 @@ protected: config.Opts->AddLongOption("incrhuge-file", "incremental huge blob keeper config file").OptionalArgument("PATH"); config.Opts->AddLongOption("memorylog-file", "set buffer size for memory log").OptionalArgument("PATH"); config.Opts->AddLongOption("pq-file", "PersQueue config file").OptionalArgument("PATH"); - config.Opts->AddLongOption("pqcd-file", "PersQueue cluster discovery config file").OptionalArgument("PATH"); - config.Opts->AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH"); + config.Opts->AddLongOption("pqcd-file", "PersQueue cluster discovery config file").OptionalArgument("PATH"); + config.Opts->AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH"); config.Opts->AddLongOption("auth-file", "authorization configuration").OptionalArgument("PATH"); config.Opts->AddLongOption("auth-token-file", "authorization token configuration").OptionalArgument("PATH"); config.Opts->AddLongOption("key-file", "tanant encryption key configuration").OptionalArgument("PATH"); @@ -422,8 +422,8 @@ protected: OPTION("dyn-nodes-file", DynamicNameserviceConfig); OPTION("cms-file", CmsConfig); OPTION("pq-file", PQConfig); - OPTION("pqcd-file", PQClusterDiscoveryConfig); - OPTION("netclassifier-file", NetClassifierConfig); + OPTION("pqcd-file", PQClusterDiscoveryConfig); + OPTION("netclassifier-file", NetClassifierConfig); OPTION("auth-file", AuthConfig); OPTION_MERGE("auth-token-file", AuthConfig); OPTION("key-file", KeyConfig); diff --git a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp index 79800a67da..01f3f12345 100644 --- a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp @@ -1,87 +1,87 @@ -#include "cli.h" - +#include "cli.h" + #include <ydb/core/driver_lib/cli_base/cli_grpc.h> - + #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> - + #include <library/cpp/protobuf/json/proto2json.h> - - -namespace NKikimr::NDriverClient { - -struct TCmdPersQueueDiscoverWriteSessionClustersConfig final : public TCliCmdConfig { - void Parse(int argc, char** argv) { - using namespace NLastGetopt; - - TOpts opts = TOpts::Default(); - - opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic); - opts.AddLongOption("source-id", "source id").Required().StoreResult(&SourceId); - opts.AddLongOption("partition-group", "partition group").Optional().StoreResult(&PartitionGroup); - opts.AddLongOption("preferred-cluster", "prioritized cluster's name").Optional().StoreResult(&PreferredCluster); - - opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose); - - ConfigureBaseLastGetopt(opts); - - TOptsParseResult res(&opts, argc, argv); - ConfigureMsgBusLastGetopt(res, argc, argv); - } - - TString Topic; - TString SourceId; - ui32 PartitionGroup = 0; - TString PreferredCluster; - - bool IsVerbose = false; -}; - -struct TCmdPersQueueDiscoverReadSessionClustersConfig final : public TCliCmdConfig { - void Parse(int argc, char** argv) { - using namespace NLastGetopt; - - TOpts opts = TOpts::Default(); - - opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic); - opts.AddLongOption("mirror-to-cluster", "do not set the option to enable all-original mode").Optional().StoreResult(&MirrorToCluster); - - opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose); - - ConfigureBaseLastGetopt(opts); - - TOptsParseResult res(&opts, argc, argv); - ConfigureMsgBusLastGetopt(res, argc, argv); - } - - TString Topic; - TString MirrorToCluster; - - bool IsVerbose = false; -}; - + + +namespace NKikimr::NDriverClient { + +struct TCmdPersQueueDiscoverWriteSessionClustersConfig final : public TCliCmdConfig { + void Parse(int argc, char** argv) { + using namespace NLastGetopt; + + TOpts opts = TOpts::Default(); + + opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic); + opts.AddLongOption("source-id", "source id").Required().StoreResult(&SourceId); + opts.AddLongOption("partition-group", "partition group").Optional().StoreResult(&PartitionGroup); + opts.AddLongOption("preferred-cluster", "prioritized cluster's name").Optional().StoreResult(&PreferredCluster); + + opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose); + + ConfigureBaseLastGetopt(opts); + + TOptsParseResult res(&opts, argc, argv); + ConfigureMsgBusLastGetopt(res, argc, argv); + } + + TString Topic; + TString SourceId; + ui32 PartitionGroup = 0; + TString PreferredCluster; + + bool IsVerbose = false; +}; + +struct TCmdPersQueueDiscoverReadSessionClustersConfig final : public TCliCmdConfig { + void Parse(int argc, char** argv) { + using namespace NLastGetopt; + + TOpts opts = TOpts::Default(); + + opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic); + opts.AddLongOption("mirror-to-cluster", "do not set the option to enable all-original mode").Optional().StoreResult(&MirrorToCluster); + + opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose); + + ConfigureBaseLastGetopt(opts); + + TOptsParseResult res(&opts, argc, argv); + ConfigureMsgBusLastGetopt(res, argc, argv); + } + + TString Topic; + TString MirrorToCluster; + + bool IsVerbose = false; +}; + static void PrintGRpcConfigAndRequest(const NGrpc::TGRpcClientConfig& grpcConfig, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest& request) { - Cerr << "Request endpoint: " << grpcConfig.Locator << " " - << "timeout: " << grpcConfig.Timeout << " " - << "max message size: " << grpcConfig.MaxMessageSize << Endl; - - Cerr << "Request message: " << NProtobufJson::Proto2Json(request, {}) << Endl; -} - + Cerr << "Request endpoint: " << grpcConfig.Locator << " " + << "timeout: " << grpcConfig.Timeout << " " + << "max message size: " << grpcConfig.MaxMessageSize << Endl; + + Cerr << "Request message: " << NProtobufJson::Proto2Json(request, {}) << Endl; +} + template <class TConfig> static int MakeRequest(const TConfig& config, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest& request) { - if (config.ClientConfig.Defined()) { + if (config.ClientConfig.Defined()) { if (std::holds_alternative<NGrpc::TGRpcClientConfig>(*config.ClientConfig)) { const auto& grpcConfig = std::get<NGrpc::TGRpcClientConfig>(*config.ClientConfig); - - if (config.IsVerbose) { - PrintGRpcConfigAndRequest(grpcConfig, request); - } - - Ydb::Operations::Operation response; + + if (config.IsVerbose) { + PrintGRpcConfigAndRequest(grpcConfig, request); + } + + Ydb::Operations::Operation response; const int res = DoGRpcRequest<Ydb::PersQueue::V1::ClusterDiscoveryService, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse, @@ -90,67 +90,67 @@ static int MakeRequest(const TConfig& config, const Ydb::PersQueue::ClusterDisco request, response, &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, ""); - - if (res == 0) { + + if (res == 0) { Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult discoveryResult; - Y_VERIFY(response.result().UnpackTo(&discoveryResult)); - - Y_VERIFY(response.ready()); // there's nothing to wait for - - Cerr << "Code: " << ToString(response.status()) << ". Result message: " << NProtobufJson::Proto2Json(discoveryResult, {}) << Endl; - return 0; - } - } - - } - - return 1; -} - -int PersQueueDiscoverClustersRequest(TCommandConfig&, int argc, char** argv) { - Y_VERIFY(argc > 2); - - if (argv[1] == TStringBuf("write-session")) { - TCmdPersQueueDiscoverWriteSessionClustersConfig config; - config.Parse(argc - 1, argv + 1); - - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; - auto* writeSessionParams = request.add_write_sessions(); - - writeSessionParams->set_topic(config.Topic); - writeSessionParams->set_source_id(config.SourceId); - writeSessionParams->set_partition_group(config.PartitionGroup); - - if (config.PreferredCluster) { - writeSessionParams->set_preferred_cluster_name(config.PreferredCluster); - } - - request.set_minimal_version(0); - + Y_VERIFY(response.result().UnpackTo(&discoveryResult)); + + Y_VERIFY(response.ready()); // there's nothing to wait for + + Cerr << "Code: " << ToString(response.status()) << ". Result message: " << NProtobufJson::Proto2Json(discoveryResult, {}) << Endl; + return 0; + } + } + + } + + return 1; +} + +int PersQueueDiscoverClustersRequest(TCommandConfig&, int argc, char** argv) { + Y_VERIFY(argc > 2); + + if (argv[1] == TStringBuf("write-session")) { + TCmdPersQueueDiscoverWriteSessionClustersConfig config; + config.Parse(argc - 1, argv + 1); + + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; + auto* writeSessionParams = request.add_write_sessions(); + + writeSessionParams->set_topic(config.Topic); + writeSessionParams->set_source_id(config.SourceId); + writeSessionParams->set_partition_group(config.PartitionGroup); + + if (config.PreferredCluster) { + writeSessionParams->set_preferred_cluster_name(config.PreferredCluster); + } + + request.set_minimal_version(0); + return MakeRequest(config, request); - } else if (argv[1] == TStringBuf("read-session")) { - TCmdPersQueueDiscoverReadSessionClustersConfig config; - config.Parse(argc - 1, argv + 1); - - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; - auto* readSessionParams = request.add_read_sessions(); - - readSessionParams->set_topic(config.Topic); - - if (config.MirrorToCluster) { - readSessionParams->set_mirror_to_cluster(config.MirrorToCluster); - } else { - readSessionParams->mutable_all_original(); // set the field - } - - request.set_minimal_version(0); - + } else if (argv[1] == TStringBuf("read-session")) { + TCmdPersQueueDiscoverReadSessionClustersConfig config; + config.Parse(argc - 1, argv + 1); + + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; + auto* readSessionParams = request.add_read_sessions(); + + readSessionParams->set_topic(config.Topic); + + if (config.MirrorToCluster) { + readSessionParams->set_mirror_to_cluster(config.MirrorToCluster); + } else { + readSessionParams->mutable_all_original(); // set the field + } + + request.set_minimal_version(0); + return MakeRequest(config, request); - } - - Cerr << "Bad commands params" << Endl; - return 1; -} - -} // namespace NKikimr::NDriverClient + } + + Cerr << "Bad commands params" << Endl; + return 1; +} + +} // namespace NKikimr::NDriverClient diff --git a/ydb/core/driver_lib/cli_utils/ya.make b/ydb/core/driver_lib/cli_utils/ya.make index cb1f9670be..c90b10db65 100644 --- a/ydb/core/driver_lib/cli_utils/ya.make +++ b/ydb/core/driver_lib/cli_utils/ya.make @@ -27,7 +27,7 @@ SRCS( cli_fakeinitshard.cpp cli_keyvalue.cpp cli_persqueue.cpp - cli_persqueue_cluster_discovery.cpp + cli_persqueue_cluster_discovery.cpp cli_persqueue_stress.cpp cli_load.cpp cli_minikql_compile_and_exec.cpp diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index faf1797413..8cafc1f9bf 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -53,9 +53,9 @@ union TBasicKikimrServicesMask { bool EnableSecurityServices:1; bool EnableTabletInfo:1; bool EnableQuoterService:1; - bool EnablePersQueueClusterDiscovery:1; - bool EnableNetClassifier:1; - bool EnablePersQueueClusterTracker:1; + bool EnablePersQueueClusterDiscovery:1; + bool EnableNetClassifier:1; + bool EnablePersQueueClusterTracker:1; bool EnableSysViewService:1; bool EnableMeteringWriter:1; bool EnableSchemeBoardMonitoring:1; diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index bb68656291..160efe3232 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -66,8 +66,8 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT"); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT"); opts.AddLongOption("pq-file", "PQ config file").OptionalArgument("PATH"); - opts.AddLongOption("pqcd-file", "PQCD config file").OptionalArgument("PATH"); - opts.AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH"); + opts.AddLongOption("pqcd-file", "PQCD config file").OptionalArgument("PATH"); + opts.AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH"); opts.AddLongOption("auth-file", "authorization config file").OptionalArgument("PATH"); opts.AddLongOption("auth-token-file", "authorization token config file").OptionalArgument("PATH"); opts.AddLongOption("key-file", "encryption key config file").OptionalArgument("PATH"); @@ -172,14 +172,14 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu Y_VERIFY(ParsePBFromFile(res.Get("pq-file"), Config.AppConfig.MutablePQConfig())); } - if (res.Has("pqcd-file")) { - Y_VERIFY(ParsePBFromFile(res.Get("pqcd-file"), Config.AppConfig.MutablePQClusterDiscoveryConfig())); - } - - if (res.Has("netclassifier-file")) { - Y_VERIFY(ParsePBFromFile(res.Get("netclassifier-file"), Config.AppConfig.MutableNetClassifierConfig())); - } - + if (res.Has("pqcd-file")) { + Y_VERIFY(ParsePBFromFile(res.Get("pqcd-file"), Config.AppConfig.MutablePQClusterDiscoveryConfig())); + } + + if (res.Has("netclassifier-file")) { + Y_VERIFY(ParsePBFromFile(res.Get("netclassifier-file"), Config.AppConfig.MutableNetClassifierConfig())); + } + if (res.Has("auth-file")) { Y_VERIFY(ParsePBFromFile(res.Get("auth-file"), Config.AppConfig.MutableAuthConfig())); } diff --git a/ydb/core/driver_lib/run/driver.h b/ydb/core/driver_lib/run/driver.h index 7ad96718d1..80cc27cfe9 100644 --- a/ydb/core/driver_lib/run/driver.h +++ b/ydb/core/driver_lib/run/driver.h @@ -34,7 +34,7 @@ namespace NKikimr { XX(EDM_KEYVALUE_REQUEST, "keyvalue-request", "send protobuf request to a keyvalue tablet") \ XX(EDM_PERSQUEUE_REQUEST, "persqueue-request", "send protobuf request to a persqueue tablet") \ XX(EDM_PERSQUEUE_STRESS, "persqueue-stress", "stress read or write to a persqueue tablet") \ - XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \ + XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \ XX(EDM_LOAD_REQUEST, "bs-load-test", "send protobuf request to blobstorage test load actor (https://wiki.yandex-team.ru/kikimr/developers/BSLoadTest/)") \ XX(EDM_ACTORSYS_PERFTEST, "actorsys-perf-test", "make actorsystem performance test") \ diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 819c1478d1..3c4c21858e 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -51,7 +51,7 @@ #include <ydb/core/kesus/tablet/tablet.h> #include <ydb/core/keyvalue/keyvalue.h> - + #include <ydb/core/test_tablet/test_tablet.h> #include <ydb/core/test_tablet/state_server_interface.h> @@ -142,7 +142,7 @@ #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> #include <library/cpp/actors/protos/services_common.pb.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/actors/core/event_local.h> @@ -1833,33 +1833,33 @@ void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); } -// TNetClassifierInitializer - -TNetClassifierInitializer::TNetClassifierInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - -void TNetClassifierInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - IActor* actor = NNetClassifier::CreateNetClassifier(); - +// TNetClassifierInitializer + +TNetClassifierInitializer::TNetClassifierInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + +void TNetClassifierInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + IActor* actor = NNetClassifier::CreateNetClassifier(); + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( - NNetClassifier::MakeNetClassifierID(), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); -} - -// TPersQueueClusterTracker - -TPersQueueClusterTrackerInitializer::TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - -void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - IActor* actor = NPQ::NClusterTracker::CreateClusterTracker(); + NNetClassifier::MakeNetClassifierID(), + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); +} + +// TPersQueueClusterTracker + +TPersQueueClusterTrackerInitializer::TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + +void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + IActor* actor = NPQ::NClusterTracker::CreateClusterTracker(); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( - NPQ::NClusterTracker::MakeClusterTrackerID(), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); -} - + NPQ::NClusterTracker::MakeClusterTrackerID(), + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); +} + // TPersQueueLibSharedInstanceInitializer TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig) @@ -2108,7 +2108,7 @@ void TSqsServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu NSQS::MakeSqsProxyServiceID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); } - + Factories->SqsAuthFactory->Initialize( setup->LocalServices, *appData, Config.GetSqsConfig()); } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 407ce1bb7b..0bdfc3a238 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -336,20 +336,20 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TNetClassifierInitializer : public IKikimrServicesInitializer { -public: - TNetClassifierInitializer(const TKikimrRunConfig& runConfig); +class TNetClassifierInitializer : public IKikimrServicesInitializer { +public: + TNetClassifierInitializer(const TKikimrRunConfig& runConfig); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - -class TPersQueueClusterTrackerInitializer : public IKikimrServicesInitializer { -public: - TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig); +}; + +class TPersQueueClusterTrackerInitializer : public IKikimrServicesInitializer { +public: + TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +}; + class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer { public: TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index f0a9315685..24d9b36cf7 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -138,8 +138,8 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> return NDriverClient::PersQueueRequest(cmdConf, argc, argv); case EDM_PERSQUEUE_STRESS: return NDriverClient::PersQueueStress(cmdConf, argc, argv); - case EDM_PERSQUEUE_DISCOVER_CLUSTERS: - return NDriverClient::PersQueueDiscoverClustersRequest(cmdConf, argc, argv); + case EDM_PERSQUEUE_DISCOVER_CLUSTERS: + return NDriverClient::PersQueueDiscoverClustersRequest(cmdConf, argc, argv); case EDM_LOAD_REQUEST: return NDriverClient::LoadRequest(cmdConf, argc, argv); case EDM_ACTORSYS_PERFTEST: diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index a4f74aa4e0..37d215b0d2 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -513,7 +513,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["pq"] = &hasPQ; bool hasPQv1 = false; names["pqv1"] = &hasPQv1; - bool hasPQCD = false; + bool hasPQCD = false; names["pqcd"] = &hasPQCD; bool hasS3Internal = false; names["s3_internal"] = &hasS3Internal; @@ -541,7 +541,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["auth"] = &hasAuth; std::unordered_set<TString> enabled; - for (const auto& name : services) { + for (const auto& name : services) { enabled.insert(name); } for (const auto& name : grpcConfig.GetServicesEnabled()) { @@ -677,8 +677,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); } - if (hasPQCD) { - // the service has its own flag since it should be capable of using custom grpc port + if (hasPQCD) { + // the service has its own flag since it should be capable of using custom grpc port const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig; TMaybe<ui64> inflightLimit = Nothing(); if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) { @@ -687,8 +687,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService( ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit )); - } - + } + if (hasCms) { server.AddService(new NGRpcService::TGRpcCmsService(ActorSystem.Get(), Counters, grpcRequestProxyId)); } @@ -870,33 +870,33 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->LocalScopeId = runConfig.ScopeId; // setup streaming config - if (runConfig.AppConfig.GetGRpcConfig().HasStreamingConfig()) { + if (runConfig.AppConfig.GetGRpcConfig().HasStreamingConfig()) { AppData->StreamingConfig.CopyFrom(runConfig.AppConfig.GetGRpcConfig().GetStreamingConfig()); - } + } - if (runConfig.AppConfig.HasPQConfig()) { + if (runConfig.AppConfig.HasPQConfig()) { AppData->PQConfig.CopyFrom(runConfig.AppConfig.GetPQConfig()); - } - - if (runConfig.AppConfig.HasPQClusterDiscoveryConfig()) { - AppData->PQClusterDiscoveryConfig.CopyFrom(runConfig.AppConfig.GetPQClusterDiscoveryConfig()); - } - - if (runConfig.AppConfig.HasNetClassifierConfig()) { - AppData->NetClassifierConfig.CopyFrom(runConfig.AppConfig.GetNetClassifierConfig()); - } - - if (runConfig.AppConfig.HasSqsConfig()) { - AppData->SqsConfig.CopyFrom(runConfig.AppConfig.GetSqsConfig()); - } - - if (runConfig.AppConfig.HasAuthConfig()) { + } + + if (runConfig.AppConfig.HasPQClusterDiscoveryConfig()) { + AppData->PQClusterDiscoveryConfig.CopyFrom(runConfig.AppConfig.GetPQClusterDiscoveryConfig()); + } + + if (runConfig.AppConfig.HasNetClassifierConfig()) { + AppData->NetClassifierConfig.CopyFrom(runConfig.AppConfig.GetNetClassifierConfig()); + } + + if (runConfig.AppConfig.HasSqsConfig()) { + AppData->SqsConfig.CopyFrom(runConfig.AppConfig.GetSqsConfig()); + } + + if (runConfig.AppConfig.HasAuthConfig()) { AppData->AuthConfig.CopyFrom(runConfig.AppConfig.GetAuthConfig()); - } + } - if (runConfig.AppConfig.HasKeyConfig()) { + if (runConfig.AppConfig.HasKeyConfig()) { AppData->KeyConfig.CopyFrom(runConfig.AppConfig.GetKeyConfig()); - } + } if (runConfig.AppConfig.HasPDiskKeyConfig()) { AppData->PDiskKeyConfig.CopyFrom(runConfig.AppConfig.GetPDiskKeyConfig()); @@ -1103,12 +1103,12 @@ void TKikimrRunner::InitializeActorSystem( } } - if (servicesMask.EnableSqs && AppData->SqsConfig.GetEnableSqs()) { - if (AppData->SqsConfig.GetHttpServerConfig().GetPort()) { - - SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig)); + if (servicesMask.EnableSqs && AppData->SqsConfig.GetEnableSqs()) { + if (AppData->SqsConfig.GetHttpServerConfig().GetPort()) { + + SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig)); SqsHttp->Initialize(ActorSystem.Get(), - GetServiceCounters(AppData->Counters, "sqs"), + GetServiceCounters(AppData->Counters, "sqs"), GetServiceCounters(AppData->Counters, "ymq_public"), AppData->UserPoolId); } @@ -1253,12 +1253,12 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers if (serviceMask.EnablePersQueueL2Cache) { sil->AddServiceInitializer(new TPersQueueL2CacheInitializer(runConfig)); } - if (serviceMask.EnableNetClassifier) { - sil->AddServiceInitializer(new TNetClassifierInitializer(runConfig)); - } - if (serviceMask.EnablePersQueueClusterTracker) { - sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig)); - } + if (serviceMask.EnableNetClassifier) { + sil->AddServiceInitializer(new TNetClassifierInitializer(runConfig)); + } + if (serviceMask.EnablePersQueueClusterTracker) { + sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig)); + } sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig)); diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 44b25c4a5f..69a88b388f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -104,7 +104,7 @@ struct TRpcServices { EvPQReadInfo, EvListOperations, EvExportToYt, - EvDiscoverPQClusters, + EvDiscoverPQClusters, EvBulkUpsert, EvWhoAmI, EvKikhouseDescribeTable, diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index 97315f6e9f..b4bc022404 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -100,7 +100,7 @@ protected: void Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx); void Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx); void Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvBulkUpsertRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCreateRateLimiterResource::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/mind/address_classification/counters.cpp b/ydb/core/mind/address_classification/counters.cpp index 7491c4251e..1aa6b04759 100644 --- a/ydb/core/mind/address_classification/counters.cpp +++ b/ydb/core/mind/address_classification/counters.cpp @@ -1,19 +1,19 @@ -#include "counters.h" - -namespace NKikimr::NNetClassifier::NCounters { - -#define SETUP_SIMPLE_COUNTER(name, derived) \ - name(counters->GetCounter(#name, derived)) - -TNetClassifierCounters::TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters) - : SETUP_SIMPLE_COUNTER(GoodConfigNotificationsCount, true) - , SETUP_SIMPLE_COUNTER(BrokenConfigNotificationsCount, true) - , SETUP_SIMPLE_COUNTER(SubscribersCount, false) - , SETUP_SIMPLE_COUNTER(NetDataSourceType, false) - , SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false) -{ -} - -#undef SETUP_SIMPLE_COUNTER - -} // namespace NKikimr::NNetClassifier::NCounters +#include "counters.h" + +namespace NKikimr::NNetClassifier::NCounters { + +#define SETUP_SIMPLE_COUNTER(name, derived) \ + name(counters->GetCounter(#name, derived)) + +TNetClassifierCounters::TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters) + : SETUP_SIMPLE_COUNTER(GoodConfigNotificationsCount, true) + , SETUP_SIMPLE_COUNTER(BrokenConfigNotificationsCount, true) + , SETUP_SIMPLE_COUNTER(SubscribersCount, false) + , SETUP_SIMPLE_COUNTER(NetDataSourceType, false) + , SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false) +{ +} + +#undef SETUP_SIMPLE_COUNTER + +} // namespace NKikimr::NNetClassifier::NCounters diff --git a/ydb/core/mind/address_classification/counters.h b/ydb/core/mind/address_classification/counters.h index 0ca6e92468..bafab67abc 100644 --- a/ydb/core/mind/address_classification/counters.h +++ b/ydb/core/mind/address_classification/counters.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/ptr.h> -#include <util/generic/noncopyable.h> - -namespace NKikimr::NNetClassifier::NCounters { - -using namespace NMonitoring; - -struct TNetClassifierCounters : TAtomicRefCount<TNetClassifierCounters>, TNonCopyable { - using TPtr = TIntrusivePtr<TNetClassifierCounters>; - using TCounterPtr = TDynamicCounters::TCounterPtr; - - TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters); - - TCounterPtr GoodConfigNotificationsCount; - TCounterPtr BrokenConfigNotificationsCount; - TCounterPtr SubscribersCount; - TCounterPtr NetDataSourceType; - - TCounterPtr NetDataUpdateLagSeconds; -}; - -} // namespace NKikimr::NNetClassifier::NCounters + +#include <util/generic/ptr.h> +#include <util/generic/noncopyable.h> + +namespace NKikimr::NNetClassifier::NCounters { + +using namespace NMonitoring; + +struct TNetClassifierCounters : TAtomicRefCount<TNetClassifierCounters>, TNonCopyable { + using TPtr = TIntrusivePtr<TNetClassifierCounters>; + using TCounterPtr = TDynamicCounters::TCounterPtr; + + TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters); + + TCounterPtr GoodConfigNotificationsCount; + TCounterPtr BrokenConfigNotificationsCount; + TCounterPtr SubscribersCount; + TCounterPtr NetDataSourceType; + + TCounterPtr NetDataUpdateLagSeconds; +}; + +} // namespace NKikimr::NNetClassifier::NCounters diff --git a/ydb/core/mind/address_classification/net_classifier.cpp b/ydb/core/mind/address_classification/net_classifier.cpp index 11c7e6750c..d6ef5c7658 100644 --- a/ydb/core/mind/address_classification/net_classifier.cpp +++ b/ydb/core/mind/address_classification/net_classifier.cpp @@ -1,416 +1,416 @@ -#include "counters.h" -#include "net_classifier.h" - +#include "counters.h" +#include "net_classifier.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> - + #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> #include <ydb/core/cms/console/net_classifier_updater.h> #include <ydb/core/mon/mon.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> - -#include <util/generic/hash.h> -#include <util/stream/file.h> -#include <util/system/fs.h> -#include <util/system/fstat.h> - -#include <vector> - -namespace NKikimr::NNetClassifier { - -using namespace NAddressClassifier; -using namespace NConsole; -using namespace NCounters; - -inline auto& Ctx() { - return TActivationContext::AsActorContext(); -} - -class TNetClassifier: public TActorBootstrapped<TNetClassifier> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::NET_CLASSIFIER_ACTOR; - } - - void Bootstrap() { - Become(&TThis::WaitingForSubscribers); - } - -private: - static TMaybe<TInstant> GetNetDataFileModTimestamp(const TString& filePath) { - try { - TFileStat stat(filePath); - if (stat.IsFile() && !stat.IsNull()) { - return TInstant::Seconds(stat.MTime); - } else { - ythrow yexception() << "file does not exist: " << filePath; - } - } catch (const yexception& ex) { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to get NetData file stats: " << ex.what()); - } - - return Nothing(); - } - - const auto& Cfg() const { - return AppData(Ctx())->NetClassifierConfig; - } - + +#include <util/generic/hash.h> +#include <util/stream/file.h> +#include <util/system/fs.h> +#include <util/system/fstat.h> + +#include <vector> + +namespace NKikimr::NNetClassifier { + +using namespace NAddressClassifier; +using namespace NConsole; +using namespace NCounters; + +inline auto& Ctx() { + return TActivationContext::AsActorContext(); +} + +class TNetClassifier: public TActorBootstrapped<TNetClassifier> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::NET_CLASSIFIER_ACTOR; + } + + void Bootstrap() { + Become(&TThis::WaitingForSubscribers); + } + +private: + static TMaybe<TInstant> GetNetDataFileModTimestamp(const TString& filePath) { + try { + TFileStat stat(filePath); + if (stat.IsFile() && !stat.IsNull()) { + return TInstant::Seconds(stat.MTime); + } else { + ythrow yexception() << "file does not exist: " << filePath; + } + } catch (const yexception& ex) { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to get NetData file stats: " << ex.what()); + } + + return Nothing(); + } + + const auto& Cfg() const { + return AppData(Ctx())->NetClassifierConfig; + } + void AddSubscriber(const TActorId subscriberId) { - Subscribers.insert(subscriberId); - - *Counters->SubscribersCount = Subscribers.size(); - } - - STATEFN(WaitingForSubscribers) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWaiting); - } - } - - void SubscribeForConfigUpdates() { - const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); - Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(kind)); - } - - void HandleWhileWaiting(TEvNetClassifier::TEvSubscribe::TPtr& ev) { - Become(&TThis::Initing); - - InitCounters(); - - AddSubscriber(ev->Sender); - - const auto& netDataFilePath = Cfg().GetNetDataFilePath(); - if (netDataFilePath) { - MaybeNetDataFilePath = netDataFilePath; - - MaybeNetDataFileModTs = GetNetDataFileModTimestamp(*MaybeNetDataFilePath); - } - - SubscribeForConfigUpdates(); - } - - STATEFN(Initing) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileIniting); - hFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWhileIniting); - hFunc(TEvConfigsDispatcher::TEvGetConfigResponse, HandleWhileIniting); - hFunc(TEvents::TEvWakeup, HandleWhileIniting); - hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters); - } - } - - void HandleWhileIniting(TEvNetClassifier::TEvSubscribe::TPtr& ev) { - AddSubscriber(ev->Sender); - - // all subsribers get classifier update during a transition to a Working state - } - - void RequestDistributableConfig() { - const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); - Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvGetConfigRequest(kind)); - } - - void HandleWhileIniting(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "subscribed to ConfigsDispatcher for NetData updates"); - - // Subscription is set but the console tablet may still be unavailable hence it's a good idea to schedule a timeout event - Schedule(TDuration::Seconds(Cfg().GetCmsConfigTimeoutSeconds()), new TEvents::TEvWakeup); - - RequestDistributableConfig(); - - // This leads to Case 1 or Case 2 - } - - // Case 1: GetConfig request timed out - void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) { - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "TIMEOUT: failed to get distributable config, fall back to file: " << MaybeNetDataFilePath); - InitFromFile(); - } - - template<typename TConfPtr> - bool ShouldUseDistributableConfigOnInit(const TConfPtr& ptr) const { - if (!ptr) { - return false; - } - - if (!ptr->HasNetClassifierDistributableConfig()) { - return false; - } - - const auto& config = ptr->GetNetClassifierDistributableConfig(); - if (config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp()) { - if (MaybeNetDataFileModTs) { - return TInstant::MicroSeconds(config.GetLastUpdateTimestamp()) > *MaybeNetDataFileModTs; - } - - return true; - } - - return false; - } - - // Case 2: GetConfig request succeeded - void HandleWhileIniting(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr& ev) { - if (ShouldUseDistributableConfigOnInit(ev->Get()->Config)) { - LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will initialize from distributable config"); - - if (UpdateFromDistributableConfig(ev->Get()->Config->GetNetClassifierDistributableConfig())) { - ReportGoodConfig(); - UpdateNetDataSource(ENetDataSourceType::DistributableConfig); - - LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from distributable config"); - - CompleteInitialization(); - return; - } else { - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from distributed config"); - } - } - - ReportBadConfig(); - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "distributable config is empty, broken or outdated, will use file: " << MaybeNetDataFilePath); - InitFromFile(); - } - - void InitFromFile() { - // NetData file may be outdated hence warning log level is used - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will try to initialize from file: " << MaybeNetDataFilePath); - LabeledAddressClassifier = TryReadNetDataFromFile(); - if (LabeledAddressClassifier) { - NetDataUpdateTimestamp = MaybeNetDataFileModTs; - UpdateNetDataSource(ENetDataSourceType::File); - - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from file: " << MaybeNetDataFilePath); - } else { - LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from file: " << MaybeNetDataFilePath); - } - - CompleteInitialization(); - } - - bool CheckDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) const { - return config.GetPackedNetData() && config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp(); - } - - bool UpdateFromDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) { - if (CheckDistributableConfig(config)) { - LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got new config with datetime: " << config.GetLastUpdateDatetimeUTC()); - - auto labeledAddressClassifier = BuildNetClassifierFromPackedNetData(config.GetPackedNetData()); - if (labeledAddressClassifier) { - LabeledAddressClassifier = std::move(labeledAddressClassifier); - NetDataUpdateTimestamp = TInstant::MicroSeconds(config.GetLastUpdateTimestamp()); - - return true; - } else { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to parse NetData from distributable configuration"); - } - } else { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got bad distributable configuration"); - } - - return false; - } - - void CompleteInitialization() { - Become(&TThis::Working); - - NActors::TMon* mon = AppData(Ctx())->Mon; - if (mon) { - NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); - mon->RegisterActorPage(page, "netclassifier", "NetClassifier", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID); - } - - BroadcastClassifierUpdate(); - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvConsole::TEvConfigNotificationRequest, HandleWhileWorking); - hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWorking); - hFunc(NMon::TEvHttpInfo, HandleHttpRequest); - hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters); - } - } - - static TString FormatTs(const TMaybe<TInstant>& time) { - if (time) { - return time->ToRfc822String(); - } - return "NULL"; - } - - void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) - { - TStringBuilder message; - message << "NetClassifier info: \n"; - message << "\tUnique subscribers: " << Subscribers.size() << "\n"; - message << "\tClassifier updates sent: " << ClassifierUpdatesSent << "\n"; - message << "\tLast classifier update broadcast ts: " << FormatTs(LastClassifierUpdateBroadcastTs) << "\n"; - message << "\tNetData file path: " << MaybeNetDataFilePath << "\n"; - message << "\tNetData file ts: " << FormatTs(MaybeNetDataFileModTs) << "\n"; - message << "\tCurrent NetData ts: " << FormatTs(NetDataUpdateTimestamp) << "\n"; - message << "\tGood config notifications: " << GoodConfigNotifications << "\n"; - message << "\tBroken or empty config notifications: " << BrokenOrEmptyConfigNotifications << "\n"; - message << "\tCurrent NetData source: " << ToString(CurrentNetDataSource) << "\n"; - message << "\tClassifier: " << (LabeledAddressClassifier ? "OK" : "NULL"); - - Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(NMonitoring::HTTPOKTEXT) + TString(message), 0, - NMon::IEvHttpInfoRes::EContentType::Custom)); - } - - void HandleWhileWorking(TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { - const auto& rec = ev->Get()->Record; - - if (UpdateFromDistributableConfig(rec.GetConfig().GetNetClassifierDistributableConfig())) { - ReportGoodConfig(); - UpdateNetDataSource(ENetDataSourceType::DistributableConfig); - - BroadcastClassifierUpdate(); - } else { - ReportBadConfig(); - } - - // report success - auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - Send(ev->Sender, resp.Release(), 0, ev->Cookie); - } - - void HandleWhileWorking(TEvNetClassifier::TEvSubscribe::TPtr& ev) { - AddSubscriber(ev->Sender); - - // respond instantly with the current classifier - SendClassifierUpdate(ev->Sender); - } - + Subscribers.insert(subscriberId); + + *Counters->SubscribersCount = Subscribers.size(); + } + + STATEFN(WaitingForSubscribers) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWaiting); + } + } + + void SubscribeForConfigUpdates() { + const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); + Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(kind)); + } + + void HandleWhileWaiting(TEvNetClassifier::TEvSubscribe::TPtr& ev) { + Become(&TThis::Initing); + + InitCounters(); + + AddSubscriber(ev->Sender); + + const auto& netDataFilePath = Cfg().GetNetDataFilePath(); + if (netDataFilePath) { + MaybeNetDataFilePath = netDataFilePath; + + MaybeNetDataFileModTs = GetNetDataFileModTimestamp(*MaybeNetDataFilePath); + } + + SubscribeForConfigUpdates(); + } + + STATEFN(Initing) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileIniting); + hFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWhileIniting); + hFunc(TEvConfigsDispatcher::TEvGetConfigResponse, HandleWhileIniting); + hFunc(TEvents::TEvWakeup, HandleWhileIniting); + hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters); + } + } + + void HandleWhileIniting(TEvNetClassifier::TEvSubscribe::TPtr& ev) { + AddSubscriber(ev->Sender); + + // all subsribers get classifier update during a transition to a Working state + } + + void RequestDistributableConfig() { + const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); + Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvGetConfigRequest(kind)); + } + + void HandleWhileIniting(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { + LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "subscribed to ConfigsDispatcher for NetData updates"); + + // Subscription is set but the console tablet may still be unavailable hence it's a good idea to schedule a timeout event + Schedule(TDuration::Seconds(Cfg().GetCmsConfigTimeoutSeconds()), new TEvents::TEvWakeup); + + RequestDistributableConfig(); + + // This leads to Case 1 or Case 2 + } + + // Case 1: GetConfig request timed out + void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) { + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "TIMEOUT: failed to get distributable config, fall back to file: " << MaybeNetDataFilePath); + InitFromFile(); + } + + template<typename TConfPtr> + bool ShouldUseDistributableConfigOnInit(const TConfPtr& ptr) const { + if (!ptr) { + return false; + } + + if (!ptr->HasNetClassifierDistributableConfig()) { + return false; + } + + const auto& config = ptr->GetNetClassifierDistributableConfig(); + if (config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp()) { + if (MaybeNetDataFileModTs) { + return TInstant::MicroSeconds(config.GetLastUpdateTimestamp()) > *MaybeNetDataFileModTs; + } + + return true; + } + + return false; + } + + // Case 2: GetConfig request succeeded + void HandleWhileIniting(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr& ev) { + if (ShouldUseDistributableConfigOnInit(ev->Get()->Config)) { + LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will initialize from distributable config"); + + if (UpdateFromDistributableConfig(ev->Get()->Config->GetNetClassifierDistributableConfig())) { + ReportGoodConfig(); + UpdateNetDataSource(ENetDataSourceType::DistributableConfig); + + LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from distributable config"); + + CompleteInitialization(); + return; + } else { + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from distributed config"); + } + } + + ReportBadConfig(); + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "distributable config is empty, broken or outdated, will use file: " << MaybeNetDataFilePath); + InitFromFile(); + } + + void InitFromFile() { + // NetData file may be outdated hence warning log level is used + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will try to initialize from file: " << MaybeNetDataFilePath); + LabeledAddressClassifier = TryReadNetDataFromFile(); + if (LabeledAddressClassifier) { + NetDataUpdateTimestamp = MaybeNetDataFileModTs; + UpdateNetDataSource(ENetDataSourceType::File); + + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from file: " << MaybeNetDataFilePath); + } else { + LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from file: " << MaybeNetDataFilePath); + } + + CompleteInitialization(); + } + + bool CheckDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) const { + return config.GetPackedNetData() && config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp(); + } + + bool UpdateFromDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) { + if (CheckDistributableConfig(config)) { + LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got new config with datetime: " << config.GetLastUpdateDatetimeUTC()); + + auto labeledAddressClassifier = BuildNetClassifierFromPackedNetData(config.GetPackedNetData()); + if (labeledAddressClassifier) { + LabeledAddressClassifier = std::move(labeledAddressClassifier); + NetDataUpdateTimestamp = TInstant::MicroSeconds(config.GetLastUpdateTimestamp()); + + return true; + } else { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to parse NetData from distributable configuration"); + } + } else { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got bad distributable configuration"); + } + + return false; + } + + void CompleteInitialization() { + Become(&TThis::Working); + + NActors::TMon* mon = AppData(Ctx())->Mon; + if (mon) { + NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); + mon->RegisterActorPage(page, "netclassifier", "NetClassifier", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID); + } + + BroadcastClassifierUpdate(); + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvConsole::TEvConfigNotificationRequest, HandleWhileWorking); + hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWorking); + hFunc(NMon::TEvHttpInfo, HandleHttpRequest); + hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters); + } + } + + static TString FormatTs(const TMaybe<TInstant>& time) { + if (time) { + return time->ToRfc822String(); + } + return "NULL"; + } + + void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) + { + TStringBuilder message; + message << "NetClassifier info: \n"; + message << "\tUnique subscribers: " << Subscribers.size() << "\n"; + message << "\tClassifier updates sent: " << ClassifierUpdatesSent << "\n"; + message << "\tLast classifier update broadcast ts: " << FormatTs(LastClassifierUpdateBroadcastTs) << "\n"; + message << "\tNetData file path: " << MaybeNetDataFilePath << "\n"; + message << "\tNetData file ts: " << FormatTs(MaybeNetDataFileModTs) << "\n"; + message << "\tCurrent NetData ts: " << FormatTs(NetDataUpdateTimestamp) << "\n"; + message << "\tGood config notifications: " << GoodConfigNotifications << "\n"; + message << "\tBroken or empty config notifications: " << BrokenOrEmptyConfigNotifications << "\n"; + message << "\tCurrent NetData source: " << ToString(CurrentNetDataSource) << "\n"; + message << "\tClassifier: " << (LabeledAddressClassifier ? "OK" : "NULL"); + + Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(NMonitoring::HTTPOKTEXT) + TString(message), 0, + NMon::IEvHttpInfoRes::EContentType::Custom)); + } + + void HandleWhileWorking(TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { + const auto& rec = ev->Get()->Record; + + if (UpdateFromDistributableConfig(rec.GetConfig().GetNetClassifierDistributableConfig())) { + ReportGoodConfig(); + UpdateNetDataSource(ENetDataSourceType::DistributableConfig); + + BroadcastClassifierUpdate(); + } else { + ReportBadConfig(); + } + + // report success + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); + Send(ev->Sender, resp.Release(), 0, ev->Cookie); + } + + void HandleWhileWorking(TEvNetClassifier::TEvSubscribe::TPtr& ev) { + AddSubscriber(ev->Sender); + + // respond instantly with the current classifier + SendClassifierUpdate(ev->Sender); + } + void SendClassifierUpdate(const TActorId recipient) { - auto ev = MakeHolder<TEvNetClassifier::TEvClassifierUpdate>(); - ev->Classifier = LabeledAddressClassifier; - ev->NetDataUpdateTimestamp = NetDataUpdateTimestamp; - - Send(recipient, ev.Release()); - - ++ClassifierUpdatesSent; - } - - void BroadcastClassifierUpdate() { - for (const auto& subscriberId : Subscribers) { - SendClassifierUpdate(subscriberId); - } - - LastClassifierUpdateBroadcastTs = Ctx().Now(); - } - - TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromPackedNetData(const TString& packedNetData) const { - const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(packedNetData); - if (!serializedNetData) { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "empty packed networks"); - return nullptr; - } - - NKikimrNetClassifier::TNetData netData; - if (!netData.ParseFromString(serializedNetData)) { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "can't deserialize networks data protobuf"); - return nullptr; - } - - return BuildNetClassifierFromNetData(netData); - } - - TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromNetData(const NKikimrNetClassifier::TNetData& netData) const { - auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData); - if (!labeledAddressClassifier) { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "invalid NetData format"); - } - - return labeledAddressClassifier; - } - - TLabeledAddressClassifier::TConstPtr TryReadNetDataFromFile() const { - if (MaybeNetDataFilePath) { - try { - return ReadNetDataFromFile(*MaybeNetDataFilePath); - } catch (const yexception& ex) { - LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to read NetData from file: " << ex.what()); - } - } - - return nullptr; - } - - TLabeledAddressClassifier::TConstPtr ReadNetDataFromFile(const TString& path) const { - NKikimrNetClassifier::TNetData netData; - - // Parse TSV-format - TFileInput knownNetsFile(path); - TString line; - while (knownNetsFile.ReadLine(line)) { - TStringBuf mask, label; - if (!TStringBuf(line).TryRSplit('\t', mask, label)) { - ythrow yexception() << "NetData file violates tsv format"; - } - - auto& subnet = *netData.AddSubnets(); - subnet.SetMask(TString(mask)); - subnet.SetLabel(TString(label)); - } - - return BuildNetClassifierFromNetData(netData); - } - - // Monitoring features - - void ReportGoodConfig() { - ++GoodConfigNotifications; - Counters->GoodConfigNotificationsCount->Inc(); - } - - void ReportBadConfig() { - ++BrokenOrEmptyConfigNotifications; - Counters->BrokenConfigNotificationsCount->Inc(); - } - - void UpdateNetDataSource(const ENetDataSourceType newSource) { - CurrentNetDataSource = newSource; - *Counters->NetDataSourceType = CurrentNetDataSource; - } - - void UpdateTimedCounters(TEvNetClassifier::TEvUpdateTimedCounters::TPtr&) { - if (NetDataUpdateTimestamp) { - (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTimestamp->Seconds(); - } - - Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvNetClassifier::TEvUpdateTimedCounters); - } - - void InitCounters() { - Counters = MakeIntrusive<TNetClassifierCounters>(GetServiceCounters(AppData(Ctx())->Counters, "netclassifier")->GetSubgroup("subsystem", "distributor")); - - Send(Ctx().SelfID, new TEvNetClassifier::TEvUpdateTimedCounters); - } - -private: - TMaybe<TString> MaybeNetDataFilePath; - TMaybe<TInstant> MaybeNetDataFileModTs; - - TLabeledAddressClassifier::TConstPtr LabeledAddressClassifier; - TMaybe<TInstant> NetDataUpdateTimestamp; + auto ev = MakeHolder<TEvNetClassifier::TEvClassifierUpdate>(); + ev->Classifier = LabeledAddressClassifier; + ev->NetDataUpdateTimestamp = NetDataUpdateTimestamp; + + Send(recipient, ev.Release()); + + ++ClassifierUpdatesSent; + } + + void BroadcastClassifierUpdate() { + for (const auto& subscriberId : Subscribers) { + SendClassifierUpdate(subscriberId); + } + + LastClassifierUpdateBroadcastTs = Ctx().Now(); + } + + TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromPackedNetData(const TString& packedNetData) const { + const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(packedNetData); + if (!serializedNetData) { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "empty packed networks"); + return nullptr; + } + + NKikimrNetClassifier::TNetData netData; + if (!netData.ParseFromString(serializedNetData)) { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "can't deserialize networks data protobuf"); + return nullptr; + } + + return BuildNetClassifierFromNetData(netData); + } + + TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromNetData(const NKikimrNetClassifier::TNetData& netData) const { + auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData); + if (!labeledAddressClassifier) { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "invalid NetData format"); + } + + return labeledAddressClassifier; + } + + TLabeledAddressClassifier::TConstPtr TryReadNetDataFromFile() const { + if (MaybeNetDataFilePath) { + try { + return ReadNetDataFromFile(*MaybeNetDataFilePath); + } catch (const yexception& ex) { + LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to read NetData from file: " << ex.what()); + } + } + + return nullptr; + } + + TLabeledAddressClassifier::TConstPtr ReadNetDataFromFile(const TString& path) const { + NKikimrNetClassifier::TNetData netData; + + // Parse TSV-format + TFileInput knownNetsFile(path); + TString line; + while (knownNetsFile.ReadLine(line)) { + TStringBuf mask, label; + if (!TStringBuf(line).TryRSplit('\t', mask, label)) { + ythrow yexception() << "NetData file violates tsv format"; + } + + auto& subnet = *netData.AddSubnets(); + subnet.SetMask(TString(mask)); + subnet.SetLabel(TString(label)); + } + + return BuildNetClassifierFromNetData(netData); + } + + // Monitoring features + + void ReportGoodConfig() { + ++GoodConfigNotifications; + Counters->GoodConfigNotificationsCount->Inc(); + } + + void ReportBadConfig() { + ++BrokenOrEmptyConfigNotifications; + Counters->BrokenConfigNotificationsCount->Inc(); + } + + void UpdateNetDataSource(const ENetDataSourceType newSource) { + CurrentNetDataSource = newSource; + *Counters->NetDataSourceType = CurrentNetDataSource; + } + + void UpdateTimedCounters(TEvNetClassifier::TEvUpdateTimedCounters::TPtr&) { + if (NetDataUpdateTimestamp) { + (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTimestamp->Seconds(); + } + + Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvNetClassifier::TEvUpdateTimedCounters); + } + + void InitCounters() { + Counters = MakeIntrusive<TNetClassifierCounters>(GetServiceCounters(AppData(Ctx())->Counters, "netclassifier")->GetSubgroup("subsystem", "distributor")); + + Send(Ctx().SelfID, new TEvNetClassifier::TEvUpdateTimedCounters); + } + +private: + TMaybe<TString> MaybeNetDataFilePath; + TMaybe<TInstant> MaybeNetDataFileModTs; + + TLabeledAddressClassifier::TConstPtr LabeledAddressClassifier; + TMaybe<TInstant> NetDataUpdateTimestamp; THashSet<TActorId> Subscribers; - - ui64 ClassifierUpdatesSent = 0; - ui64 GoodConfigNotifications = 0; - ui64 BrokenOrEmptyConfigNotifications = 0; - TMaybe<TInstant> LastClassifierUpdateBroadcastTs; - ENetDataSourceType CurrentNetDataSource = ENetDataSourceType::None; - - TNetClassifierCounters::TPtr Counters; -}; - -NActors::IActor* CreateNetClassifier() { - return new TNetClassifier(); -} - -} // namespace NKikimr::NNetClassifier + + ui64 ClassifierUpdatesSent = 0; + ui64 GoodConfigNotifications = 0; + ui64 BrokenOrEmptyConfigNotifications = 0; + TMaybe<TInstant> LastClassifierUpdateBroadcastTs; + ENetDataSourceType CurrentNetDataSource = ENetDataSourceType::None; + + TNetClassifierCounters::TPtr Counters; +}; + +NActors::IActor* CreateNetClassifier() { + return new TNetClassifier(); +} + +} // namespace NKikimr::NNetClassifier diff --git a/ydb/core/mind/address_classification/net_classifier.h b/ydb/core/mind/address_classification/net_classifier.h index 6a298cd4c4..4bc4f3b6a7 100644 --- a/ydb/core/mind/address_classification/net_classifier.h +++ b/ydb/core/mind/address_classification/net_classifier.h @@ -1,56 +1,56 @@ -#pragma once - +#pragma once + #include <ydb/core/base/events.h> #include <ydb/core/util/address_classifier.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actorid.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/event_local.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/ptr.h> - -#include <vector> - -namespace NKikimr::NNetClassifier { - + +#include <util/generic/ptr.h> + +#include <vector> + +namespace NKikimr::NNetClassifier { + using NActors::TActorId; - + inline TActorId MakeNetClassifierID() { - static const char x[12] = "net_classvc"; + static const char x[12] = "net_classvc"; return TActorId(0, TStringBuf(x, 12)); -} - -NActors::IActor* CreateNetClassifier(); - -struct TEvNetClassifier { - enum EEv { - EvClassifierUpdate = EventSpaceBegin(TKikimrEvents::ES_NET_CLASSIFIER), - EvSubscribe, - EvUpdateTimedCounters, - EvEnd - }; - - struct TEvClassifierUpdate : public NActors::TEventLocal<TEvClassifierUpdate, EvClassifierUpdate> { - NAddressClassifier::TLabeledAddressClassifier::TConstPtr Classifier; - TMaybe<TInstant> NetDataUpdateTimestamp; - }; - - struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> { - }; - - struct TEvUpdateTimedCounters : public NActors::TEventLocal<TEvUpdateTimedCounters, EvUpdateTimedCounters> { - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_NET_CLASSIFIER), "Unexpected TEvNetClassifier event range"); -}; - -enum ENetDataSourceType { - None = 0, - File, - DistributableConfig -}; - -} // namespace NKikimr::NNetClassifier +} + +NActors::IActor* CreateNetClassifier(); + +struct TEvNetClassifier { + enum EEv { + EvClassifierUpdate = EventSpaceBegin(TKikimrEvents::ES_NET_CLASSIFIER), + EvSubscribe, + EvUpdateTimedCounters, + EvEnd + }; + + struct TEvClassifierUpdate : public NActors::TEventLocal<TEvClassifierUpdate, EvClassifierUpdate> { + NAddressClassifier::TLabeledAddressClassifier::TConstPtr Classifier; + TMaybe<TInstant> NetDataUpdateTimestamp; + }; + + struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> { + }; + + struct TEvUpdateTimedCounters : public NActors::TEventLocal<TEvUpdateTimedCounters, EvUpdateTimedCounters> { + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_NET_CLASSIFIER), "Unexpected TEvNetClassifier event range"); +}; + +enum ENetDataSourceType { + None = 0, + File, + DistributableConfig +}; + +} // namespace NKikimr::NNetClassifier diff --git a/ydb/core/mind/address_classification/net_classifier_ut.cpp b/ydb/core/mind/address_classification/net_classifier_ut.cpp index 088d25b9c7..7ba36c87c4 100644 --- a/ydb/core/mind/address_classification/net_classifier_ut.cpp +++ b/ydb/core/mind/address_classification/net_classifier_ut.cpp @@ -1,219 +1,219 @@ #include <ydb/core/base/counters.h> - + #include <ydb/core/mind/address_classification/counters.h> - + #include <ydb/core/mind/address_classification/net_classifier.h> - + #include <ydb/core/testlib/test_client.h> - + #include <library/cpp/actors/http/http_proxy.cpp> #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/file.h> -#include <util/system/tempfile.h> - -namespace NKikimr::NNetClassifierTests { - -using namespace NNetClassifier; -using namespace Tests; - -static THolder<TTempFileHandle> CreateNetDataFile(const TString& content) { - auto netDataFile = MakeHolder<TTempFileHandle>("data.tsv"); - - netDataFile->Write(content.Data(), content.Size()); - netDataFile->FlushData(); - - return netDataFile; -} - -static void Wait() { - Sleep(TDuration::MilliSeconds(333)); -} - -static TString FormNetData() { - return "10.99.99.224/27\tSAS\n" - "185.32.186.0/29\tIVA\n" - "185.32.186.16/29\tMYT\n" - "185.32.186.24/31\tIVA\n" - "185.32.186.26/31\tIVA\n" - "37.29.21.132/30\tMYT\n" - "83.220.51.8/30\tMYT\n" - "2a02:6b8:f030:3000::/52\tMYT\n" - "2a02:6b8:fc00::/48\tSAS"; -} - + +#include <util/stream/file.h> +#include <util/system/tempfile.h> + +namespace NKikimr::NNetClassifierTests { + +using namespace NNetClassifier; +using namespace Tests; + +static THolder<TTempFileHandle> CreateNetDataFile(const TString& content) { + auto netDataFile = MakeHolder<TTempFileHandle>("data.tsv"); + + netDataFile->Write(content.Data(), content.Size()); + netDataFile->FlushData(); + + return netDataFile; +} + +static void Wait() { + Sleep(TDuration::MilliSeconds(333)); +} + +static TString FormNetData() { + return "10.99.99.224/27\tSAS\n" + "185.32.186.0/29\tIVA\n" + "185.32.186.16/29\tMYT\n" + "185.32.186.24/31\tIVA\n" + "185.32.186.26/31\tIVA\n" + "37.29.21.132/30\tMYT\n" + "83.220.51.8/30\tMYT\n" + "2a02:6b8:f030:3000::/52\tMYT\n" + "2a02:6b8:fc00::/48\tSAS"; +} + static TAutoPtr<IEventHandle> GetClassifierUpdate(TServer& server, const TActorId sender) { - auto& actorSystem = *server.GetRuntime(); - - actorSystem.Send( - new IEventHandle(MakeNetClassifierID(), sender, - new TEvNetClassifier::TEvSubscribe() - )); - - TAutoPtr<IEventHandle> handle; - actorSystem.GrabEdgeEvent<TEvNetClassifier::TEvClassifierUpdate>(handle); - - UNIT_ASSERT(handle); - UNIT_ASSERT_VALUES_EQUAL(handle->Recipient, sender); - - return handle; -} - -static NCounters::TNetClassifierCounters::TPtr ExtractCounters(TServer& server) { - UNIT_ASSERT_VALUES_EQUAL(server.GetRuntime()->GetNodeCount(), 1); - - return MakeIntrusive<NCounters::TNetClassifierCounters>( - GetServiceCounters(server.GetRuntime()->GetAppData(0).Counters, "netclassifier")->GetSubgroup("subsystem", "distributor") - ); -} - -Y_UNIT_TEST_SUITE(TNetClassifierTest) { - Y_UNIT_TEST(TestInitFromFile) { - TPortManager pm; - const ui16 port = pm.GetPort(2134); - - TServerSettings settings(port); - - auto netDataFile = CreateNetDataFile(FormNetData()); - settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name()); - settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1); - - TServer cleverServer = TServer(settings); - { + auto& actorSystem = *server.GetRuntime(); + + actorSystem.Send( + new IEventHandle(MakeNetClassifierID(), sender, + new TEvNetClassifier::TEvSubscribe() + )); + + TAutoPtr<IEventHandle> handle; + actorSystem.GrabEdgeEvent<TEvNetClassifier::TEvClassifierUpdate>(handle); + + UNIT_ASSERT(handle); + UNIT_ASSERT_VALUES_EQUAL(handle->Recipient, sender); + + return handle; +} + +static NCounters::TNetClassifierCounters::TPtr ExtractCounters(TServer& server) { + UNIT_ASSERT_VALUES_EQUAL(server.GetRuntime()->GetNodeCount(), 1); + + return MakeIntrusive<NCounters::TNetClassifierCounters>( + GetServiceCounters(server.GetRuntime()->GetAppData(0).Counters, "netclassifier")->GetSubgroup("subsystem", "distributor") + ); +} + +Y_UNIT_TEST_SUITE(TNetClassifierTest) { + Y_UNIT_TEST(TestInitFromFile) { + TPortManager pm; + const ui16 port = pm.GetPort(2134); + + TServerSettings settings(port); + + auto netDataFile = CreateNetDataFile(FormNetData()); + settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name()); + settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1); + + TServer cleverServer = TServer(settings); + { const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor(); - auto handle = GetClassifierUpdate(cleverServer, sender); - auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); - - UNIT_ASSERT(event->NetDataUpdateTimestamp); - UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero()); - - // basic sanity check - UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("37.29.21.135"), "MYT"); - UNIT_ASSERT(!event->Classifier->ClassifyAddress("trololo")); - } - { - auto counters = ExtractCounters(cleverServer); - - UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::File); - + auto handle = GetClassifierUpdate(cleverServer, sender); + auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); + + UNIT_ASSERT(event->NetDataUpdateTimestamp); + UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero()); + + // basic sanity check + UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("37.29.21.135"), "MYT"); + UNIT_ASSERT(!event->Classifier->ClassifyAddress("trololo")); + } + { + auto counters = ExtractCounters(cleverServer); + + UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::File); + const auto prevLagSeconds = AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic()); while (prevLagSeconds >= AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic())) { - Wait(); - } - } - } - - Y_UNIT_TEST(TestInitFromBadlyFormattedFile) { - TPortManager pm; - const ui16 port = pm.GetPort(2134); - - TServerSettings settings(port); - - auto netDataFile = CreateNetDataFile("omg\tlol"); - settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name()); - - TServer cleverServer = TServer(settings); - { + Wait(); + } + } + } + + Y_UNIT_TEST(TestInitFromBadlyFormattedFile) { + TPortManager pm; + const ui16 port = pm.GetPort(2134); + + TServerSettings settings(port); + + auto netDataFile = CreateNetDataFile("omg\tlol"); + settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name()); + + TServer cleverServer = TServer(settings); + { const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor(); - auto handle = GetClassifierUpdate(cleverServer, sender); - auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); - - UNIT_ASSERT(!event->NetDataUpdateTimestamp); - UNIT_ASSERT(!event->Classifier); - } - { - auto counters = ExtractCounters(cleverServer); - - UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::None); - } - } - - static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) { - const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: " - << netData.size() << "\r\n\r\n" << netData; - NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content); - - return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse); - } - - Y_UNIT_TEST(TestInitFromRemoteSource) { + auto handle = GetClassifierUpdate(cleverServer, sender); + auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); + + UNIT_ASSERT(!event->NetDataUpdateTimestamp); + UNIT_ASSERT(!event->Classifier); + } + { + auto counters = ExtractCounters(cleverServer); + + UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::None); + } + } + + static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) { + const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: " + << netData.size() << "\r\n\r\n" << netData; + NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content); + + return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse); + } + + Y_UNIT_TEST(TestInitFromRemoteSource) { NMonitoring::TMetricRegistry sensors; - - TPortManager pm; - const ui16 port = pm.GetPort(2134); - const ui64 netDataSourcePort = pm.GetPort(13334); - - const TString hostAndPort = TStringBuilder() << "http://[::1]:" << netDataSourcePort; - const TString uri = "/fancy_path/networks.tsv"; - - TServerSettings settings(port); - auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig(); - updaterConfig.SetNetDataSourceUrl(hostAndPort + uri); - updaterConfig.SetNetDataUpdateIntervalSeconds(1); - updaterConfig.SetRetryIntervalSeconds(1); - - settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1); - - TServer cleverServer = TServer(settings); - auto& actorSystem = *cleverServer.GetRuntime(); - - NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors); + + TPortManager pm; + const ui16 port = pm.GetPort(2134); + const ui64 netDataSourcePort = pm.GetPort(13334); + + const TString hostAndPort = TStringBuilder() << "http://[::1]:" << netDataSourcePort; + const TString uri = "/fancy_path/networks.tsv"; + + TServerSettings settings(port); + auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig(); + updaterConfig.SetNetDataSourceUrl(hostAndPort + uri); + updaterConfig.SetNetDataUpdateIntervalSeconds(1); + updaterConfig.SetRetryIntervalSeconds(1); + + settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1); + + TServer cleverServer = TServer(settings); + auto& actorSystem = *cleverServer.GetRuntime(); + + NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors); NActors::TActorId proxyId = actorSystem.Register(proxy); - - actorSystem.Send( + + actorSystem.Send( new NActors::IEventHandle(proxyId, TActorId(), new NHttp::TEvHttpProxy::TEvAddListeningPort(netDataSourcePort)), 0, true - ); - + ); + NActors::TActorId serverId = actorSystem.AllocateEdgeActor(); - actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(uri, serverId)), 0, true); - + actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(uri, serverId)), 0, true); + const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor(); - - // initially classifier's data should be empty - auto handle = GetClassifierUpdate(cleverServer, sender); - UNIT_ASSERT(!handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier); - - // then the server responds with new net data - NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); - UNIT_ASSERT_EQUAL(request->Request->URL, uri); - - actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, FormNetData())), 0, true); - - // subscriber waits for proper net data update - while (true) { - handle = GetClassifierUpdate(cleverServer, sender); - - if (handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier) { - break; - } - - Wait(); - } - - UNIT_ASSERT(handle); - auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); - - UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("185.32.186.26"), "IVA"); - - { - auto counters = ExtractCounters(cleverServer); - + + // initially classifier's data should be empty + auto handle = GetClassifierUpdate(cleverServer, sender); + UNIT_ASSERT(!handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier); + + // then the server responds with new net data + NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); + UNIT_ASSERT_EQUAL(request->Request->URL, uri); + + actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, FormNetData())), 0, true); + + // subscriber waits for proper net data update + while (true) { + handle = GetClassifierUpdate(cleverServer, sender); + + if (handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier) { + break; + } + + Wait(); + } + + UNIT_ASSERT(handle); + auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>(); + + UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("185.32.186.26"), "IVA"); + + { + auto counters = ExtractCounters(cleverServer); + UNIT_ASSERT_VALUES_EQUAL(AtomicGet(counters->SubscribersCount->GetAtomic()), 1); UNIT_ASSERT_VALUES_EQUAL(AtomicGet(counters->NetDataSourceType->GetAtomic()), ENetDataSourceType::DistributableConfig); - + const auto prevLagSeconds = AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic()); while (prevLagSeconds >= AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic())) { - Wait(); - } - } - } -} - -} // namespace NKikimr::NNetClassifierTests + Wait(); + } + } + } +} + +} // namespace NKikimr::NNetClassifierTests diff --git a/ydb/core/mind/address_classification/ut/ya.make b/ydb/core/mind/address_classification/ut/ya.make index 62157a5aff..b7cab35ea2 100644 --- a/ydb/core/mind/address_classification/ut/ya.make +++ b/ydb/core/mind/address_classification/ut/ya.make @@ -1,34 +1,34 @@ UNITTEST_FOR(ydb/core/mind/address_classification) - + OWNER( radix g:kikimr ) - -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(1200) - SIZE(LARGE) - SPLIT_FACTOR(20) - TAG(ya:fat) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1200) + SIZE(LARGE) + SPLIT_FACTOR(20) + TAG(ya:fat) REQUIREMENTS(ram:16) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) REQUIREMENTS(ram:16) -ENDIF() - -PEERDIR( +ENDIF() + +PEERDIR( library/cpp/actors/http ydb/core/mind/address_classification ydb/core/testlib -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - net_classifier_ut.cpp -) - -END() +SRCS( + net_classifier_ut.cpp +) + +END() diff --git a/ydb/core/mind/address_classification/ya.make b/ydb/core/mind/address_classification/ya.make index 73301851da..87f74b5b95 100644 --- a/ydb/core/mind/address_classification/ya.make +++ b/ydb/core/mind/address_classification/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - radix - g:kikimr -) - -SRCS( - counters.cpp - net_classifier.cpp -) - -GENERATE_ENUM_SERIALIZATION(net_classifier.h) - -PEERDIR( +LIBRARY() + +OWNER( + radix + g:kikimr +) + +SRCS( + counters.cpp + net_classifier.cpp +) + +GENERATE_ENUM_SERIALIZATION(net_classifier.h) + +PEERDIR( library/cpp/actors/core library/cpp/monlib/dynamic_counters ydb/core/base @@ -20,9 +20,9 @@ PEERDIR( ydb/core/mon ydb/core/protos ydb/core/util -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp index 59fdea5c2b..a57e284f0e 100644 --- a/ydb/core/persqueue/cluster_tracker.cpp +++ b/ydb/core/persqueue/cluster_tracker.cpp @@ -1,67 +1,67 @@ -#include "cluster_tracker.h" +#include "cluster_tracker.h" #include "pq_database.h" - + #include <ydb/core/base/appdata.h> #include <ydb/core/kqp/kqp.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> - -#include <util/generic/hash.h> -#include <util/generic/scope.h> - -#include <tuple> -#include <vector> - -namespace NKikimr::NPQ::NClusterTracker { - -inline auto& Ctx() { - return TActivationContext::AsActorContext(); -} - -bool TClustersList::TCluster::operator==(const TCluster& rhs) const { - return std::tie(Name, Datacenter, Balancer, IsLocal, IsEnabled, Weight) == - std::tie(rhs.Name, rhs.Datacenter, rhs.Balancer, rhs.IsLocal, rhs.IsEnabled, rhs.Weight); -} - -bool TClustersList::operator==(const TClustersList& rhs) const { - return Clusters == rhs.Clusters && Version == rhs.Version; -} - -class TClusterTracker: public TActorBootstrapped<TClusterTracker> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_TRACKER; - } - - const auto& Cfg() const { - return AppData(Ctx())->PQConfig; - } - - void Bootstrap() { - ListClustersQuery = MakeListClustersQuery(); - - Become(&TThis::WaitingForSubscribers); - } - -private: + +#include <util/generic/hash.h> +#include <util/generic/scope.h> + +#include <tuple> +#include <vector> + +namespace NKikimr::NPQ::NClusterTracker { + +inline auto& Ctx() { + return TActivationContext::AsActorContext(); +} + +bool TClustersList::TCluster::operator==(const TCluster& rhs) const { + return std::tie(Name, Datacenter, Balancer, IsLocal, IsEnabled, Weight) == + std::tie(rhs.Name, rhs.Datacenter, rhs.Balancer, rhs.IsLocal, rhs.IsEnabled, rhs.Weight); +} + +bool TClustersList::operator==(const TClustersList& rhs) const { + return Clusters == rhs.Clusters && Version == rhs.Version; +} + +class TClusterTracker: public TActorBootstrapped<TClusterTracker> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_TRACKER; + } + + const auto& Cfg() const { + return AppData(Ctx())->PQConfig; + } + + void Bootstrap() { + ListClustersQuery = MakeListClustersQuery(); + + Become(&TThis::WaitingForSubscribers); + } + +private: void AddSubscriber(const TActorId subscriberId) { - Subscribers.insert(subscriberId); - } - - STATEFN(WaitingForSubscribers) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWaiting); - } - } - - void HandleWhileWaiting(TEvClusterTracker::TEvSubscribe::TPtr& ev) { - AddSubscriber(ev->Sender); - + Subscribers.insert(subscriberId); + } + + STATEFN(WaitingForSubscribers) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWaiting); + } + } + + void HandleWhileWaiting(TEvClusterTracker::TEvSubscribe::TPtr& ev) { + AddSubscriber(ev->Sender); + Become(&TThis::Working); - - Send(Ctx().SelfID, new TEvents::TEvWakeup); - } - + + Send(Ctx().SelfID, new TEvents::TEvWakeup); + } + const TString& GetDatabase() { if (Database.empty()) { Database = GetDatabaseFromConfig(Cfg()); @@ -70,86 +70,86 @@ private: return Database; } - TString MakeListClustersQuery() const { - return Sprintf( - R"( + TString MakeListClustersQuery() const { + return Sprintf( + R"( --!syntax_v1 SELECT C.name, C.balancer, C.local, C.enabled, C.weight, V.version FROM `%s` AS C - CROSS JOIN + CROSS JOIN (SELECT version FROM `%s` WHERE name == 'Cluster') AS V; - )", Cfg().GetClusterTablePath().c_str(), Cfg().GetVersionTablePath().c_str()); - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWorking); - hFunc(TEvents::TEvWakeup, HandleWhileWorking); - hFunc(NKqp::TEvKqp::TEvQueryResponse, HandleWhileWorking); + )", Cfg().GetClusterTablePath().c_str(), Cfg().GetVersionTablePath().c_str()); + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWorking); + hFunc(TEvents::TEvWakeup, HandleWhileWorking); + hFunc(NKqp::TEvKqp::TEvQueryResponse, HandleWhileWorking); hFunc(NKqp::TEvKqp::TEvProcessResponse, HandleWhileWorking); - } - } - + } + } + void SendClustersList(TActorId subscriberId) { - auto ev = MakeHolder<TEvClusterTracker::TEvClustersUpdate>(); - - ev->ClustersList = ClustersList; - ev->ClustersListUpdateTimestamp = ClustersListUpdateTimestamp; - - Send(subscriberId, ev.Release()); - } - - void HandleWhileWorking(TEvClusterTracker::TEvSubscribe::TPtr& ev) { - AddSubscriber(ev->Sender); - - // List may be null due to reinit - if (ClustersList) { - SendClustersList(ev->Sender); - } - } - - void BroadcastClustersUpdate() { - for (const auto& subscriberId : Subscribers) { - SendClustersList(subscriberId); - } - } - - void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) { - auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - + auto ev = MakeHolder<TEvClusterTracker::TEvClustersUpdate>(); + + ev->ClustersList = ClustersList; + ev->ClustersListUpdateTimestamp = ClustersListUpdateTimestamp; + + Send(subscriberId, ev.Release()); + } + + void HandleWhileWorking(TEvClusterTracker::TEvSubscribe::TPtr& ev) { + AddSubscriber(ev->Sender); + + // List may be null due to reinit + if (ClustersList) { + SendClustersList(ev->Sender); + } + } + + void BroadcastClustersUpdate() { + for (const auto& subscriberId : Subscribers) { + SendClustersList(subscriberId); + } + } + + void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) { + auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); + req->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); req->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); - req->Record.MutableRequest()->SetKeepSession(false); + req->Record.MutableRequest()->SetKeepSession(false); req->Record.MutableRequest()->SetQuery(MakeListClustersQuery()); req->Record.MutableRequest()->SetDatabase(GetDatabase()); // useless without explicit session // req->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true); req->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); req->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); - - Send(NKqp::MakeKqpProxyID(Ctx().SelfID.NodeId()), req.Release()); - } - - void HandleWhileWorking(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetRef(); - if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS && record.GetResponse().GetResults(0).GetValue().GetStruct(0).ListSize()) { - UpdateClustersList(record); - - Y_VERIFY(ClustersList); - Y_VERIFY(ClustersList->Clusters.size()); - Y_VERIFY(ClustersListUpdateTimestamp && *ClustersListUpdateTimestamp); - - BroadcastClustersUpdate(); - - Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutSec()), new TEvents::TEvWakeup); - } else { - LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record); - + + Send(NKqp::MakeKqpProxyID(Ctx().SelfID.NodeId()), req.Release()); + } + + void HandleWhileWorking(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetRef(); + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS && record.GetResponse().GetResults(0).GetValue().GetStruct(0).ListSize()) { + UpdateClustersList(record); + + Y_VERIFY(ClustersList); + Y_VERIFY(ClustersList->Clusters.size()); + Y_VERIFY(ClustersListUpdateTimestamp && *ClustersListUpdateTimestamp); + + BroadcastClustersUpdate(); + + Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutSec()), new TEvents::TEvWakeup); + } else { + LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record); + ClustersList = nullptr; Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup); - } - } - + } + } + void HandleWhileWorking(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev) { const auto& record = ev->Get()->Record; LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record); @@ -159,41 +159,41 @@ private: Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup); } - template<typename TProtoRecord> - void UpdateClustersList(const TProtoRecord& record) { - auto clustersList = MakeIntrusive<TClustersList>(); - auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0); - clustersList->Clusters.resize(t.ListSize()); - - for (size_t i = 0; i < t.ListSize(); ++i) { - auto& cluster = clustersList->Clusters[i]; - - cluster.Name = t.GetList(i).GetStruct(0).GetOptional().GetText(); - cluster.Datacenter = cluster.Name; - cluster.Balancer = t.GetList(i).GetStruct(1).GetOptional().GetText(); - - cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool(); - cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool(); + template<typename TProtoRecord> + void UpdateClustersList(const TProtoRecord& record) { + auto clustersList = MakeIntrusive<TClustersList>(); + auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0); + clustersList->Clusters.resize(t.ListSize()); + + for (size_t i = 0; i < t.ListSize(); ++i) { + auto& cluster = clustersList->Clusters[i]; + + cluster.Name = t.GetList(i).GetStruct(0).GetOptional().GetText(); + cluster.Datacenter = cluster.Name; + cluster.Balancer = t.GetList(i).GetStruct(1).GetOptional().GetText(); + + cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool(); + cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool(); cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64(); - } - + } + clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64(); - - ClustersList = std::move(clustersList); - ClustersListUpdateTimestamp = Ctx().Now(); - } - -private: - TString ListClustersQuery; - TString PreparedQueryId; - TClustersList::TConstPtr ClustersList = nullptr; - TMaybe<TInstant> ClustersListUpdateTimestamp; + + ClustersList = std::move(clustersList); + ClustersListUpdateTimestamp = Ctx().Now(); + } + +private: + TString ListClustersQuery; + TString PreparedQueryId; + TClustersList::TConstPtr ClustersList = nullptr; + TMaybe<TInstant> ClustersListUpdateTimestamp; THashSet<TActorId> Subscribers; TString Database; -}; - -NActors::IActor* CreateClusterTracker() { - return new TClusterTracker(); -} - -} // namespace NKikimr::NPQ::NClusterTracker +}; + +NActors::IActor* CreateClusterTracker() { + return new TClusterTracker(); +} + +} // namespace NKikimr::NPQ::NClusterTracker diff --git a/ydb/core/persqueue/cluster_tracker.h b/ydb/core/persqueue/cluster_tracker.h index 2ae06cf915..1c6a5fbed7 100644 --- a/ydb/core/persqueue/cluster_tracker.h +++ b/ydb/core/persqueue/cluster_tracker.h @@ -1,80 +1,80 @@ -#pragma once - +#pragma once + #include <ydb/core/base/events.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actorid.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/event_local.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/string/builder.h> - -#include <vector> - -namespace NKikimr::NPQ::NClusterTracker { - + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/string/builder.h> + +#include <vector> + +namespace NKikimr::NPQ::NClusterTracker { + using NActors::TActorId; - + inline TActorId MakeClusterTrackerID() { - static const char x[12] = "clstr_trckr"; + static const char x[12] = "clstr_trckr"; return TActorId(0, TStringBuf(x, 12)); -} - -NActors::IActor* CreateClusterTracker(); - -struct TClustersList : public TAtomicRefCount<TClustersList>, TNonCopyable { - using TConstPtr = TIntrusiveConstPtr<TClustersList>; - - TClustersList() = default; - - struct TCluster { - TString Name; - TString Datacenter; // is equal to cluster name at the moment - TString Balancer; - bool IsEnabled = false; - bool IsLocal = false; - ui64 Weight = 1000; - - TString DebugString() const { - TStringBuilder builder; - builder << "(" << Name << ", " << Datacenter << ", " << Balancer << ", "; - builder << (IsEnabled ? "enabled" : "disabled") << ", "; - builder << (IsLocal ? "local" : "remote") << ", "; - builder << Weight << ")"; - - return TString(builder); - } - - bool operator==(const TCluster& other) const; - }; - - bool operator==(const TClustersList& other) const; - - std::vector<TCluster> Clusters; - - i64 Version = 0; -}; - -struct TEvClusterTracker { - enum EEv { - EvClustersUpdate = EventSpaceBegin(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), - EvSubscribe, - EvEnd - }; - - struct TEvClustersUpdate : public NActors::TEventLocal<TEvClustersUpdate, EvClustersUpdate> { - TClustersList::TConstPtr ClustersList; - TMaybe<TInstant> ClustersListUpdateTimestamp; - }; - - struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> { - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), "Unexpected TEvClusterTracker event range"); -}; - -} // namespace NKikimr::NPQ::NClusterTracker +} + +NActors::IActor* CreateClusterTracker(); + +struct TClustersList : public TAtomicRefCount<TClustersList>, TNonCopyable { + using TConstPtr = TIntrusiveConstPtr<TClustersList>; + + TClustersList() = default; + + struct TCluster { + TString Name; + TString Datacenter; // is equal to cluster name at the moment + TString Balancer; + bool IsEnabled = false; + bool IsLocal = false; + ui64 Weight = 1000; + + TString DebugString() const { + TStringBuilder builder; + builder << "(" << Name << ", " << Datacenter << ", " << Balancer << ", "; + builder << (IsEnabled ? "enabled" : "disabled") << ", "; + builder << (IsLocal ? "local" : "remote") << ", "; + builder << Weight << ")"; + + return TString(builder); + } + + bool operator==(const TCluster& other) const; + }; + + bool operator==(const TClustersList& other) const; + + std::vector<TCluster> Clusters; + + i64 Version = 0; +}; + +struct TEvClusterTracker { + enum EEv { + EvClustersUpdate = EventSpaceBegin(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), + EvSubscribe, + EvEnd + }; + + struct TEvClustersUpdate : public NActors::TEventLocal<TEvClustersUpdate, EvClustersUpdate> { + TClustersList::TConstPtr ClustersList; + TMaybe<TInstant> ClustersListUpdateTimestamp; + }; + + struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> { + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), "Unexpected TEvClusterTracker event range"); +}; + +} // namespace NKikimr::NPQ::NClusterTracker diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 50e89676ef..e2e26d5902 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -7,11 +7,11 @@ OWNER( ) SRCS( - cluster_tracker.cpp - blob.cpp + cluster_tracker.cpp + blob.cpp event_helpers.cpp - header.cpp - percentile_counter.cpp + header.cpp + percentile_counter.cpp pq.cpp pq_database.cpp pq_impl.cpp @@ -23,7 +23,7 @@ SRCS( pq_l2_cache.cpp read_balancer.cpp read_speed_limiter.cpp - subscriber.cpp + subscriber.cpp type_codecs_defs.cpp user_info.cpp write_meta.cpp diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d64169d4fc..cc725da0cd 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -703,25 +703,25 @@ message TSqsConfig { // Max time value for long polling (param WaitTimeSeconds for ReceiveMessage) optional uint64 MaxWaitTimeoutMs = 10 [default = 20000]; - - optional uint32 SchemeCacheSoftRefreshTimeSeconds = 11 [default = 5]; - optional uint32 SchemeCacheHardRefreshTimeSeconds = 12 [default = 10]; - - optional bool ForceAccessControl = 13 [default = false]; - repeated string AccountsWithoutMandatoryAuth = 14; - - optional bool YandexCloudMode = 15 [default = false]; - optional uint32 YandexCloudServiceId = 16; // only 15 lesser bits should be used! - optional string YandexCloudAccessServiceAddress = 29; // host:port - optional string YandexCloudFolderServiceAddress = 30; // host:port - optional string YandexCloudServiceRegion = 39 [default = "ru-central1"]; - - optional uint32 MeteringFlushingIntervalMs = 51 [default = 5000]; - optional string MeteringLogFilePath = 52; - - repeated string MeteringCloudNetCidr = 53; - repeated string MeteringYandexNetCidr = 54; - + + optional uint32 SchemeCacheSoftRefreshTimeSeconds = 11 [default = 5]; + optional uint32 SchemeCacheHardRefreshTimeSeconds = 12 [default = 10]; + + optional bool ForceAccessControl = 13 [default = false]; + repeated string AccountsWithoutMandatoryAuth = 14; + + optional bool YandexCloudMode = 15 [default = false]; + optional uint32 YandexCloudServiceId = 16; // only 15 lesser bits should be used! + optional string YandexCloudAccessServiceAddress = 29; // host:port + optional string YandexCloudFolderServiceAddress = 30; // host:port + optional string YandexCloudServiceRegion = 39 [default = "ru-central1"]; + + optional uint32 MeteringFlushingIntervalMs = 51 [default = 5000]; + optional string MeteringLogFilePath = 52; + + repeated string MeteringCloudNetCidr = 53; + repeated string MeteringYandexNetCidr = 54; + // TODO: remove both optional uint32 MastersDescriberUpdateTimeMs = 17 [default = 10000]; optional uint32 MasterConnectTimeoutMs = 18 [default = 10000]; @@ -741,10 +741,10 @@ message TSqsConfig { optional uint64 MaxNumberOfReceiveMessages = 24 [default = 10]; // MaxNumberOfMessages parameter in ReceiveMessage optional uint64 RequestTimeoutMs = 25 [default = 600000]; - - optional bool ForceQueueCreationV2 = 26 [default = true]; // deprecated, TODO: remove from config - optional bool ForceQueueDeletionV2 = 27 [default = true]; // deprecated, TODO: remove from config - optional bool EnableDeadLetterQueues = 41 [default = false]; + + optional bool ForceQueueCreationV2 = 26 [default = true]; // deprecated, TODO: remove from config + optional bool ForceQueueDeletionV2 = 27 [default = true]; // deprecated, TODO: remove from config + optional bool EnableDeadLetterQueues = 41 [default = false]; optional bool CreateLegacyDurationCounters = 28; @@ -814,7 +814,7 @@ message TSqsConfig { } optional TQuotingConfig QuotingConfig = 47; - optional bool EnableQueueAttributesValidation = 48 [default = true]; + optional bool EnableQueueAttributesValidation = 48 [default = true]; message TAccountSettingsDefaults { optional int64 MaxQueuesCount = 1 [default = 50]; @@ -832,13 +832,13 @@ message TSqsConfig { optional uint64 QueuesListReadBatchSize = 59 [default = 1000]; optional bool ValidateMessageBody = 60; - - optional uint64 DlqNotificationGracePeriodMs = 61 [default = 60000]; + + optional uint64 DlqNotificationGracePeriodMs = 61 [default = 60000]; optional uint64 AddMessagesToInflyCheckPeriodMs = 62 [default = 30000]; optional uint64 AddMessagesToInflyMinCheckAttempts = 63 [default = 10]; - - optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600]; + + optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600]; optional bool MeteringByNetClassifierOnly = 65 [default = false]; @@ -1370,9 +1370,9 @@ message TAppConfig { optional NKikimrSharedCache.TSharedCacheConfig SharedCacheConfig = 38; // dynamic configuration via cms optional TImmediateControlsConfig ImmediateControlsConfig = 39; optional TAllocatorConfig AllocatorConfig = 40; - optional NKikimrPQ.TPQClusterDiscoveryConfig PQClusterDiscoveryConfig = 41; - optional NKikimrNetClassifier.TNetClassifierConfig NetClassifierConfig = 42; - optional NKikimrNetClassifier.TNetClassifierDistributableConfig NetClassifierDistributableConfig = 43; // also dynamic via cms + optional NKikimrPQ.TPQClusterDiscoveryConfig PQClusterDiscoveryConfig = 41; + optional NKikimrNetClassifier.TNetClassifierConfig NetClassifierConfig = 42; + optional NKikimrNetClassifier.TNetClassifierDistributableConfig NetClassifierDistributableConfig = 43; // also dynamic via cms optional NKikimrResourceBroker.TResourceBrokerConfig ResourceBrokerConfig = 44; optional TMeteringConfig MeteringConfig = 45; optional THiveConfig HiveConfig = 46; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index d5a32d05c6..ead88eafce 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -101,10 +101,10 @@ message TConfigItem { TableServiceConfigItem = 37; SharedCacheConfigItem = 38; ImmediateControlsConfigItem = 39; - AllocatorConfigItem = 40; - PQClusterDiscoveryConfigItem = 41; - NetClassifierConfigItem = 42; - NetClassifierDistributableConfigItem = 43; + AllocatorConfigItem = 40; + PQClusterDiscoveryConfigItem = 41; + NetClassifierConfigItem = 42; + NetClassifierDistributableConfigItem = 43; ResourceBrokerConfigItem = 44; MeteringConfigItem = 45; HiveConfigItem = 46; diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index df7cda5980..d0f3cfa4d8 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -608,13 +608,13 @@ message TSqsRequest { NKikimr.NSQS.TCreateUserRequest CreateUser = 15; NKikimr.NSQS.TDeleteUserRequest DeleteUser = 16; NKikimr.NSQS.TListUsersRequest ListUsers = 17; - NKikimr.NSQS.TModifyPermissionsRequest ModifyPermissions = 18; - NKikimr.NSQS.TListPermissionsRequest ListPermissions = 19; + NKikimr.NSQS.TModifyPermissionsRequest ModifyPermissions = 18; + NKikimr.NSQS.TListPermissionsRequest ListPermissions = 19; NKikimr.NSQS.TDeleteQueueBatchRequest DeleteQueueBatch = 20; NKikimr.NSQS.TPurgeQueueBatchRequest PurgeQueueBatch = 21; NKikimr.NSQS.TGetQueueAttributesBatchRequest GetQueueAttributesBatch = 22; - NKikimr.NSQS.TListDeadLetterSourceQueuesRequest ListDeadLetterSourceQueues = 23; - NKikimr.NSQS.TCountQueuesRequest CountQueues = 24; + NKikimr.NSQS.TListDeadLetterSourceQueuesRequest ListDeadLetterSourceQueues = 23; + NKikimr.NSQS.TCountQueuesRequest CountQueues = 24; } optional string RequestId = 30; optional bool RequestRateLimit = 31 [default = true]; @@ -639,18 +639,18 @@ message TSqsResponse { NKikimr.NSQS.TCreateUserResponse CreateUser = 15; NKikimr.NSQS.TDeleteUserResponse DeleteUser = 16; NKikimr.NSQS.TListUsersResponse ListUsers = 17; - NKikimr.NSQS.TModifyPermissionsResponse ModifyPermissions = 18; - NKikimr.NSQS.TListPermissionsResponse ListPermissions = 19; + NKikimr.NSQS.TModifyPermissionsResponse ModifyPermissions = 18; + NKikimr.NSQS.TListPermissionsResponse ListPermissions = 19; NKikimr.NSQS.TDeleteQueueBatchResponse DeleteQueueBatch = 20; NKikimr.NSQS.TPurgeQueueBatchResponse PurgeQueueBatch = 21; NKikimr.NSQS.TGetQueueAttributesBatchResponse GetQueueAttributesBatch = 22; - NKikimr.NSQS.TListDeadLetterSourceQueuesResponse ListDeadLetterSourceQueues = 23; - NKikimr.NSQS.TCountQueuesResponse CountQueues = 24; + NKikimr.NSQS.TListDeadLetterSourceQueuesResponse ListDeadLetterSourceQueues = 23; + NKikimr.NSQS.TCountQueuesResponse CountQueues = 24; } optional string RequestId = 30; - optional string FolderId = 31; - optional string ResourceId = 32; - optional bool IsFifo = 33; + optional string FolderId = 31; + optional string ResourceId = 32; + optional bool IsFifo = 33; } message TS3ListingRequest { diff --git a/ydb/core/protos/netclassifier.proto b/ydb/core/protos/netclassifier.proto index 0c71eae7bf..5cf1f4be15 100644 --- a/ydb/core/protos/netclassifier.proto +++ b/ydb/core/protos/netclassifier.proto @@ -1,45 +1,45 @@ -package NKikimrNetClassifier; - -option java_package = "ru.yandex.kikimr.proto"; - -message TNetClassifierUpdaterConfig { +package NKikimrNetClassifier; + +option java_package = "ru.yandex.kikimr.proto"; + +message TNetClassifierUpdaterConfig { enum EFormat { TSV = 0; NETBOX = 1; } - optional string NetDataSourceUrl = 1; - optional uint32 RetryIntervalSeconds = 2; - optional uint32 NetDataUpdateIntervalSeconds = 3; + optional string NetDataSourceUrl = 1; + optional uint32 RetryIntervalSeconds = 2; + optional uint32 NetDataUpdateIntervalSeconds = 3; optional EFormat Format = 4 [default = TSV]; repeated string NetBoxTags = 5; -} - -message TSubnet { - optional string Mask = 1; - optional string Label = 2; -} - -message TNetData { - repeated TSubnet Subnets = 1; -} - -message TNetClassifierConfig { - optional TNetClassifierUpdaterConfig UpdaterConfig = 1; - - // The file content is used if distributable config is empty or unavailable. May be empty - optional string NetDataFilePath = 2; - - // CMS has to respond with distributable configuration before the time runs out - optional uint32 CmsConfigTimeoutSeconds = 3 [default = 30]; - - // Timed counters update interval (seconds): - optional uint32 TimedCountersUpdateIntervalSeconds = 4 [default = 15]; -} - -message TNetClassifierDistributableConfig { - // DO NOT SET the config fields manually - optional string LastUpdateDatetimeUTC = 1; - optional uint64 LastUpdateTimestamp = 2; - optional bytes PackedNetData = 3; // Contains serialized and deflated TNetData -} +} + +message TSubnet { + optional string Mask = 1; + optional string Label = 2; +} + +message TNetData { + repeated TSubnet Subnets = 1; +} + +message TNetClassifierConfig { + optional TNetClassifierUpdaterConfig UpdaterConfig = 1; + + // The file content is used if distributable config is empty or unavailable. May be empty + optional string NetDataFilePath = 2; + + // CMS has to respond with distributable configuration before the time runs out + optional uint32 CmsConfigTimeoutSeconds = 3 [default = 30]; + + // Timed counters update interval (seconds): + optional uint32 TimedCountersUpdateIntervalSeconds = 4 [default = 15]; +} + +message TNetClassifierDistributableConfig { + // DO NOT SET the config fields manually + optional string LastUpdateDatetimeUTC = 1; + optional uint64 LastUpdateTimestamp = 2; + optional bytes PackedNetData = 3; // Contains serialized and deflated TNetData +} diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index 7c85927449..7b687ed4b3 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -1,6 +1,6 @@ import "ydb/public/api/protos/draft/persqueue_error_codes.proto"; import "ydb/public/api/protos/draft/persqueue_common.proto"; - + import "ydb/core/protos/base.proto"; import "ydb/core/protos/msgbus_kv.proto"; import "ydb/core/protos/node_limits.proto"; @@ -9,11 +9,11 @@ import "ydb/core/protos/services.proto"; import "library/cpp/actors/protos/actors.proto"; - + package NKikimrPQ; option java_package = "ru.yandex.kikimr.proto"; - + message TPartitionMeta { optional uint64 StartOffset = 1; optional uint64 EndOffset = 2; @@ -42,11 +42,11 @@ message TPQConfig { optional NKikimrClient.TKeyValueRequest.ETactic Tactic = 13 [default = MAX_THROUGHPUT]; optional bool RequireCredentialsInNewProtocol = 14 [default = false]; - - optional string ClusterTablePath = 15 [default = "/Root/PQ/Config/V2/Cluster"]; - optional string VersionTablePath = 16 [default = "/Root/PQ/Config/V2/Versions"]; - - optional uint32 ClustersUpdateTimeoutOnErrorSec = 17 [default = 1]; + + optional string ClusterTablePath = 15 [default = "/Root/PQ/Config/V2/Cluster"]; + optional string VersionTablePath = 16 [default = "/Root/PQ/Config/V2/Versions"]; + + optional uint32 ClustersUpdateTimeoutOnErrorSec = 17 [default = 1]; optional uint32 WriteInitLatencyBigMs = 19 [default = 900]; optional uint32 ReadInitLatencyBigMs = 20 [default = 900]; @@ -675,13 +675,13 @@ message TClientInfoResponse { optional uint64 ResponseTimestamp = 4; repeated TClientInfo ClientInfo = 5; } - -message TPQClusterDiscoveryConfig { - optional bool Enabled = 1; - optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15]; - optional NKikimrNetClassifier.TNetData CloudNetData = 3; + +message TPQClusterDiscoveryConfig { + optional bool Enabled = 1; + optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15]; + optional NKikimrNetClassifier.TNetData CloudNetData = 3; optional uint64 RequestInflightLimit = 4; -} +} message TYdsNextToken { required uint64 CreationTimestamp = 1; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index c17c8a7dc3..3e24042da3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -256,10 +256,10 @@ enum EServiceKikimr { YF_DESCRIPTIONS_SERVICE = 709; YF_WORKER = 710; - NET_CLASSIFIER = 711; - - PERSQUEUE_CLUSTER_TRACKER = 712; - + NET_CLASSIFIER = 711; + + PERSQUEUE_CLUSTER_TRACKER = 712; + YQL_PROXY = 713; YQL_PRIVATE_PROXY = 714; @@ -678,13 +678,13 @@ message TActivity { BS_MONSTREAM_ACTOR = 363; NODE_WARDEN_STATAGGR_ACTOR = 364; UPLOAD_ROWS_INTERNAL = 365; - PERSQUEUE_CLUSTER_TRACKER = 366; - NET_CLASSIFIER_ACTOR = 367; + PERSQUEUE_CLUSTER_TRACKER = 366; + NET_CLASSIFIER_ACTOR = 367; KQP_COMPILE_SERVICE = 368; KQP_COMPILE_ACTOR = 369; - PERSQUEUE_CLUSTER_DISCOVERY = 370; + PERSQUEUE_CLUSTER_DISCOVERY = 370; TX_FILL_INDEX_SCAN = 371; - NET_CLASSIFIER_UPDATER = 372; + NET_CLASSIFIER_UPDATER = 372; BS_LOAD_PDISK_LOG_WRITE = 373; RTMR_CONFIGURATION_STORAGE = 374; KQP_SYSTEM_VIEW_SCAN = 375; diff --git a/ydb/core/protos/sqs.proto b/ydb/core/protos/sqs.proto index eae557a624..c8a83ee484 100644 --- a/ydb/core/protos/sqs.proto +++ b/ydb/core/protos/sqs.proto @@ -4,14 +4,14 @@ message TAuthentification { optional string UserName = 1; } -message TCredentials { - oneof AccessKey { - string OAuthToken = 1; - string TvmTicket = 2; - string StaticCreds = 3; // Cloud only - } -} - +message TCredentials { + oneof AccessKey { + string OAuthToken = 1; + string TvmTicket = 2; + string StaticCreds = 3; // Cloud only + } +} + message TError { message TDebug { optional uint32 TransactionStatusCode = 1; @@ -38,7 +38,7 @@ message TMessageAttribute { } message TChangeMessageVisibilityRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// Name of queue. @@ -60,7 +60,7 @@ message TChangeMessageVisibilityResponse { } message TChangeMessageVisibilityBatchRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// Name of queue. @@ -75,7 +75,7 @@ message TChangeMessageVisibilityBatchResponse { } message TCreateQueueRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// Name of queue. Should be unique for specified user. @@ -90,7 +90,7 @@ message TCreateQueueRequest { optional bool EnableAutosplit = 8; /// Size in bytes when partition will be splitted on two parts. optional uint64 SizeToSplit = 9 [default = 1073741824]; // 1GB - /// Enable internal tables' transactions out of order execution. + /// Enable internal tables' transactions out of order execution. optional bool EnableOutOfOrderTransactionsExecution = 6 [default = true]; } @@ -102,7 +102,7 @@ message TCreateQueueResponse { } message TCreateUserRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// Then user name we want to create. @@ -115,7 +115,7 @@ message TCreateUserResponse { } message TGetQueueAttributesRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -157,10 +157,10 @@ message TGetQueueAttributesResponse { optional bool FifoQueue = 13; /// Is the content-based deduplication is enabled. optional bool ContentBasedDeduplication = 14; - /// Contains maxReceiveCount and target DLQ ARN, defines DLQ params - optional string RedrivePolicy = 16; - /// Contains queue ARN - optional string QueueArn = 17; + /// Contains maxReceiveCount and target DLQ ARN, defines DLQ params + optional string RedrivePolicy = 16; + /// Contains queue ARN + optional string QueueArn = 17; } message TGetQueueAttributesBatchRequest { @@ -179,7 +179,7 @@ message TGetQueueAttributesBatchResponse { } message TGetQueueUrlRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// The name of the queue whose URL must be fetched. @@ -193,7 +193,7 @@ message TGetQueueUrlResponse { } message TDeleteMessageRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -210,7 +210,7 @@ message TDeleteMessageResponse { } message TDeleteMessageBatchRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -224,7 +224,7 @@ message TDeleteMessageBatchResponse { } message TDeleteQueueRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -253,7 +253,7 @@ message TDeleteQueueBatchResponse { } message TDeleteUserRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// Then user name we want to delete. @@ -266,7 +266,7 @@ message TDeleteUserResponse { } message TListQueuesRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// A string to use for filtering the list results. @@ -286,7 +286,7 @@ message TListQueuesResponse { } message TListUsersRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// A string to use for filtering the list results. @@ -302,7 +302,7 @@ message TListUsersResponse { } message TPurgeQueueRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -331,7 +331,7 @@ message TPurgeQueueBatchResponse { } message TReceiveMessageRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; optional string QueueName = 2; @@ -384,8 +384,8 @@ message TReceiveMessageResponse { optional uint64 SentTimestamp = 10; /// The large, non-consecutive number that SQS assigns to each message. optional uint64 SequenceNumber = 11; - /// Message sender's SID, may be empty in case of unauthenticated writes. - optional string SenderId = 13; + /// Message sender's SID, may be empty in case of unauthenticated writes. + optional string SenderId = 13; } optional TError Error = 1; @@ -398,7 +398,7 @@ message TReceiveMessageResponse { } message TSendMessageRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// The name of the queue in which the message should be placed. @@ -434,7 +434,7 @@ message TSendMessageResponse { } message TSendMessageBatchRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// The name of the queue in which the message should be placed. @@ -449,7 +449,7 @@ message TSendMessageBatchResponse { } message TSetQueueAttributesRequest { - optional TCredentials Credentials = 100; + optional TCredentials Credentials = 100; /// Sender's authentification. optional TAuthentification Auth = 1; /// The name of the queue. @@ -462,90 +462,90 @@ message TSetQueueAttributesResponse { optional TError Error = 1; optional string RequestId = 2; } - -message TPermissions { - optional string Subject = 1; - repeated string PermissionNames = 2; // https://st.yandex-team.ru/SQS-3 -} - -message TPermissionsAction { - oneof Action { - TPermissions Grant = 1; - TPermissions Revoke = 2; - TPermissions Set = 3; - } -} - -message TModifyPermissionsRequest { - optional TCredentials Credentials = 100; - optional string Resource = 1; - repeated TPermissionsAction Actions = 2; - optional bool ClearACL = 3 [default = false]; - optional TAuthentification Auth = 4; // for compatibility only -} - -message TModifyPermissionsResponse { - optional TError Error = 1; - optional string RequestId = 2; -} - -message TListPermissionsRequest { - optional TCredentials Credentials = 100; - optional string Path = 1; - optional TAuthentification Auth = 2; // for compatibility only -} - -message TQueuePermissions { - repeated TPermissions Permissions = 1; - repeated TPermissions EffectivePermissions = 2; -} - -message TAccountPermissions { - repeated TPermissions Permissions = 1; - repeated TPermissions EffectivePermissions = 2; -} - -message TDirectoryPermissions { - repeated TPermissions Permissions = 1; - repeated TPermissions EffectivePermissions = 2; -} - -message TListPermissionsResponse { - optional TError Error = 1; - optional string RequestId = 2; - - oneof NodeType { - TQueuePermissions QueuePermissions = 3; - TQueuePermissions AccountPermissions = 4; - TQueuePermissions DirectoryPermissions = 5; // not used right now - } -} - -message TListDeadLetterSourceQueuesRequest { - optional TCredentials Credentials = 100; - /// The name of the queue. - optional string QueueName = 1; + +message TPermissions { + optional string Subject = 1; + repeated string PermissionNames = 2; // https://st.yandex-team.ru/SQS-3 +} + +message TPermissionsAction { + oneof Action { + TPermissions Grant = 1; + TPermissions Revoke = 2; + TPermissions Set = 3; + } +} + +message TModifyPermissionsRequest { + optional TCredentials Credentials = 100; + optional string Resource = 1; + repeated TPermissionsAction Actions = 2; + optional bool ClearACL = 3 [default = false]; + optional TAuthentification Auth = 4; // for compatibility only +} + +message TModifyPermissionsResponse { + optional TError Error = 1; + optional string RequestId = 2; +} + +message TListPermissionsRequest { + optional TCredentials Credentials = 100; + optional string Path = 1; + optional TAuthentification Auth = 2; // for compatibility only +} + +message TQueuePermissions { + repeated TPermissions Permissions = 1; + repeated TPermissions EffectivePermissions = 2; +} + +message TAccountPermissions { + repeated TPermissions Permissions = 1; + repeated TPermissions EffectivePermissions = 2; +} + +message TDirectoryPermissions { + repeated TPermissions Permissions = 1; + repeated TPermissions EffectivePermissions = 2; +} + +message TListPermissionsResponse { + optional TError Error = 1; + optional string RequestId = 2; + + oneof NodeType { + TQueuePermissions QueuePermissions = 3; + TQueuePermissions AccountPermissions = 4; + TQueuePermissions DirectoryPermissions = 5; // not used right now + } +} + +message TListDeadLetterSourceQueuesRequest { + optional TCredentials Credentials = 100; + /// The name of the queue. + optional string QueueName = 1; optional TAuthentification Auth = 2; -} - -message TListDeadLetterSourceQueuesResponse { - message TQueueType { - optional string QueueName = 1; - optional string QueueUrl = 2; - } - optional TError Error = 1; - optional string RequestId = 2; - repeated TQueueType Queues = 3; -} - -message TCountQueuesRequest { - optional TCredentials Credentials = 100; - optional TAuthentification Auth = 1; -} - -message TCountQueuesResponse { - optional TError Error = 1; - optional string RequestId = 2; - - optional uint64 Count = 3; -} +} + +message TListDeadLetterSourceQueuesResponse { + message TQueueType { + optional string QueueName = 1; + optional string QueueUrl = 2; + } + optional TError Error = 1; + optional string RequestId = 2; + repeated TQueueType Queues = 3; +} + +message TCountQueuesRequest { + optional TCredentials Credentials = 100; + optional TAuthentification Auth = 1; +} + +message TCountQueuesResponse { + optional TError Error = 1; + optional string RequestId = 2; + + optional uint64 Count = 3; +} diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 70bb65514c..693ae8ef87 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -78,7 +78,7 @@ SRCS( msgbus_health.proto msgbus_kv.proto msgbus_pq.proto - netclassifier.proto + netclassifier.proto node_broker.proto node_limits.proto profiler.proto diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index 63f0436bff..4804dec8d8 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -126,7 +126,7 @@ namespace NActors { nodeAppData->PollerThreads = node->Poller; nodeAppData->StreamingConfig.SetEnableOutputStreams(true); nodeAppData->PQConfig = app0->PQConfig; - nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig); + nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig); nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig); nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling; nodeAppData->FeatureFlags = app0->FeatureFlags; diff --git a/ydb/core/testlib/basics/appdata.h b/ydb/core/testlib/basics/appdata.h index 409869161f..b65385b077 100644 --- a/ydb/core/testlib/basics/appdata.h +++ b/ydb/core/testlib/basics/appdata.h @@ -75,7 +75,7 @@ namespace NKikimr { TMap<ui32, NKikimrProto::TKeyConfig> Keys; bool EnableKqpSpilling = false; NKikimrConfig::TCompactionConfig CompactionConfig; - TString NetDataSourceUrl; + TString NetDataSourceUrl; NKikimrConfig::THiveConfig HiveConfig; NKikimrConfig::TDataShardConfig DataShardConfig; NKikimrConfig::TMeteringConfig MeteringConfig; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index d4907c26f1..c477916952 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -158,7 +158,7 @@ namespace Tests { return; TAppPrepare app; /* will cook TAppData */ - app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()); + app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()); app.SetEnableKqpSpilling(Settings->EnableKqpSpilling); app.SetKeepSnapshotTimeout(Settings->KeepSnapshotTimeout); app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit); @@ -255,8 +255,8 @@ namespace Tests { auto grpcMon = system->Register(NGRpcService::CreateGrpcMonService(), TMailboxType::ReadAsFilled); system->RegisterLocalService(NGRpcService::GrpcMonServiceId(), grpcMon); - GRpcServerRootCounters = MakeIntrusive<NMonitoring::TDynamicCounters>(); - auto& counters = GRpcServerRootCounters; + GRpcServerRootCounters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + auto& counters = GRpcServerRootCounters; auto& appData = Runtime->GetAppData(); @@ -307,7 +307,7 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcYdbScriptingService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcCmsService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcDiscoveryService(system, counters, grpcRequestProxyId)); @@ -692,13 +692,13 @@ namespace Tests { TActorId kesusServiceId = Runtime->Register(kesusService, nodeIdx); Runtime->RegisterService(NKesus::MakeKesusProxyServiceId(), kesusServiceId, nodeIdx); } - + { IActor* pqClusterTracker = NPQ::NClusterTracker::CreateClusterTracker(); TActorId pqClusterTrackerId = Runtime->Register(pqClusterTracker, nodeIdx); Runtime->RegisterService(NPQ::NClusterTracker::MakeClusterTrackerID(), pqClusterTrackerId, nodeIdx); } - + { IActor* netClassifier = NNetClassifier::CreateNetClassifier(); TActorId netClassifierId = Runtime->Register(netClassifier, nodeIdx); diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 2064752ab4..18af6f49c9 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -91,8 +91,8 @@ namespace Tests { ui16 GrpcPort = 0; NKikimrProto::TAuthConfig AuthConfig; NKikimrPQ::TPQConfig PQConfig; - NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig; - NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig; + NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig; + NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig; ui32 Domain = TestDomain; bool SupportsRedirect = true; TString TracePath; @@ -222,10 +222,10 @@ namespace Tests { void SetupDefaultProfiles(); - TIntrusivePtr<NMonitoring::TDynamicCounters> GetGRpcServerRootCounters() const { - return GRpcServerRootCounters; - } - + TIntrusivePtr<NMonitoring::TDynamicCounters> GetGRpcServerRootCounters() const { + return GRpcServerRootCounters; + } + void ShutdownGRpc() { if (GRpcServer) { GRpcServer->Stop(); @@ -257,7 +257,7 @@ namespace Tests { const NBus::TBusServerSessionConfig BusServerSessionConfig; //BusServer hold const & on config TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer; std::unique_ptr<NGrpc::TGRpcServer> GRpcServer; - TIntrusivePtr<NMonitoring::TDynamicCounters> GRpcServerRootCounters; + TIntrusivePtr<NMonitoring::TDynamicCounters> GRpcServerRootCounters; }; class TClient { diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h index f059923250..7b00ae5c5e 100644 --- a/ydb/core/testlib/test_pq_client.h +++ b/ydb/core/testlib/test_pq_client.h @@ -14,9 +14,9 @@ #include <library/cpp/tvmauth/unittest.h> #include <library/cpp/testing/unittest/registar.h> -#include <util/string/printf.h> +#include <util/string/printf.h> #include <util/system/tempfile.h> - + namespace NKikimr { namespace NPersQueueTests { @@ -612,7 +612,7 @@ public: RunYqlSchemeQuery(R"___( CREATE TABLE [/Root/PQ/Config/V2/Cluster] ( name Utf8, - balancer Utf8, + balancer Utf8, local Bool, enabled Bool, weight Uint64, @@ -626,11 +626,11 @@ public: )___"); RunYqlSchemeQuery(R"___( - CREATE TABLE [/Root/PQ/Config/V2/Versions] ( - name Utf8, - version Int64, - PRIMARY KEY (name) - ); + CREATE TABLE [/Root/PQ/Config/V2/Versions] ( + name Utf8, + version Int64, + PRIMARY KEY (name) + ); )___"); TStringBuilder upsertClusters; @@ -700,25 +700,25 @@ public: ("user5", "1"), ("user3", "2"); UPSERT INTO [/Root/PQ/Config/V2/Producer] (name, tvmClientId) VALUES - ("user4", "2"), - ("topic1", "1"); + ("user4", "2"), + ("topic1", "1"); )___"); } - void UpdateDC(const TString& name, bool local, bool enabled) { - const TString query = Sprintf( - R"___( - UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, local, enabled) VALUES - ("%s", %s, %s); - UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) - SELECT name, version + 1 FROM [/Root/PQ/Config/V2/Versions] WHERE name == "Cluster"; - )___", name.c_str(), (local ? "true" : "false"), (enabled ? "true" : "false")); - + void UpdateDC(const TString& name, bool local, bool enabled) { + const TString query = Sprintf( + R"___( + UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, local, enabled) VALUES + ("%s", %s, %s); + UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) + SELECT name, version + 1 FROM [/Root/PQ/Config/V2/Versions] WHERE name == "Cluster"; + )___", name.c_str(), (local ? "true" : "false"), (enabled ? "true" : "false")); + RunYqlDataQuery(query); - } - + } + void DisableDC() { - UpdateDC("dc1", true, false); + UpdateDC("dc1", true, false); } void RestartSchemeshard(TTestActorRuntime* runtime) { diff --git a/ydb/core/util/address_classifier.cpp b/ydb/core/util/address_classifier.cpp index 46f291e417..01737dc931 100644 --- a/ydb/core/util/address_classifier.cpp +++ b/ydb/core/util/address_classifier.cpp @@ -1,96 +1,96 @@ -#include "address_classifier.h" - -namespace NKikimr::NAddressClassifier { - -TString ExtractAddress(const TString& peer) { - TStringBuf buf(peer); +#include "address_classifier.h" + +namespace NKikimr::NAddressClassifier { + +TString ExtractAddress(const TString& peer) { + TStringBuf buf(peer); if (buf.SkipPrefix(TStringBuf("ipv"))) { - buf.Skip(2); // skip 4/6 and ':' - if (buf.StartsWith('[')) { - buf.Skip(1); - buf = buf.Before(']'); - return TString(buf); - } - } - - buf = buf.RBefore(':'); // remove port - - return TString(buf); -} - -bool TAddressClassifier::AddNetByCidrAndLabel(const TString& cidr, const size_t label) { - try { - // The following method still throws despite its name - const auto maybeRange = TIpAddressRange::TryFromCidrString(cidr); - if (maybeRange.Empty()) { - return false; - } - - if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv4) { - RegisteredV4Nets[label].Add(*maybeRange); - } else if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv6) { - RegisteredV6Nets[label].Add(*maybeRange); - } else { - Y_VERIFY(false); // unknown net type? - } - } catch (yexception&) { - return false; - } - - return true; -} - -std::pair<bool, size_t> TAddressClassifier::ClassifyAddress(const TString& address) const { - bool ok; - const auto inetAddress = TIpv6Address::FromString(address, ok); - if (!ok) { - return UnknownAddressClass; - } - - auto findAddress = [&inetAddress](const auto& registeredNets) -> std::pair<bool, size_t> { - for (const auto& [label, rangeSet] : registeredNets) { - if (rangeSet.Contains(inetAddress)) { - return {true, label}; - } - } - return UnknownAddressClass; - }; - - if (inetAddress.Type() == TIpv6Address::TIpType::Ipv4) { - return findAddress(RegisteredV4Nets); - } else if (inetAddress.Type() == TIpv6Address::TIpType::Ipv6) { - return findAddress(RegisteredV6Nets); - } else { - Y_VERIFY(false); // unknown net type? - } - - return UnknownAddressClass; -} - -TLabeledAddressClassifier::TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) - : Classifier(std::move(addressClassifier)) - , Labels(std::move(labels)) - { - } - -TLabeledAddressClassifier::TConstPtr TLabeledAddressClassifier::MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) { - return MakeIntrusiveConst<TLabeledAddressClassifier>(std::move(addressClassifier), std::move(labels)); -} - -TMaybe<TString> TLabeledAddressClassifier::ClassifyAddress(const TString& address) const { - TMaybe<TString> result = Nothing(); - - const auto netClass = Classifier.ClassifyAddress(address); - if (netClass != TAddressClassifier::UnknownAddressClass) { - Y_VERIFY(netClass.second < Labels.size()); - result = Labels[netClass.second]; - } - - return result; -} - -const std::vector<TString>& TLabeledAddressClassifier::GetLabels() const { - return Labels; -} - -} // namespace NKikimr::NAddressClassifier + buf.Skip(2); // skip 4/6 and ':' + if (buf.StartsWith('[')) { + buf.Skip(1); + buf = buf.Before(']'); + return TString(buf); + } + } + + buf = buf.RBefore(':'); // remove port + + return TString(buf); +} + +bool TAddressClassifier::AddNetByCidrAndLabel(const TString& cidr, const size_t label) { + try { + // The following method still throws despite its name + const auto maybeRange = TIpAddressRange::TryFromCidrString(cidr); + if (maybeRange.Empty()) { + return false; + } + + if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv4) { + RegisteredV4Nets[label].Add(*maybeRange); + } else if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv6) { + RegisteredV6Nets[label].Add(*maybeRange); + } else { + Y_VERIFY(false); // unknown net type? + } + } catch (yexception&) { + return false; + } + + return true; +} + +std::pair<bool, size_t> TAddressClassifier::ClassifyAddress(const TString& address) const { + bool ok; + const auto inetAddress = TIpv6Address::FromString(address, ok); + if (!ok) { + return UnknownAddressClass; + } + + auto findAddress = [&inetAddress](const auto& registeredNets) -> std::pair<bool, size_t> { + for (const auto& [label, rangeSet] : registeredNets) { + if (rangeSet.Contains(inetAddress)) { + return {true, label}; + } + } + return UnknownAddressClass; + }; + + if (inetAddress.Type() == TIpv6Address::TIpType::Ipv4) { + return findAddress(RegisteredV4Nets); + } else if (inetAddress.Type() == TIpv6Address::TIpType::Ipv6) { + return findAddress(RegisteredV6Nets); + } else { + Y_VERIFY(false); // unknown net type? + } + + return UnknownAddressClass; +} + +TLabeledAddressClassifier::TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) + : Classifier(std::move(addressClassifier)) + , Labels(std::move(labels)) + { + } + +TLabeledAddressClassifier::TConstPtr TLabeledAddressClassifier::MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) { + return MakeIntrusiveConst<TLabeledAddressClassifier>(std::move(addressClassifier), std::move(labels)); +} + +TMaybe<TString> TLabeledAddressClassifier::ClassifyAddress(const TString& address) const { + TMaybe<TString> result = Nothing(); + + const auto netClass = Classifier.ClassifyAddress(address); + if (netClass != TAddressClassifier::UnknownAddressClass) { + Y_VERIFY(netClass.second < Labels.size()); + result = Labels[netClass.second]; + } + + return result; +} + +const std::vector<TString>& TLabeledAddressClassifier::GetLabels() const { + return Labels; +} + +} // namespace NKikimr::NAddressClassifier diff --git a/ydb/core/util/address_classifier.h b/ydb/core/util/address_classifier.h index f5e25cd3e4..bf4246dc3b 100644 --- a/ydb/core/util/address_classifier.h +++ b/ydb/core/util/address_classifier.h @@ -1,75 +1,75 @@ -#pragma once - +#pragma once + #include <library/cpp/ipmath/range_set.h> - -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> - -#include <vector> - -namespace NKikimr::NAddressClassifier { - -TString ExtractAddress(const TString& grpcPeer); - -class TAddressClassifier { -public: - bool AddNetByCidrAndLabel(const TString& cidr, const size_t label); - std::pair<bool, size_t> ClassifyAddress(const TString& address) const; - - static constexpr std::pair<bool, size_t> UnknownAddressClass = {false, 0}; - - TAddressClassifier() = default; - TAddressClassifier(TAddressClassifier&&) = default; - TAddressClassifier(const TAddressClassifier&) = delete; - -private: - THashMap<int, TIpRangeSet> RegisteredV4Nets; - THashMap<int, TIpRangeSet> RegisteredV6Nets; -}; - -class TLabeledAddressClassifier : public TAtomicRefCount<TLabeledAddressClassifier>, TNonCopyable { -public: - using TConstPtr = TIntrusiveConstPtr<TLabeledAddressClassifier>; - -public: - TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels); - static TConstPtr MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels); - TMaybe<TString> ClassifyAddress(const TString& address) const; - - const std::vector<TString>& GetLabels() const; - -private: - TAddressClassifier Classifier; - std::vector<TString> Labels; -}; - -template <typename TNetData> -TLabeledAddressClassifier::TConstPtr BuildLabeledAddressClassifierFromNetData(const TNetData& netData) { - TAddressClassifier addressClassifier; - THashMap<TString, size_t> knownLabels; - std::vector<TString> labels; - - for (size_t i = 0; i < netData.SubnetsSize(); ++i) { - const auto& subnet = netData.GetSubnets(i); - - auto it = knownLabels.find(subnet.GetLabel()); - - size_t subnetIntLabel = 0; - if (it == knownLabels.end()) { - labels.push_back(subnet.GetLabel()); - subnetIntLabel = labels.size() - 1; - knownLabels.insert(std::make_pair(labels.back(), subnetIntLabel)); - } else { - subnetIntLabel = it->second; - } - - if (!addressClassifier.AddNetByCidrAndLabel(subnet.GetMask(), subnetIntLabel)) { - return nullptr; - } - } - - return TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels)); -} - -} // namespace NKikimr::NAddressClassifier + +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> + +#include <vector> + +namespace NKikimr::NAddressClassifier { + +TString ExtractAddress(const TString& grpcPeer); + +class TAddressClassifier { +public: + bool AddNetByCidrAndLabel(const TString& cidr, const size_t label); + std::pair<bool, size_t> ClassifyAddress(const TString& address) const; + + static constexpr std::pair<bool, size_t> UnknownAddressClass = {false, 0}; + + TAddressClassifier() = default; + TAddressClassifier(TAddressClassifier&&) = default; + TAddressClassifier(const TAddressClassifier&) = delete; + +private: + THashMap<int, TIpRangeSet> RegisteredV4Nets; + THashMap<int, TIpRangeSet> RegisteredV6Nets; +}; + +class TLabeledAddressClassifier : public TAtomicRefCount<TLabeledAddressClassifier>, TNonCopyable { +public: + using TConstPtr = TIntrusiveConstPtr<TLabeledAddressClassifier>; + +public: + TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels); + static TConstPtr MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels); + TMaybe<TString> ClassifyAddress(const TString& address) const; + + const std::vector<TString>& GetLabels() const; + +private: + TAddressClassifier Classifier; + std::vector<TString> Labels; +}; + +template <typename TNetData> +TLabeledAddressClassifier::TConstPtr BuildLabeledAddressClassifierFromNetData(const TNetData& netData) { + TAddressClassifier addressClassifier; + THashMap<TString, size_t> knownLabels; + std::vector<TString> labels; + + for (size_t i = 0; i < netData.SubnetsSize(); ++i) { + const auto& subnet = netData.GetSubnets(i); + + auto it = knownLabels.find(subnet.GetLabel()); + + size_t subnetIntLabel = 0; + if (it == knownLabels.end()) { + labels.push_back(subnet.GetLabel()); + subnetIntLabel = labels.size() - 1; + knownLabels.insert(std::make_pair(labels.back(), subnetIntLabel)); + } else { + subnetIntLabel = it->second; + } + + if (!addressClassifier.AddNetByCidrAndLabel(subnet.GetMask(), subnetIntLabel)) { + return nullptr; + } + } + + return TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels)); +} + +} // namespace NKikimr::NAddressClassifier diff --git a/ydb/core/util/address_classifier_ut.cpp b/ydb/core/util/address_classifier_ut.cpp index aaa2c8e3e5..0fb0513e7a 100644 --- a/ydb/core/util/address_classifier_ut.cpp +++ b/ydb/core/util/address_classifier_ut.cpp @@ -1,135 +1,135 @@ #include <ydb/core/util/address_classifier.h> - + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/string.h> - -#include <vector> - -namespace NKikimr::NAddressClassifier { - -struct TSubnet { - TString Mask; - TString Label; - - const TString& GetMask() const { return Mask; } - const TString& GetLabel() const { return Label; } -}; - -struct TNetData { - const auto& GetSubnets(const size_t i) const { - return Subnets[i]; - } - - size_t SubnetsSize() const { - return Subnets.size(); - } - - void AddSubnet(const TString& mask, const TString& label) { - Subnets.push_back({mask, label}); - } - -private: - std::vector<TSubnet> Subnets; -}; - -Y_UNIT_TEST_SUITE(AddressClassifierTest) { - Y_UNIT_TEST(TestAddressExtraction) { - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv6:[::1]:58100"), "::1"); - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:[192.168.0.1]:1556"), "192.168.0.1"); - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1:1550"), "192.168.0.1"); - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1"), "192.168.0.1"); - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1:1550"), "192.168.0.1"); - UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1"), "192.168.0.1"); - } - - Y_UNIT_TEST(TestClassfierWithAllIpTypes) { - TAddressClassifier classifier; - - UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("5....../", 34)); - UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("omglol", 666)); - UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/xx", 777)); - - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 42)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.196.0/24", 42)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 24)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a0d:d6c0::/29", 8)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 10)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 10)); - - const THashMap<TString, std::pair<bool, size_t>> tests = { - {"2a02:6b8:bf00::", {true, 42}}, - {"2a02:6b8:af00::", TAddressClassifier::UnknownAddressClass}, - - {"5.45.196.0", {true, 42}}, - {"5.45.196.1", {true, 42}}, - {"5.45.196.5", {true, 42}}, - {"5.45.195.0", TAddressClassifier::UnknownAddressClass}, - - {"5.45.217.0", {true, 24}}, - {"5.45.217.192", {true, 24}}, - {"5.45.215.192", TAddressClassifier::UnknownAddressClass}, - {"5.44.217.192", TAddressClassifier::UnknownAddressClass}, - - {"2a0d:d6c0::", {true, 8}}, - {"2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", {true, 8}}, - {"2a0d:d7c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", TAddressClassifier::UnknownAddressClass}, - - {"wtflol", TAddressClassifier::UnknownAddressClass}, - - {"127.255.255.255", {true, 10}}, - {"::1", {true, 10}}, - }; - - for (const auto& test : tests) { - UNIT_ASSERT_EQUAL_C(test.second, classifier.ClassifyAddress(test.first), TString("Failed at ") << test.first); - } - } - - const std::vector<TString> labels = { "local6", "local4", "some6", "some4" }; - - Y_UNIT_TEST(TestLabeledClassifier) { - TAddressClassifier classifier; - - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 0)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 1)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 2)); - UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 3)); - UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("test", 3)); - - auto labeledClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(classifier), std::vector<TString>(labels)); - UNIT_ASSERT(!labeledClassifier->ClassifyAddress("2a02:6b8:af00::")); - UNIT_ASSERT(!labeledClassifier->ClassifyAddress("lolwut")); - - UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("127.255.255.255"), "local4"); - UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("::1"), "local6"); - UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("5.45.217.192"), "some4"); - UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("2a02:6b8:bf00::"), "some6"); - - UNIT_ASSERT_EQUAL(labeledClassifier->GetLabels(), labels); - } - - Y_UNIT_TEST(TestLabeledClassifierFromNetData) { - TAddressClassifier classifier; - - TNetData netData; - - netData.AddSubnet("::1/128", labels[0]); - netData.AddSubnet("127.0.0.0/8", labels[1]); - netData.AddSubnet("2a02:6b8:bf00::/40", labels[2]); - netData.AddSubnet("5.45.217.0/24", labels[3]); - netData.AddSubnet("2a0d:d6c0::/29", labels[2]); - - auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData); - UNIT_ASSERT(labeledAddressClassifier); - UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a0d:d6c0::"), labels[2]); - UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a02:6b8:bf00::"), labels[2]); - UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("::1"), labels[0]); - UNIT_ASSERT_EQUAL(labeledAddressClassifier->GetLabels(), labels); - - netData.AddSubnet("invalid subnet", "some label"); - UNIT_ASSERT(!BuildLabeledAddressClassifierFromNetData(netData)); - } -} - -} // namespace NKikimr::NAddressClassifier + +#include <util/generic/string.h> + +#include <vector> + +namespace NKikimr::NAddressClassifier { + +struct TSubnet { + TString Mask; + TString Label; + + const TString& GetMask() const { return Mask; } + const TString& GetLabel() const { return Label; } +}; + +struct TNetData { + const auto& GetSubnets(const size_t i) const { + return Subnets[i]; + } + + size_t SubnetsSize() const { + return Subnets.size(); + } + + void AddSubnet(const TString& mask, const TString& label) { + Subnets.push_back({mask, label}); + } + +private: + std::vector<TSubnet> Subnets; +}; + +Y_UNIT_TEST_SUITE(AddressClassifierTest) { + Y_UNIT_TEST(TestAddressExtraction) { + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv6:[::1]:58100"), "::1"); + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:[192.168.0.1]:1556"), "192.168.0.1"); + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1:1550"), "192.168.0.1"); + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1"), "192.168.0.1"); + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1:1550"), "192.168.0.1"); + UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1"), "192.168.0.1"); + } + + Y_UNIT_TEST(TestClassfierWithAllIpTypes) { + TAddressClassifier classifier; + + UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("5....../", 34)); + UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("omglol", 666)); + UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/xx", 777)); + + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 42)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.196.0/24", 42)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 24)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a0d:d6c0::/29", 8)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 10)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 10)); + + const THashMap<TString, std::pair<bool, size_t>> tests = { + {"2a02:6b8:bf00::", {true, 42}}, + {"2a02:6b8:af00::", TAddressClassifier::UnknownAddressClass}, + + {"5.45.196.0", {true, 42}}, + {"5.45.196.1", {true, 42}}, + {"5.45.196.5", {true, 42}}, + {"5.45.195.0", TAddressClassifier::UnknownAddressClass}, + + {"5.45.217.0", {true, 24}}, + {"5.45.217.192", {true, 24}}, + {"5.45.215.192", TAddressClassifier::UnknownAddressClass}, + {"5.44.217.192", TAddressClassifier::UnknownAddressClass}, + + {"2a0d:d6c0::", {true, 8}}, + {"2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", {true, 8}}, + {"2a0d:d7c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", TAddressClassifier::UnknownAddressClass}, + + {"wtflol", TAddressClassifier::UnknownAddressClass}, + + {"127.255.255.255", {true, 10}}, + {"::1", {true, 10}}, + }; + + for (const auto& test : tests) { + UNIT_ASSERT_EQUAL_C(test.second, classifier.ClassifyAddress(test.first), TString("Failed at ") << test.first); + } + } + + const std::vector<TString> labels = { "local6", "local4", "some6", "some4" }; + + Y_UNIT_TEST(TestLabeledClassifier) { + TAddressClassifier classifier; + + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 0)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 1)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 2)); + UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 3)); + UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("test", 3)); + + auto labeledClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(classifier), std::vector<TString>(labels)); + UNIT_ASSERT(!labeledClassifier->ClassifyAddress("2a02:6b8:af00::")); + UNIT_ASSERT(!labeledClassifier->ClassifyAddress("lolwut")); + + UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("127.255.255.255"), "local4"); + UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("::1"), "local6"); + UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("5.45.217.192"), "some4"); + UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("2a02:6b8:bf00::"), "some6"); + + UNIT_ASSERT_EQUAL(labeledClassifier->GetLabels(), labels); + } + + Y_UNIT_TEST(TestLabeledClassifierFromNetData) { + TAddressClassifier classifier; + + TNetData netData; + + netData.AddSubnet("::1/128", labels[0]); + netData.AddSubnet("127.0.0.0/8", labels[1]); + netData.AddSubnet("2a02:6b8:bf00::/40", labels[2]); + netData.AddSubnet("5.45.217.0/24", labels[3]); + netData.AddSubnet("2a0d:d6c0::/29", labels[2]); + + auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData); + UNIT_ASSERT(labeledAddressClassifier); + UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a0d:d6c0::"), labels[2]); + UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a02:6b8:bf00::"), labels[2]); + UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("::1"), labels[0]); + UNIT_ASSERT_EQUAL(labeledAddressClassifier->GetLabels(), labels); + + netData.AddSubnet("invalid subnet", "some label"); + UNIT_ASSERT(!BuildLabeledAddressClassifierFromNetData(netData)); + } +} + +} // namespace NKikimr::NAddressClassifier diff --git a/ydb/core/util/ut/ya.make b/ydb/core/util/ut/ya.make index 315713cad8..7a558fa9b6 100644 --- a/ydb/core/util/ut/ya.make +++ b/ydb/core/util/ut/ya.make @@ -20,7 +20,7 @@ PEERDIR( ) SRCS( - address_classifier_ut.cpp + address_classifier_ut.cpp bits_ut.cpp btree_cow_ut.cpp btree_ut.cpp diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 4f325d8fcb..1debe33bd9 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -6,7 +6,7 @@ OWNER( ) SRCS( - address_classifier.cpp + address_classifier.cpp cache.cpp cache.h cache_cache.h diff --git a/ydb/core/ymq/actor/action.h b/ydb/core/ymq/actor/action.h index 1801c98d68..bb4888b301 100644 --- a/ydb/core/ymq/actor/action.h +++ b/ydb/core/ymq/actor/action.h @@ -9,7 +9,7 @@ #include "log.h" #include "proxy_actor.h" #include "serviceid.h" -#include "schema.h" +#include "schema.h" #include <ydb/core/base/path.h> #include <ydb/core/base/ticket_parser.h> @@ -25,9 +25,9 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> -#include <util/folder/path.h> +#include <util/folder/path.h> #include <util/generic/guid.h> -#include <util/generic/is_in.h> +#include <util/generic/is_in.h> #include <util/string/ascii.h> #include <util/string/join.h> @@ -66,27 +66,27 @@ public: return true; } - static constexpr bool CreateMissingAccount() { - return false; - } + static constexpr bool CreateMissingAccount() { + return false; + } static constexpr bool NeedUserSpecified() { return true; } void DoCloudBootstrap() { - if (!SecurityToken_) { - // TODO: use access service + if (!SecurityToken_) { + // TODO: use access service MakeError(MutableErrorDesc(), NErrors::INVALID_CLIENT_TOKEN_ID, "Failed to parse cloud id."); SendReplyAndDie(); - return; - } - - TStringBuf tokenBuf(SecurityToken_); - UserName_ = TString(tokenBuf.NextTok(':')); - FolderId_ = TString(tokenBuf); - } - + return; + } + + TStringBuf tokenBuf(SecurityToken_); + UserName_ = TString(tokenBuf.NextTok(':')); + FolderId_ = TString(tokenBuf); + } + void DoBootstrap() { ui64 configurationFlags = 0; if (TDerived::NeedQueueAttributes()) { @@ -102,67 +102,67 @@ public: GetQueueName(), configurationFlags) ); - } + } void CreateAccountOnTheFly() const { - // TODO: move to separate actor + // TODO: move to separate actor this->Register( new TCreateUserSchemaActor(Cfg().GetRoot(), UserName_, this->SelfId(), RequestId_, UserCounters_) - ); - } - + ); + } + void HandleAccountCreated(TSqsEvents::TEvUserCreated::TPtr& ev) { auto* detailedCounters = UserCounters_ ? UserCounters_->GetDetailedCounters() : nullptr; - if (ev->Get()->Success) { + if (ev->Get()->Success) { INC_COUNTER(detailedCounters, CreateAccountOnTheFly_Success); - } else { + } else { RLOG_SQS_ERROR("Failed to create cloud account on the fly. Account name: " << UserName_); - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); INC_COUNTER(detailedCounters, CreateAccountOnTheFly_Errors); SendReplyAndDie(); - return; - } - + return; + } + DoBootstrap(); - } - + } + void Bootstrap(const NActors::TActorContext&) { RLOG_SQS_DEBUG("Request started. Actor: " << this->SelfId()); // log new request id StartTs_ = TActivationContext::Now(); - + const auto& cfg = Cfg(); - + this->Become(&TActionActor::InitialState); - + // Set timeout if (cfg.GetRequestTimeoutMs()) { this->Schedule(TDuration::MilliSeconds(cfg.GetRequestTimeoutMs()), new TEvWakeup(REQUEST_TIMEOUT_WAKEUP_TAG), TimeoutCookie_.Get()); } - if (IsCloud()) { + if (IsCloud()) { DoCloudBootstrap(); - - if (TDerived::CreateMissingAccount()) { + + if (TDerived::CreateMissingAccount()) { CreateAccountOnTheFly(); - - return; - } - } - + + return; + } + } + DoBootstrap(); } protected: - template<typename TReq> - void CopySecurityToken(const TReq& request) { - SecurityToken_ = ExtractSecurityToken<TReq, TCredentials>(request); - } - - template<typename TReq> - void CopyAccountName(const TReq& request) { - UserName_ = request.GetAuth().GetUserName(); - } - + template<typename TReq> + void CopySecurityToken(const TReq& request) { + SecurityToken_ = ExtractSecurityToken<TReq, TCredentials>(request); + } + + template<typename TReq> + void CopyAccountName(const TReq& request) { + UserName_ = request.GetAuth().GetUserName(); + } + virtual void DoAction() = 0; virtual TError* MutableErrorDesc() = 0; @@ -180,11 +180,11 @@ protected: return true; } - virtual bool IsFifoQueue() const { - Y_VERIFY(IsFifo_); - return *IsFifo_; - } - + virtual bool IsFifoQueue() const { + Y_VERIFY(IsFifo_); + return *IsFifo_; + } + virtual void DoStart() { } virtual void DoFinish() { } @@ -289,13 +289,13 @@ protected: PrintSlowRequestWarning(); } Finish(); - + if (Cfg().GetYandexCloudMode()) { - Response_.SetFolderId(FolderId_); - Response_.SetIsFifo(IsFifo_ ? *IsFifo_ : false); - Response_.SetResourceId(GetQueueName()); - } - + Response_.SetFolderId(FolderId_); + Response_.SetIsFifo(IsFifo_ ? *IsFifo_ : false); + Response_.SetResourceId(GetQueueName()); + } + Cb_->DoSendReply(Response_); PassAway(); } @@ -317,11 +317,11 @@ protected: Start(); if (Validate()) { DoAction(); - } else { + } else { SendReplyAndDie(); - } - } - + } + } + // Duration of request virtual TDuration GetRequestDuration() const { return FinishTs_ - StartTs_; @@ -337,10 +337,10 @@ protected: return GetRequestDuration() - GetRequestWaitDuration(); } - virtual TString GetCustomACLPath() const { - return GetQueuePath().GetQueuePath(); - } - + virtual TString GetCustomACLPath() const { + return GetQueuePath().GetQueuePath(); + } + virtual bool IsRequestSlow() const { return GetRequestWorkingDuration() >= TDuration::MilliSeconds(Cfg().GetSlowRequestTimeMs()); } @@ -349,57 +349,57 @@ protected: RLOG_SQS_INFO("Request [" << UserName_ << "] [" << GetQueueName() << "] [" << Action_ << "] is slow. Working duration: " << GetRequestWorkingDuration().MilliSeconds() << "ms"); } - TString SanitizeNodePath(const TString& path) const { - TStringBuf sanitizedPath(path); - // just skip SQS root path if there's such a prefix + TString SanitizeNodePath(const TString& path) const { + TStringBuf sanitizedPath(path); + // just skip SQS root path if there's such a prefix if (sanitizedPath.SkipPrefix(TStringBuf(Cfg().GetRoot()))) { // always skip SQS root prefix - return TString(sanitizedPath); - } else { - Y_VERIFY(false); // should never be applied in any other way - } - - return {}; - } - - TString MakeAbsolutePath(const TString& relativePath) const { - TStringBuilder fullPath; + return TString(sanitizedPath); + } else { + Y_VERIFY(false); // should never be applied in any other way + } + + return {}; + } + + TString MakeAbsolutePath(const TString& relativePath) const { + TStringBuilder fullPath; fullPath << Cfg().GetRoot(); - if (!relativePath.StartsWith("/")) { - fullPath << "/"; - } - fullPath << relativePath; - - return TString(fullPath); - } - - size_t CalculatePathDepth(const TString& path) const { - const TString sanitizedResource = TFsPath(path).Fix().GetPath(); - size_t count = 0; - for (size_t i = 0, sz = sanitizedResource.size(); i < sz; ++i) { - if (sanitizedResource[i] == '/') { - ++count; - } - } - - return count; - } - - bool IsCloud() const { + if (!relativePath.StartsWith("/")) { + fullPath << "/"; + } + fullPath << relativePath; + + return TString(fullPath); + } + + size_t CalculatePathDepth(const TString& path) const { + const TString sanitizedResource = TFsPath(path).Fix().GetPath(); + size_t count = 0; + for (size_t i = 0, sz = sanitizedResource.size(); i < sz; ++i) { + if (sanitizedResource[i] == '/') { + ++count; + } + } + + return count; + } + + bool IsCloud() const { return Cfg().GetYandexCloudMode(); - } - - bool IsInternalResource(const TString& path) const { - return CalculatePathDepth(SanitizeNodePath(path)) > 2; - } - - bool IsForbiddenPath(const TString& path) const { - return path.Contains("..") || path.Contains("//") || IsInternalResource(path); - } + } + + bool IsInternalResource(const TString& path) const { + return CalculatePathDepth(SanitizeNodePath(path)) > 2; + } + + bool IsForbiddenPath(const TString& path) const { + return path.Contains("..") || path.Contains("//") || IsInternalResource(path); + } struct TActionCountersPack { TActionCounters* CoreCounters = nullptr; TActionCounters* YmqCounters = nullptr; }; - + TCountersCouple<TActionCounters*> GetActionCounters() const { TCountersCouple<TActionCounters*> result{nullptr, nullptr}; if (IsActionForQueue(Action_) && QueueCounters_) { @@ -424,16 +424,16 @@ protected: } void RequestSchemeCache(const TString& path) { - auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - - entry.Path = SplitPath(path); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - schemeCacheRequest->ResultSet.emplace_back(entry); - + auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + + entry.Path = SplitPath(path); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + schemeCacheRequest->ResultSet.emplace_back(entry); + this->Send(SchemeCache_, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); - } - + } + private: STATEFN(InitialState) { switch (ev->GetTypeRewrite()) { @@ -444,13 +444,13 @@ private: } STATEFN(WaitAuthCheckMessages) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse); hFunc(TEvTicketParser::TEvAuthorizeTicketResult, HandleTicketParserResponse); hFunc(TEvWakeup, HandleWakeup); - } - } - + } + } + STATEFN(WaitQuotaState) { switch (ev->GetTypeRewrite()) { hFunc(TEvQuota::TEvClearance, HandleQuota); @@ -458,42 +458,42 @@ private: } } - TString GetActionACLSourcePath() const { + TString GetActionACLSourcePath() const { const EACLSourceType aclSourceType = GetActionACLSourceType(ToString(Action_)); - switch (aclSourceType) { - case EACLSourceType::Unknown: { - return {}; - } - case EACLSourceType::RootDir: { - return GetQueuePath().GetRootPath(); - } - case EACLSourceType::AccountDir: { - return GetQueuePath().GetUserPath(); - } - case EACLSourceType::QueueDir: { - return GetQueuePath().GetQueuePath(); - } - case EACLSourceType::Custom: { - return GetCustomACLPath(); - } - } - - return {}; - } - + switch (aclSourceType) { + case EACLSourceType::Unknown: { + return {}; + } + case EACLSourceType::RootDir: { + return GetQueuePath().GetRootPath(); + } + case EACLSourceType::AccountDir: { + return GetQueuePath().GetUserPath(); + } + case EACLSourceType::QueueDir: { + return GetQueuePath().GetQueuePath(); + } + case EACLSourceType::Custom: { + return GetCustomACLPath(); + } + } + + return {}; + } + void RequestTicketParser() { this->Send(MakeTicketParserID(), new TEvTicketParser::TEvAuthorizeTicket(SecurityToken_)); - } - - bool IsACLProtectedAccount(const TString& accountName) const { - if (accountName) { - // temporary O(N) solution since the list contains up to 100 items + } + + bool IsACLProtectedAccount(const TString& accountName) const { + if (accountName) { + // temporary O(N) solution since the list contains up to 100 items return !IsIn(Cfg().GetAccountsWithoutMandatoryAuth(), accountName); - } - - return true; - } - + } + + return true; + } + void HandleConfiguration(TSqsEvents::TEvConfiguration::TPtr& ev) { const TDuration confDuration = TActivationContext::Now() - StartTs_; RLOG_SQS_DEBUG("Get configuration duration: " << confDuration.MilliSeconds() << "ms"); @@ -502,17 +502,17 @@ private: UserExists_ = ev->Get()->UserExists; QueueExists_ = ev->Get()->QueueExists; Shards_ = ev->Get()->Shards; - IsFifo_ = ev->Get()->Fifo; + IsFifo_ = ev->Get()->Fifo; QueueAttributes_ = std::move(ev->Get()->QueueAttributes); - SchemeCache_ = ev->Get()->SchemeCache; + SchemeCache_ = ev->Get()->SchemeCache; SqsCoreCounters_ = std::move(ev->Get()->SqsCoreCounters); QueueCounters_ = std::move(ev->Get()->QueueCounters); UserCounters_ = std::move(ev->Get()->UserCounters); QueueLeader_ = ev->Get()->QueueLeader; QuoterResources_ = std::move(ev->Get()->QuoterResources); - Y_VERIFY(SchemeCache_); - + Y_VERIFY(SchemeCache_); + RLOG_SQS_TRACE("Got configuration. Root url: " << RootUrl_ << ", Shards: " << Shards_ << ", Fail: " << ev->Get()->Fail); @@ -548,15 +548,15 @@ private: bool isACLProtectedAccount = Cfg().GetForceAccessControl(); if (!IsCloud() && (SecurityToken_ || (Cfg().GetForceAccessControl() && (isACLProtectedAccount = IsACLProtectedAccount(UserName_))))) { - this->Become(&TActionActor::WaitAuthCheckMessages); - const auto& actionACLSourcePath = GetActionACLSourcePath(); - if (!actionACLSourcePath || IsForbiddenPath(actionACLSourcePath)) { + this->Become(&TActionActor::WaitAuthCheckMessages); + const auto& actionACLSourcePath = GetActionACLSourcePath(); + if (!actionACLSourcePath || IsForbiddenPath(actionACLSourcePath)) { RLOG_SQS_ERROR("Bad ACL source path " << actionACLSourcePath << " for " << Action_ << " action"); - MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); + MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); SendReplyAndDie(); - return; - } - + return; + } + if (!SecurityToken_) { MakeError(MutableErrorDesc(), NErrors::INVALID_CLIENT_TOKEN_ID, "No security token was provided."); SendReplyAndDie(); @@ -569,69 +569,69 @@ private: if (!isACLProtectedAccount) { // !IsCloud && !SecurityToken_ && account is in AccountsWithoutMandatoryAuth setting. INC_COUNTER(UserCounters_, UnauthenticatedAccess); // if !ForceAccessControl, this counter is not initialized. } - // old habits + // old habits DoGetQuotaAndProcess(); - } - } - + } + } + void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); - const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); - - Y_VERIFY(navigate->ResultSet.size() == 1); - - if (navigate->ErrorCount > 0) { - const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; + TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); + const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); + + Y_VERIFY(navigate->ResultSet.size() == 1); + + if (navigate->ErrorCount > 0) { + const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; RLOG_SQS_ERROR("Failed to read ACL for " << GetActionACLSourcePath() << ". Scheme cache error: " << status); - - if (status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) { - MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); - } else { - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); - } - + + if (status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) { + MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); + } else { + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + } + SendReplyAndDie(); - return; + return; } - - SecurityObject_ = navigate->ResultSet.front().SecurityObject; - + + SecurityObject_ = navigate->ResultSet.front().SecurityObject; + OnAuthCheckMessage(); - } - + } + void HandleTicketParserResponse(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev) { const TEvTicketParser::TEvAuthorizeTicketResult& result(*ev->Get()); - if (!result.Error.empty()) { + if (!result.Error.empty()) { RLOG_SQS_ERROR("Got ticket parser error: " << result.Error << ". " << Action_ << " was rejected"); - MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); + MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); SendReplyAndDie(); - return; - } else { - UserToken_ = ev->Get()->Token; - Y_VERIFY(UserToken_); - } - + return; + } else { + UserToken_ = ev->Get()->Token; + Y_VERIFY(UserToken_); + } + OnAuthCheckMessage(); } void OnAuthCheckMessage() { - --SecurityCheckRequestsToWaitFor_; - - if (SecurityCheckRequestsToWaitFor_ == 0) { + --SecurityCheckRequestsToWaitFor_; + + if (SecurityCheckRequestsToWaitFor_ == 0) { const TString& actionName = ToString(Action_); const ui32 requiredAccess = GetActionRequiredAccess(actionName); - UserSID_ = UserToken_->GetUserSID(); - if (requiredAccess != 0 && SecurityObject_ && !SecurityObject_->CheckAccess(requiredAccess, *UserToken_)) { + UserSID_ = UserToken_->GetUserSID(); + if (requiredAccess != 0 && SecurityObject_ && !SecurityObject_->CheckAccess(requiredAccess, *UserToken_)) { if (Action_ == EAction::ModifyPermissions) { - // do not spam for other actions + // do not spam for other actions RLOG_SQS_WARN("User " << UserSID_ << " tried to modify ACL for " << GetActionACLSourcePath() << ". Access denied"); - } + } MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED, Sprintf("%s on %s was denied for %s due to missing permission %s.", actionName.c_str(), SanitizeNodePath(GetActionACLSourcePath()).c_str(), UserSID_.c_str(), GetActionMatchingACE(actionName).c_str())); SendReplyAndDie(); - return; - } - + return; + } + DoGetQuotaAndProcess(); } } @@ -662,9 +662,9 @@ private: deadline)); } else { DoRoutine(); - } - } - + } + } + void HandleQuota(TEvQuota::TEvClearance::TPtr& ev) { const TDuration quotaWaitDuration = TActivationContext::Now() - QuotaRequestTs_; switch (ev->Get()->Result) { @@ -743,16 +743,16 @@ protected: THolder<IReplyCallback> Cb_; TString RootUrl_; TString UserName_; - TString SecurityToken_; - TString FolderId_; - size_t SecurityCheckRequestsToWaitFor_ = 2; - TIntrusivePtr<TSecurityObject> SecurityObject_; - TIntrusivePtr<NACLib::TUserToken> UserToken_; - TString UserSID_; // identifies the client who sent this request + TString SecurityToken_; + TString FolderId_; + size_t SecurityCheckRequestsToWaitFor_ = 2; + TIntrusivePtr<TSecurityObject> SecurityObject_; + TIntrusivePtr<NACLib::TUserToken> UserToken_; + TString UserSID_; // identifies the client who sent this request bool UserExists_ = false; bool QueueExists_ = false; ui64 Shards_; - TMaybe<bool> IsFifo_; + TMaybe<bool> IsFifo_; TInstant StartTs_; TInstant FinishTs_; TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_; // Raw counters interface. Is is not prefered to use them diff --git a/ydb/core/ymq/actor/actor.cpp b/ydb/core/ymq/actor/actor.cpp index d2ebd7cf50..cf7292a89a 100644 --- a/ydb/core/ymq/actor/actor.cpp +++ b/ydb/core/ymq/actor/actor.cpp @@ -52,21 +52,21 @@ IActor* CreateActionActor(const NKikimrClient::TSqsRequest& req, THolder<IReplyC REQUEST_CASE(DeleteQueue) REQUEST_CASE(DeleteQueueBatch) REQUEST_CASE(DeleteUser) - REQUEST_CASE(ListPermissions) + REQUEST_CASE(ListPermissions) REQUEST_CASE(GetQueueAttributes) REQUEST_CASE(GetQueueAttributesBatch) REQUEST_CASE(GetQueueUrl) REQUEST_CASE(ListQueues) REQUEST_CASE(ListUsers) - REQUEST_CASE(ModifyPermissions) + REQUEST_CASE(ModifyPermissions) REQUEST_CASE(PurgeQueue) REQUEST_CASE(PurgeQueueBatch) REQUEST_CASE(ReceiveMessage) REQUEST_CASE(SendMessage) REQUEST_CASE(SendMessageBatch) REQUEST_CASE(SetQueueAttributes) - REQUEST_CASE(ListDeadLetterSourceQueues) - REQUEST_CASE(CountQueues) + REQUEST_CASE(ListDeadLetterSourceQueues) + REQUEST_CASE(CountQueues) #undef REQUEST_CASE diff --git a/ydb/core/ymq/actor/count_queues.cpp b/ydb/core/ymq/actor/count_queues.cpp index 8f217a192a..2a47a9349a 100644 --- a/ydb/core/ymq/actor/count_queues.cpp +++ b/ydb/core/ymq/actor/count_queues.cpp @@ -1,78 +1,78 @@ -#include "action.h" -#include "error.h" -#include "log.h" -#include "params.h" -#include "serviceid.h" -#include "executor.h" - +#include "action.h" +#include "error.h" +#include "log.h" +#include "params.h" +#include "serviceid.h" +#include "executor.h" + #include <ydb/public/lib/value/value.h> - -#include <util/string/ascii.h> -#include <util/string/cast.h> -#include <util/string/join.h> - -using NKikimr::NClient::TValue; - -namespace NKikimr::NSQS { - -class TCountQueuesActor - : public TActionActor<TCountQueuesActor> -{ -public: - static constexpr bool NeedExistingQueue() { - return false; - } - + +#include <util/string/ascii.h> +#include <util/string/cast.h> +#include <util/string/join.h> + +using NKikimr::NClient::TValue; + +namespace NKikimr::NSQS { + +class TCountQueuesActor + : public TActionActor<TCountQueuesActor> +{ +public: + static constexpr bool NeedExistingQueue() { + return false; + } + TCountQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) : TActionActor(sourceSqsRequest, EAction::CountQueues, std::move(cb)) - { + { CopyAccountName(Request()); - Response_.MutableCountQueues()->SetRequestId(RequestId_); - + Response_.MutableCountQueues()->SetRequestId(RequestId_); + CopySecurityToken(Request()); - } - -private: - TError* MutableErrorDesc() override { - return Response_.MutableCountQueues()->MutableError(); - } - + } + +private: + TError* MutableErrorDesc() override { + return Response_.MutableCountQueues()->MutableError(); + } + void DoAction() override { - Become(&TThis::StateFunc); - + Become(&TThis::StateFunc); + Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvCountQueues(RequestId_, UserName_, FolderId_)); - } - - TString DoGetQueueName() const override { - return TString(); - } - + } + + TString DoGetQueueName() const override { + return TString(); + } + STATEFN(StateFunc) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); hFunc(TSqsEvents::TEvCountQueuesResponse, HandleCountQueuesResponse); - } - } - + } + } + void HandleCountQueuesResponse(TSqsEvents::TEvCountQueuesResponse::TPtr& ev) { - if (ev->Get()->Failed) { + if (ev->Get()->Failed) { RLOG_SQS_WARN("Count queues failed"); - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); - } else { - auto* result = Response_.MutableCountQueues(); - result->SetCount(ev->Get()->Count); - } - + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + } else { + auto* result = Response_.MutableCountQueues(); + result->SetCount(ev->Get()->Count); + } + SendReplyAndDie(); - } - + } + const TCountQueuesRequest& Request() const { return SourceSqsRequest_.GetCountQueues(); } -}; - +}; + IActor* CreateCountQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) { return new TCountQueuesActor(sourceSqsRequest, std::move(cb)); -} - -} // namespace NKikimr::NSQS +} + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/create_queue.cpp b/ydb/core/ymq/actor/create_queue.cpp index 2d4b3af214..69c8e3078f 100644 --- a/ydb/core/ymq/actor/create_queue.cpp +++ b/ydb/core/ymq/actor/create_queue.cpp @@ -1,12 +1,12 @@ #include "action.h" #include "error.h" #include "log.h" -#include "queue_schema.h" +#include "queue_schema.h" #include <ydb/core/ymq/base/constants.h> #include <ydb/core/ymq/base/helpers.h> #include <ydb/core/ymq/base/queue_id.h> - + #include <util/string/join.h> #include <util/string/type.h> @@ -20,22 +20,22 @@ public: return false; } - static constexpr bool CreateMissingAccount() { - return true; - } - + static constexpr bool CreateMissingAccount() { + return true; + } + TCreateQueueActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) : TActionActor(sourceSqsRequest, EAction::CreateQueue, std::move(cb)) { CopyAccountName(Request()); // will be replaced during bootstrap for cloud mode Response_.MutableCreateQueue()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } -protected: - bool IsFifoQueue() const override { +protected: + bool IsFifoQueue() const override { return AsciiHasSuffixIgnoreCase(Request().GetQueueName(), ".fifo"); // works for cloud too, since the custom name should end with '.fifo' - } + } private: bool DoValidate() override { @@ -104,19 +104,19 @@ private: new TCreateQueueSchemaActorV2(TQueuePath(cfg.GetRoot(), accountName, queueName), Request(), SelfId(), RequestId_, customQueueName, FolderId_, IsCloud(), cfg.GetEnableQueueAttributesValidation(), UserCounters_, QuoterResources_) - ); - } - + ); + } + void DoAction() override { Become(&TThis::StateFunc); - if (IsCloud()) { + if (IsCloud()) { Register(new TAtomicCounterActor(SelfId(), Cfg().GetRoot(), RequestId_)); - } else { - static const TString emptyCustomQueueName = ""; - + } else { + static const TString emptyCustomQueueName = ""; + StartQueueCreation(Request().GetQueueName(), UserName_, emptyCustomQueueName); - } + } } TString DoGetQueueName() const override { @@ -132,25 +132,25 @@ private: } void HandleAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev) { - auto event = ev->Get(); + auto event = ev->Get(); auto* result = Response_.MutableCreateQueue(); - - if (event->Success) { + + if (event->Success) { const ui16 serviceId = Cfg().GetYandexCloudServiceId(); - const TString cloudId = UserName_; // should decode from creds - ResourceId_ = MakeQueueId(serviceId, event->NewValue, UserName_); + const TString cloudId = UserName_; // should decode from creds + ResourceId_ = MakeQueueId(serviceId, event->NewValue, UserName_); RLOG_SQS_DEBUG("Created resource id: " << MakeQueueId(serviceId, event->NewValue, UserName_) - << " for service id: " << serviceId - << " unique num: " << event->NewValue - << " account name: " << UserName_); - + << " for service id: " << serviceId + << " unique num: " << event->NewValue + << " account name: " << UserName_); + StartQueueCreation(ResourceId_, cloudId, Request().GetQueueName()); - } else { - MakeError(result, NErrors::INTERNAL_FAILURE); + } else { + MakeError(result, NErrors::INTERNAL_FAILURE); SendReplyAndDie(); - } - } - + } + } + void HandleQueueCreated(TSqsEvents::TEvQueueCreated::TPtr& ev) { SchemaActor_ = TActorId(); auto event = ev->Get(); @@ -168,14 +168,14 @@ private: case EQueueState::Active: if (event->Success) { const TString& name = Request().GetQueueName(); - if (IsCloud()) { - const auto finalResourceId = event->AlreadyExists ? event->ExistingQueueResourceId : ResourceId_; - result->SetQueueName(finalResourceId); - result->SetQueueUrl(MakeQueueUrl(TString::Join(finalResourceId, '/', name))); - } else { - result->SetQueueName(name); - result->SetQueueUrl(MakeQueueUrl(name)); - } + if (IsCloud()) { + const auto finalResourceId = event->AlreadyExists ? event->ExistingQueueResourceId : ResourceId_; + result->SetQueueName(finalResourceId); + result->SetQueueUrl(MakeQueueUrl(TString::Join(finalResourceId, '/', name))); + } else { + result->SetQueueName(name); + result->SetQueueUrl(MakeQueueUrl(name)); + } } else { MakeError(result, *ev->Get()->ErrorClass, errMsg); } @@ -202,7 +202,7 @@ private: } private: - TString ResourceId_; + TString ResourceId_; TActorId SchemaActor_; }; diff --git a/ydb/core/ymq/actor/create_user.cpp b/ydb/core/ymq/actor/create_user.cpp index deebfc54e8..f8386a7609 100644 --- a/ydb/core/ymq/actor/create_user.cpp +++ b/ydb/core/ymq/actor/create_user.cpp @@ -19,7 +19,7 @@ public: { CopyAccountName(Request()); Response_.MutableCreateUser()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } diff --git a/ydb/core/ymq/actor/delete_message.cpp b/ydb/core/ymq/actor/delete_message.cpp index 50b59ddcfb..ec923b4992 100644 --- a/ydb/core/ymq/actor/delete_message.cpp +++ b/ydb/core/ymq/actor/delete_message.cpp @@ -103,18 +103,18 @@ private: void ProcessAnswer(TDeleteMessageResponse* resp, const TSqsEvents::TEvDeleteMessageBatchResponse::TMessageResult& answer) { switch (answer.Status) { - case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: { + case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: { INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second); - break; - } - case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: { - // ignore missing handle just like proper SQS does - break; - } - case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::Failed: { - MakeError(resp, NErrors::INTERNAL_FAILURE); - break; - } + break; + } + case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: { + // ignore missing handle just like proper SQS does + break; + } + case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::Failed: { + MakeError(resp, NErrors::INTERNAL_FAILURE); + break; + } } } diff --git a/ydb/core/ymq/actor/delete_queue.cpp b/ydb/core/ymq/actor/delete_queue.cpp index 9711b91734..af76041aae 100644 --- a/ydb/core/ymq/actor/delete_queue.cpp +++ b/ydb/core/ymq/actor/delete_queue.cpp @@ -1,7 +1,7 @@ #include "action.h" #include "common_batch_actor.h" #include "error.h" -#include "queue_schema.h" +#include "queue_schema.h" #include <util/string/join.h> @@ -18,8 +18,8 @@ public: Response_.MutableDeleteQueue()->SetRequestId(RequestId_); CopySecurityToken(Request()); - } - + } + private: bool DoValidate() override { if (!GetQueueName()) { @@ -38,9 +38,9 @@ private: Become(&TThis::StateFunc); SchemaActor_ = Register( - new TDeleteQueueSchemaActorV2( + new TDeleteQueueSchemaActorV2( TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName()), SelfId(), RequestId_, UserCounters_) - ); + ); } TString DoGetQueueName() const override { @@ -101,11 +101,11 @@ private: const auto& entry = Request().GetEntries(i); auto& req = *ret[i].MutableDeleteQueue(); req.MutableAuth()->SetUserName(UserName_); - + if (Request().HasCredentials()) { *req.MutableCredentials() = Request().GetCredentials(); - } - + } + req.SetQueueName(entry.GetQueueName()); req.SetId(entry.GetId()); } diff --git a/ydb/core/ymq/actor/delete_user.cpp b/ydb/core/ymq/actor/delete_user.cpp index bea58c5a7a..0a3d230265 100644 --- a/ydb/core/ymq/actor/delete_user.cpp +++ b/ydb/core/ymq/actor/delete_user.cpp @@ -2,7 +2,7 @@ #include "error.h" #include "executor.h" #include "log.h" -#include "queue_schema.h" +#include "queue_schema.h" #include <ydb/public/lib/value/value.h> @@ -25,7 +25,7 @@ public: { CopyAccountName(Request()); Response_.MutableDeleteUser()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } @@ -51,8 +51,8 @@ private: .QueryId(LIST_QUEUES_ID) .Counters(QueueCounters_) .RetryOnTimeout() - .Params() - .Utf8("FOLDERID", "") + .Params() + .Utf8("FOLDERID", "") .Utf8("USER_NAME", UserName_) .ParentBuilder().Start(); } @@ -94,7 +94,7 @@ private: Queues_.insert(name); Register( - new TDeleteQueueSchemaActorV2( + new TDeleteQueueSchemaActorV2( TQueuePath(Cfg().GetRoot(), Request().GetUserName(), name), SelfId(), RequestId_, UserCounters_) ); } diff --git a/ydb/core/ymq/actor/error.cpp b/ydb/core/ymq/actor/error.cpp index 23a65c56f6..b10337579e 100644 --- a/ydb/core/ymq/actor/error.cpp +++ b/ydb/core/ymq/actor/error.cpp @@ -62,7 +62,7 @@ size_t ErrorsCount(const NKikimrClient::TSqsResponse& response, TAPIStatusesCoun RESPONSE_CASE(DeleteQueue) RESPONSE_BATCH_CASE(DeleteQueueBatch) RESPONSE_CASE(DeleteUser) - RESPONSE_CASE(ListPermissions) + RESPONSE_CASE(ListPermissions) RESPONSE_CASE(GetQueueAttributes) RESPONSE_BATCH_CASE(GetQueueAttributesBatch) RESPONSE_CASE(GetQueueUrl) @@ -75,8 +75,8 @@ size_t ErrorsCount(const NKikimrClient::TSqsResponse& response, TAPIStatusesCoun RESPONSE_CASE(SendMessage) RESPONSE_BATCH_CASE(SendMessageBatch) RESPONSE_CASE(SetQueueAttributes) - RESPONSE_CASE(ListDeadLetterSourceQueues) - RESPONSE_CASE(CountQueues) + RESPONSE_CASE(ListDeadLetterSourceQueues) + RESPONSE_CASE(CountQueues) case NKikimrClient::TSqsResponse::RESPONSE_NOT_SET: return 0; diff --git a/ydb/core/ymq/actor/events.h b/ydb/core/ymq/actor/events.h index 9fff62c7cf..5cb94b5aba 100644 --- a/ydb/core/ymq/actor/events.h +++ b/ydb/core/ymq/actor/events.h @@ -1,6 +1,6 @@ #pragma once #include "defs.h" - + #include <ydb/core/base/defs.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/tx_proxy/proxy.h> @@ -20,7 +20,7 @@ #include <library/cpp/actors/core/event_local.h> #include <library/cpp/monlib/dynamic_counters/counters.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> @@ -69,7 +69,7 @@ struct TSqsEvents { /// Update queue attributes cache EvClearQueueAttributesCache, /// Incrementing of atomic counter - EvAtomicCounterIncrementResult, + EvAtomicCounterIncrementResult, /// Request for finding leader node for the given queue EvGetLeaderNodeForQueueRequest, @@ -81,10 +81,10 @@ struct TSqsEvents { EvQueueId, // Cloud specific - EvGetQueueFolderIdAndCustomName, - EvQueueFolderIdAndCustomName, - EvCountQueues, - EvCountQueuesResponse, + EvGetQueueFolderIdAndCustomName, + EvQueueFolderIdAndCustomName, + EvCountQueues, + EvCountQueuesResponse, // Send/Receive/Delete requests. Action actor sends these requests to queue leader EvSendMessageBatch, @@ -107,8 +107,8 @@ struct TSqsEvents { EvMigrationDone, - EvReportProcessedRequestAttributes, - + EvReportProcessedRequestAttributes, + EvInsertQueueCounters, EvUserSettingsChanged, @@ -116,14 +116,14 @@ struct TSqsEvents { EvReadQueuesList, EvQueuesList, - EvDeadLetterQueueNotification, - - EvSchemeTraversalResult, - - EvGarbageCleaningResult, - - EvGarbageSearchResult, - + EvDeadLetterQueueNotification, + + EvSchemeTraversalResult, + + EvGarbageCleaningResult, + + EvGarbageSearchResult, + EvCleanupQueryComplete, EvEnd, @@ -350,21 +350,21 @@ struct TSqsEvents { } }; - struct TEvAtomicCounterIncrementResult : public NActors::TEventLocal<TEvAtomicCounterIncrementResult, EvAtomicCounterIncrementResult> { - bool Success; - TString Error; - ui64 NewValue; - - TEvAtomicCounterIncrementResult(bool success, const TString& error = TString(), ui64 newValue = 0) - : Success(success) - , Error(error) - , NewValue(newValue) - { - } - - TEvAtomicCounterIncrementResult(const TEvAtomicCounterIncrementResult& other) = default; - }; - + struct TEvAtomicCounterIncrementResult : public NActors::TEventLocal<TEvAtomicCounterIncrementResult, EvAtomicCounterIncrementResult> { + bool Success; + TString Error; + ui64 NewValue; + + TEvAtomicCounterIncrementResult(bool success, const TString& error = TString(), ui64 newValue = 0) + : Success(success) + , Error(error) + , NewValue(newValue) + { + } + + TEvAtomicCounterIncrementResult(const TEvAtomicCounterIncrementResult& other) = default; + }; + struct TEvPurgeQueue : public NActors::TEventLocal<TEvPurgeQueue, EvPurgeQueue> { /// Queue path in the catalog TQueuePath QueuePath; @@ -516,29 +516,29 @@ struct TSqsEvents { bool Exists = false; bool Failed = false; TString QueueId; // resource id in case of Yandex.Cloud mode and queue name in case of Yandex - ui64 Version = 0; // last queue version registered in service actor - ui64 ShardsCount = 0; // number of queue shards + ui64 Version = 0; // last queue version registered in service actor + ui64 ShardsCount = 0; // number of queue shards - TEvQueueId(const bool failed = false) + TEvQueueId(const bool failed = false) : Failed(failed) { } - explicit TEvQueueId(const TString queueId, const ui64 version, const ui64 shardsCount) + explicit TEvQueueId(const TString queueId, const ui64 version, const ui64 shardsCount) : Exists(true) , QueueId(std::move(queueId)) - , Version(version) - , ShardsCount(shardsCount) + , Version(version) + , ShardsCount(shardsCount) { } }; - struct TEvGetQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvGetQueueFolderIdAndCustomName, EvGetQueueFolderIdAndCustomName> { + struct TEvGetQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvGetQueueFolderIdAndCustomName, EvGetQueueFolderIdAndCustomName> { TString RequestId; TString UserName; TString QueueName; - TEvGetQueueFolderIdAndCustomName(TString requestId, TString userName, TString queueName) + TEvGetQueueFolderIdAndCustomName(TString requestId, TString userName, TString queueName) : RequestId(std::move(requestId)) , UserName(std::move(userName)) , QueueName(std::move(queueName)) @@ -546,51 +546,51 @@ struct TSqsEvents { } }; - struct TEvQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvQueueFolderIdAndCustomName, EvQueueFolderIdAndCustomName> { + struct TEvQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvQueueFolderIdAndCustomName, EvQueueFolderIdAndCustomName> { bool Exists = false; bool Failed = false; TString QueueFolderId; - TString QueueCustomName; + TString QueueCustomName; - TEvQueueFolderIdAndCustomName(bool failed = false) + TEvQueueFolderIdAndCustomName(bool failed = false) : Failed(failed) { } - explicit TEvQueueFolderIdAndCustomName(TString queueFolderId, TString queueCustomName) + explicit TEvQueueFolderIdAndCustomName(TString queueFolderId, TString queueCustomName) : Exists(true) , QueueFolderId(std::move(queueFolderId)) - , QueueCustomName(std::move(queueCustomName)) - { - } - }; - - struct TEvCountQueues : public NActors::TEventLocal<TEvCountQueues, EvCountQueues> { - TString RequestId; - TString UserName; - TString FolderId; - - TEvCountQueues(TString requestId, TString userName, TString folderId) - : RequestId(std::move(requestId)) - , UserName(std::move(userName)) - , FolderId(std::move(folderId)) - { - } - }; - - struct TEvCountQueuesResponse : public NActors::TEventLocal<TEvCountQueuesResponse, EvCountQueuesResponse> { - bool Failed = false; - bool Exists = false; - ui64 Count = 0; - - explicit TEvCountQueuesResponse(bool failed, bool exists = false, ui64 count = 0) - : Failed(failed) - , Exists(exists) - , Count(count) + , QueueCustomName(std::move(queueCustomName)) { } }; + struct TEvCountQueues : public NActors::TEventLocal<TEvCountQueues, EvCountQueues> { + TString RequestId; + TString UserName; + TString FolderId; + + TEvCountQueues(TString requestId, TString userName, TString folderId) + : RequestId(std::move(requestId)) + , UserName(std::move(userName)) + , FolderId(std::move(folderId)) + { + } + }; + + struct TEvCountQueuesResponse : public NActors::TEventLocal<TEvCountQueuesResponse, EvCountQueuesResponse> { + bool Failed = false; + bool Exists = false; + ui64 Count = 0; + + explicit TEvCountQueuesResponse(bool failed, bool exists = false, ui64 count = 0) + : Failed(failed) + , Exists(exists) + , Count(count) + { + } + }; + struct TEvSendMessageBatch : public NActors::TEventLocal<TEvSendMessageBatch, EvSendMessageBatch> { struct TMessageEntry { TString MessageId; @@ -750,10 +750,10 @@ struct TSqsEvents { { } }; - - struct TEvReportProcessedRequestAttributes : public NActors::TEventLocal<TEvReportProcessedRequestAttributes, EvReportProcessedRequestAttributes> { - TProcessedRequestAttributes Data; - }; + + struct TEvReportProcessedRequestAttributes : public NActors::TEventLocal<TEvReportProcessedRequestAttributes, EvReportProcessedRequestAttributes> { + TProcessedRequestAttributes Data; + }; struct TEvInsertQueueCounters : public NActors::TEventLocal<TEvInsertQueueCounters, EvInsertQueueCounters> { TEvInsertQueueCounters(const TString& user, const TString& queue, ui64 leaderTabletId) @@ -792,7 +792,7 @@ struct TSqsEvents { ui64 LeaderTabletId = 0; TString CustomName; TString FolderId; - TString DlqName; + TString DlqName; ui64 Version = 0; ui64 ShardsCount = 0; TInstant CreatedTimestamp; @@ -810,72 +810,72 @@ struct TSqsEvents { { } }; - + // Used by service to notify dead letter queue leader - struct TEvDeadLetterQueueNotification : public NActors::TEventLocal<TEvDeadLetterQueueNotification, EvDeadLetterQueueNotification> { - }; - - using TSchemePath = TVector<TString>; - using TSchemeCacheNavigate = NSchemeCache::TSchemeCacheNavigate; - - struct TSchemeNode { - TSchemePath Path; - TSchemeCacheNavigate::EKind Kind = TSchemeCacheNavigate::EKind::KindUnknown; - TInstant CreationTs = TInstant::Zero(); - - THashMap<TString, TSchemeNode> Children; - }; - - struct TEvSchemeTraversalResult : public NActors::TEventLocal<TEvSchemeTraversalResult, EvSchemeTraversalResult> { - explicit TEvSchemeTraversalResult(bool success = true) - : Success(success) - { - } - - bool Success = true; - THolder<TSchemeNode> RootHolder; - }; - - struct TEvGarbageCleaningResult : public NActors::TEventLocal<TEvGarbageCleaningResult, EvGarbageCleaningResult> { - explicit TEvGarbageCleaningResult(bool success = true) { - Record.Success = success; - } - - struct TCleaningResult { - bool Success = true; - - TString Account; - TString HintPath; - TVector<TString> RemovedNodes; - - TInstant StartedAt; - TDuration Duration; - }; - - TCleaningResult Record; - }; - - struct TEvGarbageSearchResult : public NActors::TEventLocal<TEvGarbageSearchResult, EvGarbageSearchResult> { - explicit TEvGarbageSearchResult(bool success = true) - : Success(success) - { - } - - struct TGarbageHint { - enum class EReason { - Unknown, - UnusedVersion, - UnregisteredQueue - }; - - TString Account; - TSchemeNode SchemeNode; - EReason Reason = EReason::Unknown; - }; - - bool Success = true; - THashMap<TString, TGarbageHint> GarbageHints; - }; + struct TEvDeadLetterQueueNotification : public NActors::TEventLocal<TEvDeadLetterQueueNotification, EvDeadLetterQueueNotification> { + }; + + using TSchemePath = TVector<TString>; + using TSchemeCacheNavigate = NSchemeCache::TSchemeCacheNavigate; + + struct TSchemeNode { + TSchemePath Path; + TSchemeCacheNavigate::EKind Kind = TSchemeCacheNavigate::EKind::KindUnknown; + TInstant CreationTs = TInstant::Zero(); + + THashMap<TString, TSchemeNode> Children; + }; + + struct TEvSchemeTraversalResult : public NActors::TEventLocal<TEvSchemeTraversalResult, EvSchemeTraversalResult> { + explicit TEvSchemeTraversalResult(bool success = true) + : Success(success) + { + } + + bool Success = true; + THolder<TSchemeNode> RootHolder; + }; + + struct TEvGarbageCleaningResult : public NActors::TEventLocal<TEvGarbageCleaningResult, EvGarbageCleaningResult> { + explicit TEvGarbageCleaningResult(bool success = true) { + Record.Success = success; + } + + struct TCleaningResult { + bool Success = true; + + TString Account; + TString HintPath; + TVector<TString> RemovedNodes; + + TInstant StartedAt; + TDuration Duration; + }; + + TCleaningResult Record; + }; + + struct TEvGarbageSearchResult : public NActors::TEventLocal<TEvGarbageSearchResult, EvGarbageSearchResult> { + explicit TEvGarbageSearchResult(bool success = true) + : Success(success) + { + } + + struct TGarbageHint { + enum class EReason { + Unknown, + UnusedVersion, + UnregisteredQueue + }; + + TString Account; + TSchemeNode SchemeNode; + EReason Reason = EReason::Unknown; + }; + + bool Success = true; + THashMap<TString, TGarbageHint> GarbageHints; + }; struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> { explicit TEvCleanupQueryComplete(const TString& name, ui64 type) : Name(name) diff --git a/ydb/core/ymq/actor/executor.cpp b/ydb/core/ymq/actor/executor.cpp index e17dc68fe9..3c60e38b7d 100644 --- a/ydb/core/ymq/actor/executor.cpp +++ b/ydb/core/ymq/actor/executor.cpp @@ -4,7 +4,7 @@ #include <ydb/core/protos/tx_proxy.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> - + #include <ydb/core/engine/mkql_proto.h> #include <ydb/public/lib/value/value.h> #include <ydb/core/ymq/base/debug_info.h> @@ -109,10 +109,10 @@ void TExecutorBuilder::SendToQueueLeader() { TActivationContext::Send(new IEventHandle(QueueLeaderActor_, Parent_, ev.Release())); } -const char* TExecutorBuilder::GetQueryById(size_t idx) { - const char* query = IsFifoQueue_ ? GetFifoQueryById(idx) : GetStdQueryById(idx); - Y_VERIFY(query); - return query; +const char* TExecutorBuilder::GetQueryById(size_t idx) { + const char* query = IsFifoQueue_ ? GetFifoQueryById(idx) : GetStdQueryById(idx); + Y_VERIFY(query); + return query; } TMiniKqlExecutionActor::TMiniKqlExecutionActor( @@ -274,19 +274,19 @@ void TMiniKqlExecutionActor::HandleCompile(TMiniKQLCompileServiceEvents::TEvComp ProceedWithExecution(); } -template<typename TKikimrResultRecord> -bool TMiniKqlExecutionActor::ShouldRetryOnFail(const TKikimrResultRecord& record) const { - const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus()); - return NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(status) || - RetryOnTimeout_ && record.GetStatusCode() == NKikimrIssues::TStatusIds::TIMEOUT || +template<typename TKikimrResultRecord> +bool TMiniKqlExecutionActor::ShouldRetryOnFail(const TKikimrResultRecord& record) const { + const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus()); + return NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(status) || + RetryOnTimeout_ && record.GetStatusCode() == NKikimrIssues::TStatusIds::TIMEOUT || (record.HasSchemeShardStatus() && record.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusMultipleModifications); // very rare case in queue creation -} - +} + void TMiniKqlExecutionActor::HandleResponse(TResponse::TPtr& ev) { const TDuration executionDuration = TActivationContext::Now() - StartExecutionTs_; auto& response = *ev->Get(); - auto& record = response.Record; - const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus()); + auto& record = response.Record; + const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus()); RLOG_SQS_TRACE(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " HandleResponse " << record); RLOG_SQS_DEBUG(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " Attempt " << AttemptNumber_ << " execution duration: " << executionDuration.MilliSeconds() << "ms"); if (Counters_) { @@ -314,7 +314,7 @@ void TMiniKqlExecutionActor::HandleResponse(TResponse::TPtr& ev) { && status != TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress && status != TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAlready) { failed = true; - retryableError = ShouldRetryOnFail(record); + retryableError = ShouldRetryOnFail(record); if (retryableError) { RLOG_SQS_WARN(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " Retryable error in mkql execution result: " << response.Record); } else { diff --git a/ydb/core/ymq/actor/executor.h b/ydb/core/ymq/actor/executor.h index 5dd51c6da2..30d142b4ed 100644 --- a/ydb/core/ymq/actor/executor.h +++ b/ydb/core/ymq/actor/executor.h @@ -36,10 +36,10 @@ public: Shard_ = shard; return *this; } - TExecutorBuilder& QueueVersion(ui64 version) { - QueueVersion_ = version; - return *this; - } + TExecutorBuilder& QueueVersion(ui64 version) { + QueueVersion_ = version; + return *this; + } TExecutorBuilder& QueueLeader(const TActorId& queueLeaderActor) { QueueLeaderActor_ = queueLeaderActor; return *this; @@ -60,10 +60,10 @@ public: QueryId_ = id; return *this; } - TExecutorBuilder& Fifo(bool isFifo) { - IsFifoQueue_ = isFifo; - return *this; - } + TExecutorBuilder& Fifo(bool isFifo) { + IsFifoQueue_ = isFifo; + return *this; + } TExecutorBuilder& Mode(NKikimrTxUserProxy::TMiniKQLTransaction::EMode mode) { Request().Record.MutableTransaction()->MutableMiniKQLTransaction()->SetMode(mode); return *this; @@ -126,7 +126,7 @@ private: return QueryId_ != EQueryId::QUERY_VECTOR_SIZE; } - const char* GetQueryById(size_t idx); + const char* GetQueryById(size_t idx); private: const TActorId Parent_; @@ -134,11 +134,11 @@ private: THolder<TEvTxUserProxy::TEvProposeTransaction> ProposeTransactionRequest_; TMaybe<TParameters> Parameters_; bool RetryOnTimeout_ = false; - bool IsFifoQueue_ = false; + bool IsFifoQueue_ = false; TString UserName_; TString QueueName_; ui64 Shard_ = 0; - ui64 QueueVersion_ = 0; + ui64 QueueVersion_ = 0; TActorId QueueLeaderActor_; TSqsEvents::TExecutedCallback Callback_; EQueryId QueryId_ = EQueryId::QUERY_VECTOR_SIZE; @@ -197,10 +197,10 @@ private: void PassAway(); void HandleCompile(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr& ev); - - template<typename TKikimrResultRecord> + + template<typename TKikimrResultRecord> bool ShouldRetryOnFail(const TKikimrResultRecord& record) const; - + void HandleResponse(TResponse::TPtr& ev); void HandleWakeup(TEvWakeup::TPtr& ev); void HandleResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev); diff --git a/ydb/core/ymq/actor/garbage_collector.cpp b/ydb/core/ymq/actor/garbage_collector.cpp index e82284c020..c67a72dc36 100644 --- a/ydb/core/ymq/actor/garbage_collector.cpp +++ b/ydb/core/ymq/actor/garbage_collector.cpp @@ -1,796 +1,796 @@ -#include "garbage_collector.h" - -#include "cfg.h" -#include "log.h" -#include "events.h" -#include "executor.h" -#include "schema.h" - +#include "garbage_collector.h" + +#include "cfg.h" +#include "log.h" +#include "events.h" +#include "executor.h" +#include "schema.h" + #include <ydb/core/base/path.h> #include <ydb/core/mon/mon.h> - + #include <ydb/core/protos/services.pb.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - + #include <library/cpp/monlib/service/pages/templates.h> - -#include <util/string/join.h> - -#include <algorithm> -#include <queue> -#include <vector> - -namespace NKikimr::NSQS { - -using TQueueRecord = TSqsEvents::TEvQueuesList::TQueueRecord; - -using TSchemePath = TSqsEvents::TSchemePath; -using TSchemeNode = TSqsEvents::TSchemeNode; -using TGarbageHint = TSqsEvents::TEvGarbageSearchResult::TGarbageHint; -using TSchemeCacheNavigate = TSqsEvents::TSchemeCacheNavigate; -using TCleaningResult = TSqsEvents::TEvGarbageCleaningResult::TCleaningResult; - -static const TString GARBAGE_CLEANER_LABEL = "GarbageCleaner"; -static const TString MINIMUM_ITEM_AGE_SECONDS = "minimum_item_age_seconds"; -static const TString PATH_TO_CLEAN = "path_to_clean"; - -static void PrintAllPaths(const TSchemeNode& node, TStringStream& ss) { - if (node.Path.size()) { - ss << CanonizePath(node.Path) << ", kind: " << static_cast<size_t>(node.Kind) << ", ts: " << node.CreationTs << "\n"; - } - - for (const auto& [name, child] : node.Children) { - PrintAllPaths(child, ss); - } -} - -static TSchemeNode& FindOrAllocateSchemeNode(TSchemeNode* root, const TSchemePath& path) { - TSchemeNode* currentNode = root; - - for (const auto& part : path) { - currentNode = ¤tNode->Children[part]; - } - - return *currentNode; -} - -class TSchemeTraversalActor : public TActorBootstrapped<TSchemeTraversalActor> { -public: + +#include <util/string/join.h> + +#include <algorithm> +#include <queue> +#include <vector> + +namespace NKikimr::NSQS { + +using TQueueRecord = TSqsEvents::TEvQueuesList::TQueueRecord; + +using TSchemePath = TSqsEvents::TSchemePath; +using TSchemeNode = TSqsEvents::TSchemeNode; +using TGarbageHint = TSqsEvents::TEvGarbageSearchResult::TGarbageHint; +using TSchemeCacheNavigate = TSqsEvents::TSchemeCacheNavigate; +using TCleaningResult = TSqsEvents::TEvGarbageCleaningResult::TCleaningResult; + +static const TString GARBAGE_CLEANER_LABEL = "GarbageCleaner"; +static const TString MINIMUM_ITEM_AGE_SECONDS = "minimum_item_age_seconds"; +static const TString PATH_TO_CLEAN = "path_to_clean"; + +static void PrintAllPaths(const TSchemeNode& node, TStringStream& ss) { + if (node.Path.size()) { + ss << CanonizePath(node.Path) << ", kind: " << static_cast<size_t>(node.Kind) << ", ts: " << node.CreationTs << "\n"; + } + + for (const auto& [name, child] : node.Children) { + PrintAllPaths(child, ss); + } +} + +static TSchemeNode& FindOrAllocateSchemeNode(TSchemeNode* root, const TSchemePath& path) { + TSchemeNode* currentNode = root; + + for (const auto& part : path) { + currentNode = ¤tNode->Children[part]; + } + + return *currentNode; +} + +class TSchemeTraversalActor : public TActorBootstrapped<TSchemeTraversalActor> { +public: TSchemeTraversalActor(const TActorId parentId, const TActorId schemeCacheId, - const TVector<TSchemePath>& pathsToTraverse, - const ui64 maxDepth = Max()) - : ParentId(parentId) - , SchemeCacheId(schemeCacheId) - , PathsToTraverse(pathsToTraverse) - , MaxDepth(maxDepth) - , CurrentDepth(0) - { - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; - } - - void Bootstrap() { - Become(&TThis::Working); - - RootHolder = MakeHolder<TSchemeNode>(); - - SendListChildrenRequest(PathsToTraverse); - } - -private: - void SendListChildrenRequest(const TVector<TSchemePath>& paths) { - const size_t elementsCount = paths.size(); - - Y_VERIFY(elementsCount); - - auto schemeCacheRequest = MakeHolder<TSchemeCacheNavigate>(); - - schemeCacheRequest->ResultSet.resize(elementsCount); - - for (size_t i = 0; i < elementsCount; ++i) { - schemeCacheRequest->ResultSet[i].Path = paths[i]; - - schemeCacheRequest->ResultSet[i].Operation = - ((CurrentDepth == MaxDepth) ? TSchemeCacheNavigate::OpPath : TSchemeCacheNavigate::OpList); - } - - Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse); - } - } - - void OnFinishedTraversal(bool success) { - auto ev = MakeHolder<TSqsEvents::TEvSchemeTraversalResult>(success); - - if (success) { - ev->RootHolder = std::move(RootHolder); - } - - Send(ParentId, ev.Release()); - - PassAway(); - } - - void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); - - if (navigate->ErrorCount) { - OnFinishedTraversal(false); - return; - } - - TVector<TSchemePath> newPathsToTraverse; - - ++CurrentDepth; - - for (const auto& entry : navigate->ResultSet) { - auto& node = FindOrAllocateSchemeNode(&*RootHolder, entry.Path); - - node.Path = entry.Path; - node.Kind = entry.Kind; - node.CreationTs = TInstant::MilliSeconds(entry.CreateStep); - - if (entry.ListNodeEntry && CurrentDepth <= MaxDepth) { - for (const auto& child : entry.ListNodeEntry->Children) { - auto yetAnotherPathToTraverse = entry.Path; + const TVector<TSchemePath>& pathsToTraverse, + const ui64 maxDepth = Max()) + : ParentId(parentId) + , SchemeCacheId(schemeCacheId) + , PathsToTraverse(pathsToTraverse) + , MaxDepth(maxDepth) + , CurrentDepth(0) + { + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; + } + + void Bootstrap() { + Become(&TThis::Working); + + RootHolder = MakeHolder<TSchemeNode>(); + + SendListChildrenRequest(PathsToTraverse); + } + +private: + void SendListChildrenRequest(const TVector<TSchemePath>& paths) { + const size_t elementsCount = paths.size(); + + Y_VERIFY(elementsCount); + + auto schemeCacheRequest = MakeHolder<TSchemeCacheNavigate>(); + + schemeCacheRequest->ResultSet.resize(elementsCount); + + for (size_t i = 0; i < elementsCount; ++i) { + schemeCacheRequest->ResultSet[i].Path = paths[i]; + + schemeCacheRequest->ResultSet[i].Operation = + ((CurrentDepth == MaxDepth) ? TSchemeCacheNavigate::OpPath : TSchemeCacheNavigate::OpList); + } + + Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse); + } + } + + void OnFinishedTraversal(bool success) { + auto ev = MakeHolder<TSqsEvents::TEvSchemeTraversalResult>(success); + + if (success) { + ev->RootHolder = std::move(RootHolder); + } + + Send(ParentId, ev.Release()); + + PassAway(); + } + + void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); + + if (navigate->ErrorCount) { + OnFinishedTraversal(false); + return; + } + + TVector<TSchemePath> newPathsToTraverse; + + ++CurrentDepth; + + for (const auto& entry : navigate->ResultSet) { + auto& node = FindOrAllocateSchemeNode(&*RootHolder, entry.Path); + + node.Path = entry.Path; + node.Kind = entry.Kind; + node.CreationTs = TInstant::MilliSeconds(entry.CreateStep); + + if (entry.ListNodeEntry && CurrentDepth <= MaxDepth) { + for (const auto& child : entry.ListNodeEntry->Children) { + auto yetAnotherPathToTraverse = entry.Path; yetAnotherPathToTraverse.push_back(child.Name); - - newPathsToTraverse.push_back(std::move(yetAnotherPathToTraverse)); - } - } - } - - if (newPathsToTraverse.empty()) { - OnFinishedTraversal(true); - } else { - SendListChildrenRequest(newPathsToTraverse); - } - } - -private: + + newPathsToTraverse.push_back(std::move(yetAnotherPathToTraverse)); + } + } + } + + if (newPathsToTraverse.empty()) { + OnFinishedTraversal(true); + } else { + SendListChildrenRequest(newPathsToTraverse); + } + } + +private: const TActorId ParentId; const TActorId SchemeCacheId; - const TVector<TSchemePath> PathsToTraverse; - - const ui64 MaxDepth; - ui64 CurrentDepth; - - THolder<TSchemeNode> RootHolder; -}; - -class TGarbageSearcher : public TActorBootstrapped<TGarbageSearcher> { -public: + const TVector<TSchemePath> PathsToTraverse; + + const ui64 MaxDepth; + ui64 CurrentDepth; + + THolder<TSchemeNode> RootHolder; +}; + +class TGarbageSearcher : public TActorBootstrapped<TGarbageSearcher> { +public: TGarbageSearcher(const TActorId parentId, const TActorId schemeCacheId, const TActorId queuesListReaderId, - const ui64 minimumItemAgeSeconds) - : ParentId(parentId) - , SchemeCacheId(schemeCacheId) - , QueuesListReaderId(queuesListReaderId) - , MinimumItemAge(TDuration::Seconds(minimumItemAgeSeconds)) - { - Y_VERIFY(parentId); - Y_VERIFY(schemeCacheId); - Y_VERIFY(queuesListReaderId); - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; - } - - void Bootstrap() { - Become(&TThis::ListingAccounts); - - RootPath = SplitPath(Cfg().GetRoot()); - - // list accounts and system tables only - Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {RootPath}, 1)); - - Send(QueuesListReaderId, new TSqsEvents::TEvReadQueuesList()); - - RequiredResponses = 2; - } - -private: - void ProcessQueuesListingResult(TSqsEvents::TEvQueuesList::TPtr& ev) { - if (ev->Get()->Success) { - SortedQueues = std::move(ev->Get()->SortedQueues); - - CompleteListing(); - } else { - ReplyToParentAndDie(false); - } - } - - void ProcessAccountsListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { - if (ev->Get()->Success) { - auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, RootPath); - - Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); - - for (const auto& [name, child] : node.Children) { - if (child.Kind == TSchemeCacheNavigate::EKind::KindPath && !name.StartsWith(".")) { - AccountsToProcess.push(name); - } - } - - CompleteListing(); - } else { - ReplyToParentAndDie(false); - } - } - - void CompleteListing() { - Y_VERIFY(RequiredResponses); - - if (!--RequiredResponses) { - Become(&TThis::ProcessingAccounts); - - ListNextAccountOrReply(); - - return; - } - } - - void ListNextAccountOrReply() { - if (AccountsToProcess.size()) { - CurrentAccount = AccountsToProcess.front(); - AccountsToProcess.pop(); - - CurrentAccountPath = RootPath; - CurrentAccountPath.push_back(CurrentAccount); - - // depth == 2 since the path pattern is account/queue/queue_version - Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {CurrentAccountPath}, 2)); - } else { - ReplyToParentAndDie(true); - } - } - - STATEFN(ListingAccounts) { - switch (ev->GetTypeRewrite()) { - hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessAccountsListingResult); - hFunc(TSqsEvents::TEvQueuesList, ProcessQueuesListingResult); - } - } - - // per-account processing section - - STATEFN(ProcessingAccounts) { - switch (ev->GetTypeRewrite()) { - hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessSingleAccount); - } - } - - void ProcessSingleAccount(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { - if (ev->Get()->Success) { - auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, CurrentAccountPath); - - // an account is always a directory - Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); - - for (auto& [queueName, queueNode] : node.Children) { - // ignore tables (e. g. local "Queues" table) - if (queueNode.Kind == TSchemeCacheNavigate::EKind::KindPath) { - ProcessSingleQueue(queueName, queueNode); - } - } - } else { - // Skip account on an error. Possible reason: some elements were removed during a listing - } - - ListNextAccountOrReply(); - } - - void ProcessSingleQueue(const TString& queueName, TSchemeNode& node) { - TQueueRecord fakeRecord; - fakeRecord.UserName = CurrentAccount; - fakeRecord.QueueName = queueName; - - auto it = std::lower_bound(SortedQueues.begin(), SortedQueues.end(), fakeRecord); - if (it != SortedQueues.end() && it->UserName == CurrentAccount && it->QueueName == queueName) { - const bool isLegacyQueue = it->Version == 0; - - if (isLegacyQueue) { - // TODO: check State field and report - } else { - const TString versionStr = "v" + ToString(it->Version); - for (auto& [name, child] : node.Children) { - // Do not ever touch fresh queues or actual versions - const TDuration itemAge = TActivationContext::AsActorContext().Now() - node.CreationTs; - if (name != versionStr && itemAge >= MinimumItemAge) { - AddGarbageHint(CurrentAccount, std::move(child), TGarbageHint::EReason::UnusedVersion); - } - } - } - } else { - AddGarbageHint(CurrentAccount, std::move(node), TGarbageHint::EReason::UnregisteredQueue); - } - } - - void AddGarbageHint(const TString& account, TSchemeNode&& node, const TGarbageHint::EReason reason) { - // remove excessive data - node.Children.clear(); - - TGarbageHint hint{account, std::move(node), reason}; - - const auto path = CanonizePath(hint.SchemeNode.Path); - - GarbageHints[path] = std::move(hint); - } - - void ReplyToParentAndDie(bool success) { - auto ev = MakeHolder<TSqsEvents::TEvGarbageSearchResult>(success); - - if (success) { - ev->GarbageHints = std::move(GarbageHints); - } - - Send(ParentId, ev.Release()); - - PassAway(); - } - -private: + const ui64 minimumItemAgeSeconds) + : ParentId(parentId) + , SchemeCacheId(schemeCacheId) + , QueuesListReaderId(queuesListReaderId) + , MinimumItemAge(TDuration::Seconds(minimumItemAgeSeconds)) + { + Y_VERIFY(parentId); + Y_VERIFY(schemeCacheId); + Y_VERIFY(queuesListReaderId); + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; + } + + void Bootstrap() { + Become(&TThis::ListingAccounts); + + RootPath = SplitPath(Cfg().GetRoot()); + + // list accounts and system tables only + Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {RootPath}, 1)); + + Send(QueuesListReaderId, new TSqsEvents::TEvReadQueuesList()); + + RequiredResponses = 2; + } + +private: + void ProcessQueuesListingResult(TSqsEvents::TEvQueuesList::TPtr& ev) { + if (ev->Get()->Success) { + SortedQueues = std::move(ev->Get()->SortedQueues); + + CompleteListing(); + } else { + ReplyToParentAndDie(false); + } + } + + void ProcessAccountsListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { + if (ev->Get()->Success) { + auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, RootPath); + + Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); + + for (const auto& [name, child] : node.Children) { + if (child.Kind == TSchemeCacheNavigate::EKind::KindPath && !name.StartsWith(".")) { + AccountsToProcess.push(name); + } + } + + CompleteListing(); + } else { + ReplyToParentAndDie(false); + } + } + + void CompleteListing() { + Y_VERIFY(RequiredResponses); + + if (!--RequiredResponses) { + Become(&TThis::ProcessingAccounts); + + ListNextAccountOrReply(); + + return; + } + } + + void ListNextAccountOrReply() { + if (AccountsToProcess.size()) { + CurrentAccount = AccountsToProcess.front(); + AccountsToProcess.pop(); + + CurrentAccountPath = RootPath; + CurrentAccountPath.push_back(CurrentAccount); + + // depth == 2 since the path pattern is account/queue/queue_version + Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {CurrentAccountPath}, 2)); + } else { + ReplyToParentAndDie(true); + } + } + + STATEFN(ListingAccounts) { + switch (ev->GetTypeRewrite()) { + hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessAccountsListingResult); + hFunc(TSqsEvents::TEvQueuesList, ProcessQueuesListingResult); + } + } + + // per-account processing section + + STATEFN(ProcessingAccounts) { + switch (ev->GetTypeRewrite()) { + hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessSingleAccount); + } + } + + void ProcessSingleAccount(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { + if (ev->Get()->Success) { + auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, CurrentAccountPath); + + // an account is always a directory + Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); + + for (auto& [queueName, queueNode] : node.Children) { + // ignore tables (e. g. local "Queues" table) + if (queueNode.Kind == TSchemeCacheNavigate::EKind::KindPath) { + ProcessSingleQueue(queueName, queueNode); + } + } + } else { + // Skip account on an error. Possible reason: some elements were removed during a listing + } + + ListNextAccountOrReply(); + } + + void ProcessSingleQueue(const TString& queueName, TSchemeNode& node) { + TQueueRecord fakeRecord; + fakeRecord.UserName = CurrentAccount; + fakeRecord.QueueName = queueName; + + auto it = std::lower_bound(SortedQueues.begin(), SortedQueues.end(), fakeRecord); + if (it != SortedQueues.end() && it->UserName == CurrentAccount && it->QueueName == queueName) { + const bool isLegacyQueue = it->Version == 0; + + if (isLegacyQueue) { + // TODO: check State field and report + } else { + const TString versionStr = "v" + ToString(it->Version); + for (auto& [name, child] : node.Children) { + // Do not ever touch fresh queues or actual versions + const TDuration itemAge = TActivationContext::AsActorContext().Now() - node.CreationTs; + if (name != versionStr && itemAge >= MinimumItemAge) { + AddGarbageHint(CurrentAccount, std::move(child), TGarbageHint::EReason::UnusedVersion); + } + } + } + } else { + AddGarbageHint(CurrentAccount, std::move(node), TGarbageHint::EReason::UnregisteredQueue); + } + } + + void AddGarbageHint(const TString& account, TSchemeNode&& node, const TGarbageHint::EReason reason) { + // remove excessive data + node.Children.clear(); + + TGarbageHint hint{account, std::move(node), reason}; + + const auto path = CanonizePath(hint.SchemeNode.Path); + + GarbageHints[path] = std::move(hint); + } + + void ReplyToParentAndDie(bool success) { + auto ev = MakeHolder<TSqsEvents::TEvGarbageSearchResult>(success); + + if (success) { + ev->GarbageHints = std::move(GarbageHints); + } + + Send(ParentId, ev.Release()); + + PassAway(); + } + +private: const TActorId ParentId; const TActorId SchemeCacheId; const TActorId QueuesListReaderId; - const TDuration MinimumItemAge; - - TSchemePath RootPath; - - std::queue<TString> AccountsToProcess; - std::vector<TQueueRecord> SortedQueues; - - ui64 RequiredResponses = 0; - - TString CurrentAccount; - TSchemePath CurrentAccountPath; - - THashMap<TString, TGarbageHint> GarbageHints; -}; - -class TGarbageCleaner : public TActorBootstrapped<TGarbageCleaner> { -public: + const TDuration MinimumItemAge; + + TSchemePath RootPath; + + std::queue<TString> AccountsToProcess; + std::vector<TQueueRecord> SortedQueues; + + ui64 RequiredResponses = 0; + + TString CurrentAccount; + TSchemePath CurrentAccountPath; + + THashMap<TString, TGarbageHint> GarbageHints; +}; + +class TGarbageCleaner : public TActorBootstrapped<TGarbageCleaner> { +public: TGarbageCleaner(const TActorId parentId, const TActorId schemeCacheId, - TGarbageHint&& garbageHint) - : ParentId(parentId) - , SchemeCacheId(schemeCacheId) - , GarbageHint(std::move(garbageHint)) - { - Y_VERIFY(parentId); - Y_VERIFY(schemeCacheId); - Y_VERIFY(GarbageHint.Reason != TGarbageHint::EReason::Unknown); - Y_VERIFY(GarbageHint.SchemeNode.Path.size()); - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; - } - - void Bootstrap() { - Become(&TThis::Working); - - StartedAt = TActivationContext::AsActorContext().Now(); - - Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {GarbageHint.SchemeNode.Path})); - } - -private: - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessGarbageListingResult); - hFunc(TSqsEvents::TEvExecuted, OnNodeRemoved); - } - } - - void DFS(const TSchemeNode& node) { - for (const auto& [name, child] : node.Children) { - DFS(child); - } - - NodesToRemove.push(node); - } - - void ProcessGarbageListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { - if (ev->Get()->Success) { - auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, GarbageHint.SchemeNode.Path); - - Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); - - DFS(node); - - RemoveNextNodeOrDie(); - } else { - LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to list nodes at " << CanonizePath(GarbageHint.SchemeNode.Path)); - - ReportToParentAndDie(false); - } - } - - void RemoveNextNodeOrDie() { - if (!NodesToRemove.empty()) { - CurrentNode = std::move(NodesToRemove.front()); - NodesToRemove.pop(); - - RemoveCurrentNode(); - } else { - LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " finished cleaning at " << CanonizePath(GarbageHint.SchemeNode.Path)); - - ReportToParentAndDie(true); - } - } - - void ReportToParentAndDie(const bool success) { - auto ev = MakeHolder<TSqsEvents::TEvGarbageCleaningResult>(success); - - ev->Record.Account = GarbageHint.Account; - ev->Record.HintPath = CanonizePath(GarbageHint.SchemeNode.Path); - ev->Record.RemovedNodes = std::move(RemovedNodes); - ev->Record.StartedAt = StartedAt; - ev->Record.Duration = TActivationContext::AsActorContext().Now() - StartedAt; - - Send(ParentId, ev.Release()); - - PassAway(); - } - - TQueuePath MakeAndVerifyQueuePath(const TSchemePath& path) const { - // A certain dir structure is assumed here: SqsRoot/Account/Queue/Version - // All path components except "Version" are required - // The code guarantees scheme modifications locality - - Y_VERIFY(CanonizePath(path).StartsWith(CanonizePath(GarbageHint.SchemeNode.Path))); - - auto rootPath = SplitPath(Cfg().GetRoot()); - - auto rootPathIt = rootPath.begin(); - auto pathIt = path.begin(); - - while (rootPathIt != rootPath.end()) { - Y_VERIFY(*rootPathIt++ == *pathIt++); - } - - TQueuePath result; - result.Root = Cfg().GetRoot(); - - Y_VERIFY(pathIt != path.end()); - - result.UserName = *pathIt++; - - Y_VERIFY(result.UserName == GarbageHint.Account); - - Y_VERIFY(pathIt != path.end()); - - result.QueueName = *pathIt++; - - if (pathIt != path.end()) { - result.VersionSuffix = *pathIt; - } - - return result; - } - - void RemoveCurrentNode() const { - auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); - - auto path = CurrentNode.Path; - - Y_VERIFY(path.size() > 1); - - const auto name = path.back(); - - path.pop_back(); - - trans->SetWorkingDir(CanonizePath(path)); - - trans->MutableDrop()->SetName(name); - - if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindPath) { + TGarbageHint&& garbageHint) + : ParentId(parentId) + , SchemeCacheId(schemeCacheId) + , GarbageHint(std::move(garbageHint)) + { + Y_VERIFY(parentId); + Y_VERIFY(schemeCacheId); + Y_VERIFY(GarbageHint.Reason != TGarbageHint::EReason::Unknown); + Y_VERIFY(GarbageHint.SchemeNode.Path.size()); + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; + } + + void Bootstrap() { + Become(&TThis::Working); + + StartedAt = TActivationContext::AsActorContext().Now(); + + Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {GarbageHint.SchemeNode.Path})); + } + +private: + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessGarbageListingResult); + hFunc(TSqsEvents::TEvExecuted, OnNodeRemoved); + } + } + + void DFS(const TSchemeNode& node) { + for (const auto& [name, child] : node.Children) { + DFS(child); + } + + NodesToRemove.push(node); + } + + void ProcessGarbageListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) { + if (ev->Get()->Success) { + auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, GarbageHint.SchemeNode.Path); + + Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath); + + DFS(node); + + RemoveNextNodeOrDie(); + } else { + LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to list nodes at " << CanonizePath(GarbageHint.SchemeNode.Path)); + + ReportToParentAndDie(false); + } + } + + void RemoveNextNodeOrDie() { + if (!NodesToRemove.empty()) { + CurrentNode = std::move(NodesToRemove.front()); + NodesToRemove.pop(); + + RemoveCurrentNode(); + } else { + LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " finished cleaning at " << CanonizePath(GarbageHint.SchemeNode.Path)); + + ReportToParentAndDie(true); + } + } + + void ReportToParentAndDie(const bool success) { + auto ev = MakeHolder<TSqsEvents::TEvGarbageCleaningResult>(success); + + ev->Record.Account = GarbageHint.Account; + ev->Record.HintPath = CanonizePath(GarbageHint.SchemeNode.Path); + ev->Record.RemovedNodes = std::move(RemovedNodes); + ev->Record.StartedAt = StartedAt; + ev->Record.Duration = TActivationContext::AsActorContext().Now() - StartedAt; + + Send(ParentId, ev.Release()); + + PassAway(); + } + + TQueuePath MakeAndVerifyQueuePath(const TSchemePath& path) const { + // A certain dir structure is assumed here: SqsRoot/Account/Queue/Version + // All path components except "Version" are required + // The code guarantees scheme modifications locality + + Y_VERIFY(CanonizePath(path).StartsWith(CanonizePath(GarbageHint.SchemeNode.Path))); + + auto rootPath = SplitPath(Cfg().GetRoot()); + + auto rootPathIt = rootPath.begin(); + auto pathIt = path.begin(); + + while (rootPathIt != rootPath.end()) { + Y_VERIFY(*rootPathIt++ == *pathIt++); + } + + TQueuePath result; + result.Root = Cfg().GetRoot(); + + Y_VERIFY(pathIt != path.end()); + + result.UserName = *pathIt++; + + Y_VERIFY(result.UserName == GarbageHint.Account); + + Y_VERIFY(pathIt != path.end()); + + result.QueueName = *pathIt++; + + if (pathIt != path.end()) { + result.VersionSuffix = *pathIt; + } + + return result; + } + + void RemoveCurrentNode() const { + auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); + + auto path = CurrentNode.Path; + + Y_VERIFY(path.size() > 1); + + const auto name = path.back(); + + path.pop_back(); + + trans->SetWorkingDir(CanonizePath(path)); + + trans->MutableDrop()->SetName(name); + + if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindPath) { trans->SetOperationType(NKikimrSchemeOp::ESchemeOpRmDir); - } else if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindTable) { + } else if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindTable) { trans->SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); - } else { - Y_VERIFY("Unexpected node kind"); - } - - LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " attempts to remove the node " << CanonizePath(CurrentNode.Path)); - - const TQueuePath queuePath = MakeAndVerifyQueuePath(CurrentNode.Path); - - Register(new TMiniKqlExecutionActor(SelfId(), GARBAGE_CLEANER_LABEL + "Tx", std::move(ev), false, queuePath, nullptr)); - } - - void OnNodeRemoved(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); - - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { - const auto path = CanonizePath(CurrentNode.Path); - - RemovedNodes.push_back(path); - - LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " removed the node " << path); - - RemoveNextNodeOrDie(); - } else { - LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to remove the node " << CanonizePath(CurrentNode.Path)); - - ReportToParentAndDie(false); - } - } - -private: + } else { + Y_VERIFY("Unexpected node kind"); + } + + LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " attempts to remove the node " << CanonizePath(CurrentNode.Path)); + + const TQueuePath queuePath = MakeAndVerifyQueuePath(CurrentNode.Path); + + Register(new TMiniKqlExecutionActor(SelfId(), GARBAGE_CLEANER_LABEL + "Tx", std::move(ev), false, queuePath, nullptr)); + } + + void OnNodeRemoved(TSqsEvents::TEvExecuted::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); + + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { + const auto path = CanonizePath(CurrentNode.Path); + + RemovedNodes.push_back(path); + + LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " removed the node " << path); + + RemoveNextNodeOrDie(); + } else { + LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to remove the node " << CanonizePath(CurrentNode.Path)); + + ReportToParentAndDie(false); + } + } + +private: const TActorId ParentId; const TActorId SchemeCacheId; - const TGarbageHint GarbageHint; - std::queue<TSchemeNode> NodesToRemove; - TSchemeNode CurrentNode; - TVector<TString> RemovedNodes; - TInstant StartedAt; -}; - -class TGarbageCollector : public TActorBootstrapped<TGarbageCollector> { -public: + const TGarbageHint GarbageHint; + std::queue<TSchemeNode> NodesToRemove; + TSchemeNode CurrentNode; + TVector<TString> RemovedNodes; + TInstant StartedAt; +}; + +class TGarbageCollector : public TActorBootstrapped<TGarbageCollector> { +public: TGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId) - : SchemeCacheId(schemeCacheId) - , QueuesListReaderId(queuesListReaderId) - { - Y_VERIFY(schemeCacheId); - Y_VERIFY(queuesListReaderId); - Y_UNUSED(PrintAllPaths); - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; - } - - void Bootstrap() { - Become(&TThis::Working); - - RegisterMonitoringPage(); - } - - void RegisterMonitoringPage() const { - NActors::TMon* mon = AppData()->Mon; - if (mon) { - NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); - mon->RegisterActorPage(page, "sqsgc", "SQS Garbage Collector", false, - TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); - } - } - - void ProcessGarbageHints(TSqsEvents::TEvGarbageSearchResult::TPtr& ev) { - Scanning = false; - - if (ev->Get()->Success) { - CurrentGarbageHints = std::move(ev->Get()->GarbageHints); - } - } - - void InitRescanAndReport(NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - - HTML(str) { - if (Scanning) { - str << "Already scanning. Patience is a virtue, dear user"; - } else { - const TCgiParameters& params = ev->Get()->Request.GetParams(); - if (params.Has(MINIMUM_ITEM_AGE_SECONDS)) { - const TString& minimumItemAgeSecondsParam = params.Get(MINIMUM_ITEM_AGE_SECONDS); - - ui64 minimumItemAgeSeconds = 0; - if (TryFromString<ui64>(minimumItemAgeSecondsParam, minimumItemAgeSeconds)) { - Scanning = true; - - Register(new TGarbageSearcher(SelfId(), - SchemeCacheId, - QueuesListReaderId, - minimumItemAgeSeconds)); - - str << "Started new scan process"; - } else { - str << "Can't parse minimum item age param value: " << minimumItemAgeSecondsParam; - } - } else { - str << "Minimum item age param is missing"; - } - } - str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); - } - - void StartCleaner(TGarbageHint&& garbageHint) { - Register(new TGarbageCleaner(SelfId(), SchemeCacheId, std::move(garbageHint))); - } - - void InitCleaningAndReport(NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - - HTML(str) { - const TCgiParameters& params = ev->Get()->Request.GetParams(); - - if (params.Has(PATH_TO_CLEAN)) { - const TString& pathToClean = params.Get(PATH_TO_CLEAN); - auto it = CurrentGarbageHints.find(pathToClean); - if (it == CurrentGarbageHints.end()) { - str << "Specified path doesn't match any garbage hint: " << params.Get(PATH_TO_CLEAN); - } else { - StartCleaner(std::move(it->second)); - - CurrentGarbageHints.erase(it); - - str << "Started cleaning process for " << params.Get(PATH_TO_CLEAN); - } - } else { - str << "Path to clean is missing"; - } - str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); - } - - void InitClearHistoryAndReport(NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - - CleaningHistory.clear(); - - HTML(str) { - str << "Cleaning history was reset"; - str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); - } - - static void AddCleanGarbageLink(TStringStream& str, const TString& pathToClean) { - TCgiParameters params; - params.InsertUnescaped(PATH_TO_CLEAN, pathToClean); - - str << "<a href=\"/actors/sqsgc/clean?" << params() << "\">" << "Clean" << "</a>"; - } - - static void AddRescanForm(TStringStream& str) { - str << "<form method=\"GET\" action=\"/actors/sqsgc/rescan\" id=\"tblMonSQSGCfrm\" name=\"tblMonSQSGCfrm\">" << Endl; - str << "<legend> Rescan structure and filter by item age: </legend>"; - str << "<label> " << "Minimum item age (seconds): " << " <input type=\"number\" id=\"" << "lblRescan"; - str << "\" name=\"" << MINIMUM_ITEM_AGE_SECONDS << "\"" << " value=\"" << Cfg().GetMinimumGarbageAgeSeconds() << "\""; - str << " min=\"0\" /> </label>"; - str << "<input class=\"btn btn-default\" type=\"submit\" value=\"Rescan\"/>" << Endl; - str << "</form>" << Endl; - } - - void RenderGarbageHints(TStringStream& str) const { - if (CurrentGarbageHints.empty()) { - return; - } - - IOutputStream& __stream(str); - - H4() { - str << "Garbage hints"; - } - TABLE_SORTABLE_CLASS("hints-table") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "Path"; } - TABLEH() { str << "Created at (UTC)"; } - TABLEH() { str << "Reason"; } - TABLEH() { str << "Action"; } - } - } - TABLEBODY() { - for (const auto& [pathToClean, hint] : CurrentGarbageHints) { - TABLER() { - TABLED() { str << pathToClean; } - TABLED() { str << hint.SchemeNode.CreationTs.ToRfc822String(); } - TABLED() { str << hint.Reason; } - TABLED() { AddCleanGarbageLink(str, pathToClean); } - } - } - } - } - } - - void RenderCleaningResults(TStringStream& str) const { - if (CleaningHistory.empty()) { - return; - } - - IOutputStream& __stream(str); - - H4() { - str << "Cleaning history"; - } - TABLE_SORTABLE_CLASS("cr-table") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "Account"; } - TABLEH() { str << "Hint path"; } - TABLEH() { str << "Started at (UTC)"; } - TABLEH() { str << "Duration (ms)"; } - TABLEH() { str << "Nodes removed"; } - TABLEH() { str << "Result"; } - } - } - TABLEBODY() { - for (const auto& result : CleaningHistory) { - TABLER() { - TABLED() { str << result.Account; } - TABLED() { str << result.HintPath; } - TABLED() { str << result.StartedAt.ToRfc822String(); } - TABLED() { str << result.Duration.MilliSeconds(); } - TABLED() { str << result.RemovedNodes.size(); } - TABLED() { str << (result.Success ? "OK" : "FAIL"); } - } - } - } - } - str << "<a href=\"/actors/sqsgc/clear_history\">" << "Reset local cleaning history" << "</a>"; - } - - void RenderMainControlPage(NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - - HTML(str) { - AddRescanForm(str); - RenderGarbageHints(str); - RenderCleaningResults(str); - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); - } - - void HandleCleaningResult(TSqsEvents::TEvGarbageCleaningResult::TPtr& ev) { - CleaningHistory.push_back(std::move(ev->Get()->Record)); - } - - void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) { - const TStringBuf path = ev->Get()->Request.GetPath(); - - if (path.EndsWith("/rescan")) { - InitRescanAndReport(ev); - } else if (path.EndsWith("/clean")) { - InitCleaningAndReport(ev); - } else if (path.EndsWith("/clear_history")) { - InitClearHistoryAndReport(ev); - } else { - RenderMainControlPage(ev); - } - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(TSqsEvents::TEvGarbageSearchResult, ProcessGarbageHints); - hFunc(TSqsEvents::TEvGarbageCleaningResult, HandleCleaningResult); - hFunc(NMon::TEvHttpInfo, HandleHttpRequest); - } - } - -private: + : SchemeCacheId(schemeCacheId) + , QueuesListReaderId(queuesListReaderId) + { + Y_VERIFY(schemeCacheId); + Y_VERIFY(queuesListReaderId); + Y_UNUSED(PrintAllPaths); + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR; + } + + void Bootstrap() { + Become(&TThis::Working); + + RegisterMonitoringPage(); + } + + void RegisterMonitoringPage() const { + NActors::TMon* mon = AppData()->Mon; + if (mon) { + NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); + mon->RegisterActorPage(page, "sqsgc", "SQS Garbage Collector", false, + TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); + } + } + + void ProcessGarbageHints(TSqsEvents::TEvGarbageSearchResult::TPtr& ev) { + Scanning = false; + + if (ev->Get()->Success) { + CurrentGarbageHints = std::move(ev->Get()->GarbageHints); + } + } + + void InitRescanAndReport(NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + + HTML(str) { + if (Scanning) { + str << "Already scanning. Patience is a virtue, dear user"; + } else { + const TCgiParameters& params = ev->Get()->Request.GetParams(); + if (params.Has(MINIMUM_ITEM_AGE_SECONDS)) { + const TString& minimumItemAgeSecondsParam = params.Get(MINIMUM_ITEM_AGE_SECONDS); + + ui64 minimumItemAgeSeconds = 0; + if (TryFromString<ui64>(minimumItemAgeSecondsParam, minimumItemAgeSeconds)) { + Scanning = true; + + Register(new TGarbageSearcher(SelfId(), + SchemeCacheId, + QueuesListReaderId, + minimumItemAgeSeconds)); + + str << "Started new scan process"; + } else { + str << "Can't parse minimum item age param value: " << minimumItemAgeSecondsParam; + } + } else { + str << "Minimum item age param is missing"; + } + } + str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + } + + void StartCleaner(TGarbageHint&& garbageHint) { + Register(new TGarbageCleaner(SelfId(), SchemeCacheId, std::move(garbageHint))); + } + + void InitCleaningAndReport(NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + + HTML(str) { + const TCgiParameters& params = ev->Get()->Request.GetParams(); + + if (params.Has(PATH_TO_CLEAN)) { + const TString& pathToClean = params.Get(PATH_TO_CLEAN); + auto it = CurrentGarbageHints.find(pathToClean); + if (it == CurrentGarbageHints.end()) { + str << "Specified path doesn't match any garbage hint: " << params.Get(PATH_TO_CLEAN); + } else { + StartCleaner(std::move(it->second)); + + CurrentGarbageHints.erase(it); + + str << "Started cleaning process for " << params.Get(PATH_TO_CLEAN); + } + } else { + str << "Path to clean is missing"; + } + str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + } + + void InitClearHistoryAndReport(NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + + CleaningHistory.clear(); + + HTML(str) { + str << "Cleaning history was reset"; + str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>"; + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + } + + static void AddCleanGarbageLink(TStringStream& str, const TString& pathToClean) { + TCgiParameters params; + params.InsertUnescaped(PATH_TO_CLEAN, pathToClean); + + str << "<a href=\"/actors/sqsgc/clean?" << params() << "\">" << "Clean" << "</a>"; + } + + static void AddRescanForm(TStringStream& str) { + str << "<form method=\"GET\" action=\"/actors/sqsgc/rescan\" id=\"tblMonSQSGCfrm\" name=\"tblMonSQSGCfrm\">" << Endl; + str << "<legend> Rescan structure and filter by item age: </legend>"; + str << "<label> " << "Minimum item age (seconds): " << " <input type=\"number\" id=\"" << "lblRescan"; + str << "\" name=\"" << MINIMUM_ITEM_AGE_SECONDS << "\"" << " value=\"" << Cfg().GetMinimumGarbageAgeSeconds() << "\""; + str << " min=\"0\" /> </label>"; + str << "<input class=\"btn btn-default\" type=\"submit\" value=\"Rescan\"/>" << Endl; + str << "</form>" << Endl; + } + + void RenderGarbageHints(TStringStream& str) const { + if (CurrentGarbageHints.empty()) { + return; + } + + IOutputStream& __stream(str); + + H4() { + str << "Garbage hints"; + } + TABLE_SORTABLE_CLASS("hints-table") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Path"; } + TABLEH() { str << "Created at (UTC)"; } + TABLEH() { str << "Reason"; } + TABLEH() { str << "Action"; } + } + } + TABLEBODY() { + for (const auto& [pathToClean, hint] : CurrentGarbageHints) { + TABLER() { + TABLED() { str << pathToClean; } + TABLED() { str << hint.SchemeNode.CreationTs.ToRfc822String(); } + TABLED() { str << hint.Reason; } + TABLED() { AddCleanGarbageLink(str, pathToClean); } + } + } + } + } + } + + void RenderCleaningResults(TStringStream& str) const { + if (CleaningHistory.empty()) { + return; + } + + IOutputStream& __stream(str); + + H4() { + str << "Cleaning history"; + } + TABLE_SORTABLE_CLASS("cr-table") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Account"; } + TABLEH() { str << "Hint path"; } + TABLEH() { str << "Started at (UTC)"; } + TABLEH() { str << "Duration (ms)"; } + TABLEH() { str << "Nodes removed"; } + TABLEH() { str << "Result"; } + } + } + TABLEBODY() { + for (const auto& result : CleaningHistory) { + TABLER() { + TABLED() { str << result.Account; } + TABLED() { str << result.HintPath; } + TABLED() { str << result.StartedAt.ToRfc822String(); } + TABLED() { str << result.Duration.MilliSeconds(); } + TABLED() { str << result.RemovedNodes.size(); } + TABLED() { str << (result.Success ? "OK" : "FAIL"); } + } + } + } + } + str << "<a href=\"/actors/sqsgc/clear_history\">" << "Reset local cleaning history" << "</a>"; + } + + void RenderMainControlPage(NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + + HTML(str) { + AddRescanForm(str); + RenderGarbageHints(str); + RenderCleaningResults(str); + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + } + + void HandleCleaningResult(TSqsEvents::TEvGarbageCleaningResult::TPtr& ev) { + CleaningHistory.push_back(std::move(ev->Get()->Record)); + } + + void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) { + const TStringBuf path = ev->Get()->Request.GetPath(); + + if (path.EndsWith("/rescan")) { + InitRescanAndReport(ev); + } else if (path.EndsWith("/clean")) { + InitCleaningAndReport(ev); + } else if (path.EndsWith("/clear_history")) { + InitClearHistoryAndReport(ev); + } else { + RenderMainControlPage(ev); + } + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(TSqsEvents::TEvGarbageSearchResult, ProcessGarbageHints); + hFunc(TSqsEvents::TEvGarbageCleaningResult, HandleCleaningResult); + hFunc(NMon::TEvHttpInfo, HandleHttpRequest); + } + } + +private: const TActorId SchemeCacheId; const TActorId QueuesListReaderId; - bool Scanning = false; - THashMap<TString, TGarbageHint> CurrentGarbageHints; - TVector<TCleaningResult> CleaningHistory; -}; - + bool Scanning = false; + THashMap<TString, TGarbageHint> CurrentGarbageHints; + TVector<TCleaningResult> CleaningHistory; +}; + IActor* CreateGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId) { - return new TGarbageCollector(schemeCacheId, queuesListReaderId); -} - -}; // namespace NKikimr::NSQS + return new TGarbageCollector(schemeCacheId, queuesListReaderId); +} + +}; // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/garbage_collector.h b/ydb/core/ymq/actor/garbage_collector.h index 713b31cc1d..d70bbb7491 100644 --- a/ydb/core/ymq/actor/garbage_collector.h +++ b/ydb/core/ymq/actor/garbage_collector.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include "defs.h" - -#include "actor.h" - + +#include "actor.h" + #include <ydb/core/ymq/base/action.h> - -namespace NKikimr::NSQS { + +namespace NKikimr::NSQS { IActor* CreateGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId); -} +} diff --git a/ydb/core/ymq/actor/get_queue_attributes.cpp b/ydb/core/ymq/actor/get_queue_attributes.cpp index 28dc67b14f..4cfe7f6402 100644 --- a/ydb/core/ymq/actor/get_queue_attributes.cpp +++ b/ydb/core/ymq/actor/get_queue_attributes.cpp @@ -20,24 +20,24 @@ namespace NKikimr::NSQS { struct TAttributeInfo { bool NeedRuntimeAttributes = false; bool NeedAttributesTable = false; - bool NeedArn = false; + bool NeedArn = false; bool FifoOnly = false; }; static const std::map<TString, TAttributeInfo> AttributesInfo = { - { "ApproximateNumberOfMessages", { true, false, false, false } }, - { "ApproximateNumberOfMessagesDelayed", { true, false, false, false } }, - { "ApproximateNumberOfMessagesNotVisible", { true, false, false, false } }, - { "CreatedTimestamp", { true, false, false, false } }, - { "DelaySeconds", { false, true, false, false } }, - { "MaximumMessageSize", { false, true, false, false } }, - { "MessageRetentionPeriod", { false, true, false, false } }, - { "ReceiveMessageWaitTimeSeconds", { false, true, false, false } }, - { "RedrivePolicy", { false, true, false, false } }, - { "VisibilityTimeout", { false, true, false, false } }, - { "FifoQueue", { false, true, false, true } }, - { "ContentBasedDeduplication", { false, true, false, true } }, - { "QueueArn", { false, false, true, false } }, + { "ApproximateNumberOfMessages", { true, false, false, false } }, + { "ApproximateNumberOfMessagesDelayed", { true, false, false, false } }, + { "ApproximateNumberOfMessagesNotVisible", { true, false, false, false } }, + { "CreatedTimestamp", { true, false, false, false } }, + { "DelaySeconds", { false, true, false, false } }, + { "MaximumMessageSize", { false, true, false, false } }, + { "MessageRetentionPeriod", { false, true, false, false } }, + { "ReceiveMessageWaitTimeSeconds", { false, true, false, false } }, + { "RedrivePolicy", { false, true, false, false } }, + { "VisibilityTimeout", { false, true, false, false } }, + { "FifoQueue", { false, true, false, true } }, + { "ContentBasedDeduplication", { false, true, false, true } }, + { "QueueArn", { false, false, true, false } }, }; class TGetQueueAttributesActor @@ -49,16 +49,16 @@ public: { CopyAccountName(Request()); Response_.MutableGetQueueAttributes()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } private: bool ExpandNames() { if (!Request().NamesSize()) { - return false; - } - + return false; + } + bool all = false; for (const auto& name : Request().names()) { if (name == "All") { @@ -75,10 +75,10 @@ private: if (info->second.NeedRuntimeAttributes) { NeedRuntimeAttributes_ = true; } - if (info->second.NeedArn) { - NeedArn_ = true; - } - + if (info->second.NeedArn) { + NeedArn_ = true; + } + AttributesSet_.insert(name); } } @@ -92,7 +92,7 @@ private: } NeedRuntimeAttributes_ = true; NeedAttributesTable_ = true; - NeedArn_ = true; + NeedArn_ = true; } return true; } @@ -102,10 +102,10 @@ private: return IsIn(AttributesSet_, name); } - TString MakeQueueArn(const TString& prefix, const TString& region, const TString& account, const TString& queueName) const { - return Join(":", prefix, region, account, queueName); - } - + TString MakeQueueArn(const TString& prefix, const TString& region, const TString& account, const TString& queueName) const { + return Join(":", prefix, region, account, queueName); + } + bool DoValidate() override { if (!GetQueueName()) { MakeError(Response_.MutableGetQueueAttributes(), NErrors::MISSING_PARAMETER, "No QueueName parameter."); @@ -120,11 +120,11 @@ private: } void ReplyIfReady() { - if (WaitCount_ == 0) { + if (WaitCount_ == 0) { SendReplyAndDie(); - } - } - + } + } + void DoAction() override { Become(&TThis::StateFunc); @@ -132,7 +132,7 @@ private: SendReplyAndDie(); return; } - + if (NeedAttributesTable_) { TExecutorBuilder builder(SelfId(), RequestId_); builder @@ -150,17 +150,17 @@ private: Send(QueueLeader_, MakeHolder<TSqsEvents::TEvGetRuntimeQueueAttributes>(RequestId_)); ++WaitCount_; } - - if (NeedArn_) { - if (IsCloud()) { + + if (NeedArn_) { + if (IsCloud()) { Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueFolderIdAndCustomName(RequestId_, UserName_, GetQueueName())); - ++WaitCount_; - } else { - auto* result = Response_.MutableGetQueueAttributes(); + ++WaitCount_; + } else { + auto* result = Response_.MutableGetQueueAttributes(); result->SetQueueArn(MakeQueueArn(yaSqsArnPrefix, Cfg().GetYandexCloudServiceRegion(), UserName_, GetQueueName())); - } - } - + } + } + ReplyIfReady(); } @@ -207,16 +207,16 @@ private: if (HasAttributeName("VisibilityTimeout")) { result->SetVisibilityTimeout(TDuration::MilliSeconds(ui64(attrs["VisibilityTimeout"])).Seconds()); } - if (HasAttributeName("RedrivePolicy")) { - const TValue& dlqArn(attrs["DlqArn"]); - if (dlqArn.HaveValue() && !TString(dlqArn).empty()) { - // the attributes can't be set separately, so we check only one - TRedrivePolicy redrivePolicy; - redrivePolicy.TargetArn = TString(dlqArn); - redrivePolicy.MaxReceiveCount = ui64(attrs["MaxReceiveCount"]); - result->SetRedrivePolicy(redrivePolicy.ToJson()); - } - } + if (HasAttributeName("RedrivePolicy")) { + const TValue& dlqArn(attrs["DlqArn"]); + if (dlqArn.HaveValue() && !TString(dlqArn).empty()) { + // the attributes can't be set separately, so we check only one + TRedrivePolicy redrivePolicy; + redrivePolicy.TargetArn = TString(dlqArn); + redrivePolicy.MaxReceiveCount = ui64(attrs["MaxReceiveCount"]); + result->SetRedrivePolicy(redrivePolicy.ToJson()); + } + } } else { RLOG_SQS_ERROR("Get queue attributes query failed"); MakeError(result, NErrors::INTERNAL_FAILURE); @@ -253,23 +253,23 @@ private: --WaitCount_; ReplyIfReady(); - } - + } + void HandleQueueFolderIdAndCustomName(TSqsEvents::TEvQueueFolderIdAndCustomName::TPtr& ev) { - auto* result = Response_.MutableGetQueueAttributes(); - - if (ev->Get()->Failed || !ev->Get()->Exists) { + auto* result = Response_.MutableGetQueueAttributes(); + + if (ev->Get()->Failed || !ev->Get()->Exists) { RLOG_SQS_DEBUG("Get queue folder id and custom name failed. Failed: " << ev->Get()->Failed << ". Exists: " << ev->Get()->Exists); - MakeError(result, NErrors::INTERNAL_FAILURE); + MakeError(result, NErrors::INTERNAL_FAILURE); SendReplyAndDie(); - return; + return; } - - if (NeedArn_) { + + if (NeedArn_) { result->SetQueueArn(MakeQueueArn(cloudArnPrefix, Cfg().GetYandexCloudServiceRegion(), ev->Get()->QueueFolderId, ev->Get()->QueueCustomName)); - } - - --WaitCount_; + } + + --WaitCount_; ReplyIfReady(); } @@ -281,7 +281,7 @@ private: THashSet<TString> AttributesSet_; bool NeedRuntimeAttributes_ = false; bool NeedAttributesTable_ = false; - bool NeedArn_ = false; + bool NeedArn_ = false; size_t WaitCount_ = 0; }; @@ -306,11 +306,11 @@ private: const auto& entry = Request().GetEntries(i); auto& req = *ret[i].MutableGetQueueAttributes(); req.MutableAuth()->SetUserName(UserName_); - + if (Request().HasCredentials()) { *req.MutableCredentials() = Request().GetCredentials(); - } - + } + req.SetQueueName(entry.GetQueueName()); req.SetId(entry.GetId()); *req.MutableNames() = Request().GetNames(); diff --git a/ydb/core/ymq/actor/get_queue_url.cpp b/ydb/core/ymq/actor/get_queue_url.cpp index 95554d7bb0..5094214104 100644 --- a/ydb/core/ymq/actor/get_queue_url.cpp +++ b/ydb/core/ymq/actor/get_queue_url.cpp @@ -23,14 +23,14 @@ public: { CopyAccountName(Request()); Response_.MutableGetQueueUrl()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } - static constexpr bool NeedExistingQueue() { - return false; - } - + static constexpr bool NeedExistingQueue() { + return false; + } + private: bool DoValidate() override { if (!GetQueueName()) { @@ -68,11 +68,11 @@ private: } else { if (ev->Get()->Exists) { auto* result = Response_.MutableGetQueueUrl(); - if (IsCloud()) { + if (IsCloud()) { result->SetQueueUrl(MakeQueueUrl(TString::Join(ev->Get()->QueueId, "/", GetQueueName()))); - } else { + } else { result->SetQueueUrl(MakeQueueUrl(ev->Get()->QueueId)); - } + } } else { MakeError(MutableErrorDesc(), NErrors::NON_EXISTENT_QUEUE); } diff --git a/ydb/core/ymq/actor/infly.cpp b/ydb/core/ymq/actor/infly.cpp index a08cb65387..2e6bd1e45d 100644 --- a/ydb/core/ymq/actor/infly.cpp +++ b/ydb/core/ymq/actor/infly.cpp @@ -109,11 +109,11 @@ TInflyMessages::TReceiveCandidates::TReceiveCandidates(TIntrusivePtr<TInflyMessa { } -void TInflyMessages::TReceiveCandidates::SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount) { +void TInflyMessages::TReceiveCandidates::SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount) { Y_ASSERT(Parent && ReceivedMessages); if (auto* msg = ReceivedMessages->Find(offset)) { msg->Message().SetVisibilityDeadline(visibilityDeadline); - msg->Message().SetReceiveCount(receiveCount); + msg->Message().SetReceiveCount(receiveCount); } } diff --git a/ydb/core/ymq/actor/infly.h b/ydb/core/ymq/actor/infly.h index 1dea6baa6c..571eb3fcd5 100644 --- a/ydb/core/ymq/actor/infly.h +++ b/ydb/core/ymq/actor/infly.h @@ -41,11 +41,11 @@ class TInflyMessage : public TInflyMessageWithVisibilityDeadlineKey, public TInf public: TInflyMessage() = default; - TInflyMessage(const ui64 offset, const ui64 randomId, const TInstant visibilityDeadline, const ui32 receiveCount) + TInflyMessage(const ui64 offset, const ui64 randomId, const TInstant visibilityDeadline, const ui32 receiveCount) : Offset(offset) , RandomId(randomId) , VisibilityDeadline(visibilityDeadline) - , ReceiveCount(receiveCount) + , ReceiveCount(receiveCount) { } @@ -61,23 +61,23 @@ public: return VisibilityDeadline; } - ui32 GetReceiveCount() const { - return ReceiveCount; - } - - void SetVisibilityDeadline(const TInstant visibilityDeadline) { + ui32 GetReceiveCount() const { + return ReceiveCount; + } + + void SetVisibilityDeadline(const TInstant visibilityDeadline) { VisibilityDeadline = visibilityDeadline; } - void SetReceiveCount(const ui32 receiveCount) { - ReceiveCount = receiveCount; - } - + void SetReceiveCount(const ui32 receiveCount) { + ReceiveCount = receiveCount; + } + private: ui64 Offset = 0; ui64 RandomId = 0; TInstant VisibilityDeadline; - ui32 ReceiveCount = 0; + ui32 ReceiveCount = 0; }; class TInflyMessages : public TAtomicRefCount<TInflyMessages> { @@ -109,8 +109,8 @@ public: return !operator!(); } - void SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount); - + void SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount); + THolder<TInflyMessage> Delete(ui64 offset); bool Has(ui64 offset) const; diff --git a/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp b/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp index fd4866fd49..2ccd8613ef 100644 --- a/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp +++ b/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp @@ -1,114 +1,114 @@ -#include "action.h" -#include "error.h" -#include "log.h" -#include "params.h" -#include "serviceid.h" - +#include "action.h" +#include "error.h" +#include "log.h" +#include "params.h" +#include "serviceid.h" + #include <ydb/public/lib/value/value.h> #include <ydb/core/ymq/actor/executor.h> - -#include <util/string/join.h> -#include <util/string/printf.h> - -using NKikimr::NClient::TValue; - -namespace NKikimr::NSQS { - -class TListDeadLetterSourceQueuesActor - : public TActionActor<TListDeadLetterSourceQueuesActor> -{ -public: + +#include <util/string/join.h> +#include <util/string/printf.h> + +using NKikimr::NClient::TValue; + +namespace NKikimr::NSQS { + +class TListDeadLetterSourceQueuesActor + : public TActionActor<TListDeadLetterSourceQueuesActor> +{ +public: TListDeadLetterSourceQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) : TActionActor(sourceSqsRequest, EAction::ListDeadLetterSourceQueues, std::move(cb)) - { + { CopyAccountName(Request()); - Response_.MutableListDeadLetterSourceQueues()->SetRequestId(RequestId_); - + Response_.MutableListDeadLetterSourceQueues()->SetRequestId(RequestId_); + CopySecurityToken(Request()); - } - - static constexpr bool NeedExistingQueue() { - return true; - } - - TString DoGetQueueName() const override { + } + + static constexpr bool NeedExistingQueue() { + return true; + } + + TString DoGetQueueName() const override { return Request().GetQueueName(); - } - -private: + } + +private: STATEFN(StateFunc) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); hFunc(TSqsEvents::TEvExecuted, HandleExecuted); - } - } - - bool DoValidate() override { - if (!GetQueueName()) { - MakeError(MutableErrorDesc(), NErrors::MISSING_PARAMETER, "No QueueName parameter."); - return false; - } - - return true; - } - - TError* MutableErrorDesc() override { - return Response_.MutableListDeadLetterSourceQueues()->MutableError(); - } - + } + } + + bool DoValidate() override { + if (!GetQueueName()) { + MakeError(MutableErrorDesc(), NErrors::MISSING_PARAMETER, "No QueueName parameter."); + return false; + } + + return true; + } + + TError* MutableErrorDesc() override { + return Response_.MutableListDeadLetterSourceQueues()->MutableError(); + } + void DoAction() override { - Become(&TThis::StateFunc); - + Become(&TThis::StateFunc); + TExecutorBuilder builder(SelfId(), RequestId_); - builder - .User(UserName_) - .Queue(GetQueueName()) + builder + .User(UserName_) + .Queue(GetQueueName()) .QueueLeader(QueueLeader_) - .QueryId(LIST_DEAD_LETTER_SOURCE_QUEUES_ID) - .Counters(QueueCounters_) - .RetryOnTimeout() - .Params() + .QueryId(LIST_DEAD_LETTER_SOURCE_QUEUES_ID) + .Counters(QueueCounters_) + .RetryOnTimeout() + .Params() .Utf8("USER_NAME", UserName_) - .Utf8("FOLDERID", FolderId_); - - builder.Start(); - } - + .Utf8("FOLDERID", FolderId_); + + builder.Start(); + } + void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - auto* result = Response_.MutableListDeadLetterSourceQueues(); - - if (ev->Get()->IsOk()) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - const TValue queues(val["queues"]); - - for (size_t i = 0; i < queues.Size(); ++i) { - const TString name((TString(queues[i]["QueueName"]))); - const TString customQueueName((TString(queues[i]["CustomQueueName"]))); - - auto* item = result->AddQueues(); - item->SetQueueName(name); - if (IsCloud()) { - item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName))); - } else { - item->SetQueueUrl(MakeQueueUrl(name)); - } - } - } else { + const auto& record = ev->Get()->Record; + auto* result = Response_.MutableListDeadLetterSourceQueues(); + + if (ev->Get()->IsOk()) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + const TValue queues(val["queues"]); + + for (size_t i = 0; i < queues.Size(); ++i) { + const TString name((TString(queues[i]["QueueName"]))); + const TString customQueueName((TString(queues[i]["CustomQueueName"]))); + + auto* item = result->AddQueues(); + item->SetQueueName(name); + if (IsCloud()) { + item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName))); + } else { + item->SetQueueUrl(MakeQueueUrl(name)); + } + } + } else { RLOG_SQS_WARN("Request failed: " << record); MakeError(result, NErrors::INTERNAL_FAILURE); - } - + } + SendReplyAndDie(); - } - + } + const TListDeadLetterSourceQueuesRequest& Request() const { return SourceSqsRequest_.GetListDeadLetterSourceQueues(); } -}; - +}; + IActor* CreateListDeadLetterSourceQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) { return new TListDeadLetterSourceQueuesActor(sourceSqsRequest, std::move(cb)); -} - -} // namespace NKikimr::NSQS +} + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/list_permissions.cpp b/ydb/core/ymq/actor/list_permissions.cpp index 93a15e1dbf..8fcccb084a 100644 --- a/ydb/core/ymq/actor/list_permissions.cpp +++ b/ydb/core/ymq/actor/list_permissions.cpp @@ -1,167 +1,167 @@ -#include "action.h" - +#include "action.h" + #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - + #include <google/protobuf/text_format.h> - -#include <util/string/ascii.h> -#include <util/string/vector.h> -#include <util/string/cast.h> -#include <util/string/join.h> - + +#include <util/string/ascii.h> +#include <util/string/vector.h> +#include <util/string/cast.h> +#include <util/string/join.h> + namespace NKikimr::NSQS { -class TListPermissionsActor - : public TActionActor<TListPermissionsActor> { -public: +class TListPermissionsActor + : public TActionActor<TListPermissionsActor> { +public: TListPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) : TActionActor(sourceSqsRequest, EAction::ListPermissions, std::move(cb)) - { - Response_.MutableListPermissions()->SetRequestId(RequestId_); + { + Response_.MutableListPermissions()->SetRequestId(RequestId_); CopySecurityToken(Request()); - } - - static constexpr bool NeedExistingQueue() { - return false; - } - + } + + static constexpr bool NeedExistingQueue() { + return false; + } + static constexpr bool NeedUserSpecified() { return false; } -private: - bool DoValidate() override { +private: + bool DoValidate() override { if (!Request().GetPath()) { - MakeError(Response_.MutableListPermissions(), NErrors::MISSING_PARAMETER, "No Path parameter."); - return false; - } - + MakeError(Response_.MutableListPermissions(), NErrors::MISSING_PARAMETER, "No Path parameter."); + return false; + } + Path_ = MakeAbsolutePath(Request().GetPath()); - - if (IsForbiddenPath(Path_)) { + + if (IsForbiddenPath(Path_)) { MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Path_).c_str())); - return false; - } - - return true; - } - - TError* MutableErrorDesc() override { - return Response_.MutableListPermissions()->MutableError(); - } - - virtual TString GetCustomACLPath() const override { + return false; + } + + return true; + } + + TError* MutableErrorDesc() override { + return Response_.MutableListPermissions()->MutableError(); + } + + virtual TString GetCustomACLPath() const override { return MakeAbsolutePath(Request().GetPath()); - } - - TString DoGetQueueName() const override { - return {}; - } - + } + + TString DoGetQueueName() const override { + return {}; + } + void RequestSchemeShard(const TString& path) { - std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); + std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath(); - record->SetPath(path); - + record->SetPath(path); + Send(MakeTxProxyID(), navigateRequest.release()); - } - + } + void DoAction() override { - Become(&TThis::WaitSchemeShardResponse); + Become(&TThis::WaitSchemeShardResponse); RequestSchemeShard(Path_); - } - + } + STATEFN(WaitSchemeShardResponse) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, HandleSchemeShardResponse); - } - } - - using TGroupedACE = THashMap<TString, TSet<TStringBuf>>; - - TGroupedACE GroupACEBySID(const NACLibProto::TACL& acl) const { - TGroupedACE result; - for (const NACLibProto::TACE& ace : acl.GetACE()) { - if (static_cast<NACLib::EAccessType>(ace.GetAccessType()) == NACLib::EAccessType::Allow) { - if (ace.GetSID()) { - auto& acesForSIDSet = result[ace.GetSID()]; - const auto aces = GetAccessMatchingACE(ace.GetAccessRight()); - for (const TStringBuf& aceName : aces) { - acesForSIDSet.insert(aceName); - } - } - } - } - - return result; - } - - template<typename TMutablePermissions> - void ConvertACLToPermissions(const NACLibProto::TACL& acl, TMutablePermissions& nodePermissions) const { - const auto acesBySID = GroupACEBySID(acl); - for (auto&& pair : acesBySID) { - const auto& SID = pair.first; - const auto& acesSet = pair.second; - - auto* permissions = nodePermissions.Add(); - - permissions->SetSubject(SID); - for (const auto& aceName : acesSet) { - permissions->MutablePermissionNames()->Add(TString(aceName)); - } - } - } - + } + } + + using TGroupedACE = THashMap<TString, TSet<TStringBuf>>; + + TGroupedACE GroupACEBySID(const NACLibProto::TACL& acl) const { + TGroupedACE result; + for (const NACLibProto::TACE& ace : acl.GetACE()) { + if (static_cast<NACLib::EAccessType>(ace.GetAccessType()) == NACLib::EAccessType::Allow) { + if (ace.GetSID()) { + auto& acesForSIDSet = result[ace.GetSID()]; + const auto aces = GetAccessMatchingACE(ace.GetAccessRight()); + for (const TStringBuf& aceName : aces) { + acesForSIDSet.insert(aceName); + } + } + } + } + + return result; + } + + template<typename TMutablePermissions> + void ConvertACLToPermissions(const NACLibProto::TACL& acl, TMutablePermissions& nodePermissions) const { + const auto acesBySID = GroupACEBySID(acl); + for (auto&& pair : acesBySID) { + const auto& SID = pair.first; + const auto& acesSet = pair.second; + + auto* permissions = nodePermissions.Add(); + + permissions->SetSubject(SID); + for (const auto& aceName : acesSet) { + permissions->MutablePermissionNames()->Add(TString(aceName)); + } + } + } + void HandleSchemeShardResponse(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - const auto& record = ev->Get()->GetRecord(); - const auto status = record.GetStatus(); - switch (status) { + const auto& record = ev->Get()->GetRecord(); + const auto status = record.GetStatus(); + switch (status) { case NKikimrScheme::StatusSuccess: { - const auto& pathDescription = record.GetPathDescription(); + const auto& pathDescription = record.GetPathDescription(); const NKikimrSchemeOp::TDirEntry& entry = pathDescription.GetSelf(); - - TSecurityObject secObjWithACL(entry.GetOwner(), entry.GetACL(), false); - TSecurityObject secObjWithEffectiveACL(entry.GetOwner(), entry.GetEffectiveACL(), false); - - auto convertACLToPermissions = [this, &secObjWithACL, &secObjWithEffectiveACL](auto* mutableNodePermissions) { - ConvertACLToPermissions(secObjWithACL.GetACL(), *mutableNodePermissions->MutablePermissions()); - ConvertACLToPermissions(secObjWithEffectiveACL.GetACL(), *mutableNodePermissions->MutableEffectivePermissions()); - }; - - const size_t depth = CalculatePathDepth(SanitizeNodePath(Path_)); - if (depth < 2) { - convertACLToPermissions(Response_.MutableListPermissions()->MutableAccountPermissions()); - } else { - convertACLToPermissions(Response_.MutableListPermissions()->MutableQueuePermissions()); - } - - break; - } + + TSecurityObject secObjWithACL(entry.GetOwner(), entry.GetACL(), false); + TSecurityObject secObjWithEffectiveACL(entry.GetOwner(), entry.GetEffectiveACL(), false); + + auto convertACLToPermissions = [this, &secObjWithACL, &secObjWithEffectiveACL](auto* mutableNodePermissions) { + ConvertACLToPermissions(secObjWithACL.GetACL(), *mutableNodePermissions->MutablePermissions()); + ConvertACLToPermissions(secObjWithEffectiveACL.GetACL(), *mutableNodePermissions->MutableEffectivePermissions()); + }; + + const size_t depth = CalculatePathDepth(SanitizeNodePath(Path_)); + if (depth < 2) { + convertACLToPermissions(Response_.MutableListPermissions()->MutableAccountPermissions()); + } else { + convertACLToPermissions(Response_.MutableListPermissions()->MutableQueuePermissions()); + } + + break; + } case NKikimrScheme::StatusPathDoesNotExist: { MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Path_).c_str())); - break; - } - default: { - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); - break; - } - } - + break; + } + default: { + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + break; + } + } + SendReplyAndDie(); - } - + } + const TListPermissionsRequest& Request() const { return SourceSqsRequest_.GetListPermissions(); } -private: - TString Path_; -}; - +private: + TString Path_; +}; + IActor* CreateListPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) { return new TListPermissionsActor(sourceSqsRequest, std::move(cb)); -} - +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/list_queues.cpp b/ydb/core/ymq/actor/list_queues.cpp index 25ae108376..c9cba83f4e 100644 --- a/ydb/core/ymq/actor/list_queues.cpp +++ b/ydb/core/ymq/actor/list_queues.cpp @@ -1,7 +1,7 @@ #include "action.h" #include "error.h" #include "log.h" -#include "params.h" +#include "params.h" #include "serviceid.h" #include "executor.h" @@ -28,7 +28,7 @@ public: { CopyAccountName(Request()); Response_.MutableListQueues()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } @@ -50,19 +50,19 @@ private: } void DoAction() override { - Become(&TThis::StateFunc); - + Become(&TThis::StateFunc); + if (!UserExists_) { if (!IsCloud()) { MakeError(Response_.MutableListQueues(), NErrors::OPT_IN_REQUIRED, "The specified account does not exist."); } // else respond with an empty list for inexistent account SendReplyAndDie(); - return; - } - + return; + } + DiscoverQueues(); - } - + } + TString DoGetQueueName() const override { return TString(); } @@ -86,16 +86,16 @@ private: for (size_t i = 0; i < queues.Size(); ++i) { const TString name((TString(queues[i]["QueueName"]))); - const TString customQueueName((TString(queues[i]["CustomQueueName"]))); + const TString customQueueName((TString(queues[i]["CustomQueueName"]))); - if (prefix.empty() || AsciiHasPrefix((IsCloud() ? customQueueName : name), prefix)) { + if (prefix.empty() || AsciiHasPrefix((IsCloud() ? customQueueName : name), prefix)) { auto* item = result->AddQueues(); item->SetQueueName(name); - if (IsCloud()) { - item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName))); - } else { - item->SetQueueUrl(MakeQueueUrl(name)); - } + if (IsCloud()) { + item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName))); + } else { + item->SetQueueUrl(MakeQueueUrl(name)); + } } } } else { diff --git a/ydb/core/ymq/actor/list_users.cpp b/ydb/core/ymq/actor/list_users.cpp index d653fc20a3..0cc23a1f92 100644 --- a/ydb/core/ymq/actor/list_users.cpp +++ b/ydb/core/ymq/actor/list_users.cpp @@ -24,7 +24,7 @@ public: { CopyAccountName(Request()); Response_.MutableListUsers()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } diff --git a/ydb/core/ymq/actor/metering.cpp b/ydb/core/ymq/actor/metering.cpp index 94ccef7eb0..17b32be63a 100644 --- a/ydb/core/ymq/actor/metering.cpp +++ b/ydb/core/ymq/actor/metering.cpp @@ -1,29 +1,29 @@ -#include "metering.h" - -#include "cfg.h" -#include "serviceid.h" -#include "proxy_actor.h" - +#include "metering.h" + +#include "cfg.h" +#include "serviceid.h" +#include "proxy_actor.h" + #include <ydb/core/mind/address_classification/net_classifier.h> #include <ydb/core/ymq/base/action.h> #include <ydb/core/ymq/base/counters.h> #include <ydb/core/ymq/base/processed_request_attributes.h> - + #include <library/cpp/logger/global/global.h> #include <library/cpp/logger/record.h> - -#include <util/generic/guid.h> + +#include <util/generic/guid.h> #include <util/generic/serialized_enum.h> -#include <util/string/builder.h> -#include <util/system/hostname.h> - -#include <tuple> - -namespace NKikimr::NSQS { - -using namespace NAddressClassifier; - +#include <util/string/builder.h> +#include <util/system/hostname.h> + +#include <tuple> + +namespace NKikimr::NSQS { + +using namespace NAddressClassifier; + const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = { {"yacloud", ToString(ENetworkClass::cloud)} }; @@ -42,17 +42,17 @@ TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig:: InitAddressClassifier(config, std::move(labels)); } } - -bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const { + +bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const { return std::tie(ResourceId, TrafficType, NetworkClassLabel) < std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel); -} - -bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const { - return std::tie(ResourceId, QueueType) < - std::tie(rhs.ResourceId, rhs.QueueType); -} - +} + +bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const { + return std::tie(ResourceId, QueueType) < + std::tie(rhs.ResourceId, rhs.QueueType); +} + TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) { if (classifier) { auto result = classifier->ClassifyAddress(address); @@ -77,41 +77,41 @@ TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) co } else { result = ClassifyNetwork(AddressClassifier, address); INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]); - } + } INC_COUNTER(Counters, ClassifierRequestsResults[result]); return result; -} - -ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const { - return (bytes - 1) / blockSize + 1; -} - -bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) { - if (attrs.HttpStatusCode >= 500 || !attrs.FolderId || !attrs.SourceAddress) { - return false; // do not log with insufficient attributes - } - - TString resourceId = attrs.ResourceId; - EQueueType queueType = attrs.IsFifo ? EQueueType::fifo : EQueueType::std; - if (!IsActionForQueue(attrs.Action) || attrs.Action == EAction::GetQueueUrl) { - queueType = EQueueType::other; - resourceId = ""; - } +} + +ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const { + return (bytes - 1) / blockSize + 1; +} + +bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) { + if (attrs.HttpStatusCode >= 500 || !attrs.FolderId || !attrs.SourceAddress) { + return false; // do not log with insufficient attributes + } + + TString resourceId = attrs.ResourceId; + EQueueType queueType = attrs.IsFifo ? EQueueType::fifo : EQueueType::std; + if (!IsActionForQueue(attrs.Action) || attrs.Action == EAction::GetQueueUrl) { + queueType = EQueueType::other; + resourceId = ""; + } const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress); - - Y_VERIFY(attrs.RequestSizeInBytes); - Y_VERIFY(attrs.ResponseSizeInBytes); - + + Y_VERIFY(attrs.RequestSizeInBytes); + Y_VERIFY(attrs.ResponseSizeInBytes); + ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes; ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes; - - static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22 - ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes, - defaultRequestBlockSize); - - return true; -} - + + static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22 + ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes, + defaultRequestBlockSize); + + return true; +} + NSc::TValue CreateMeteringBillingRecord( const TString& folderId, const TString& resourceId, @@ -122,87 +122,87 @@ NSc::TValue CreateMeteringBillingRecord( const TString& unit, const NSc::TValue& tags ) { - const TString& billingRecordVersion = "v1"; - const ui64 utcSeconds = now.Seconds(); - - NSc::TValue record; - record["folder_id"] = folderId; - record["id"] = CreateGuidAsString(); - if (resourceId) { - record["resource_id"] = resourceId; - } - record["schema"] = schema; - record["source_id"] = fqdn; - record["source_wt"] = utcSeconds; - - auto& usage = record["usage"].SetDict(); - usage["start"] = utcSeconds; - usage["finish"] = utcSeconds; - usage["type"] = "delta"; - usage["unit"] = unit; - usage["quantity"] = quantity; - - record["tags"] = tags; - - record["version"] = billingRecordVersion; - - return record; -} - -TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const { - TVector<NSc::TValue> result; - - for (const auto& perFolderData : ReportedTraffic) { - const TString& folderId = perFolderData.first; - for (const auto& [trafficKey, trafficValue] : perFolderData.second) { - NSc::TValue tags; - tags.SetDict(); + const TString& billingRecordVersion = "v1"; + const ui64 utcSeconds = now.Seconds(); + + NSc::TValue record; + record["folder_id"] = folderId; + record["id"] = CreateGuidAsString(); + if (resourceId) { + record["resource_id"] = resourceId; + } + record["schema"] = schema; + record["source_id"] = fqdn; + record["source_wt"] = utcSeconds; + + auto& usage = record["usage"].SetDict(); + usage["start"] = utcSeconds; + usage["finish"] = utcSeconds; + usage["type"] = "delta"; + usage["unit"] = unit; + usage["quantity"] = quantity; + + record["tags"] = tags; + + record["version"] = billingRecordVersion; + + return record; +} + +TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const { + TVector<NSc::TValue> result; + + for (const auto& perFolderData : ReportedTraffic) { + const TString& folderId = perFolderData.first; + for (const auto& [trafficKey, trafficValue] : perFolderData.second) { + NSc::TValue tags; + tags.SetDict(); tags["type"] = trafficKey.NetworkClassLabel; - tags["direction"] = ToString(trafficKey.TrafficType); - + tags["direction"] = ToString(trafficKey.TrafficType); + result.push_back(CreateMeteringBillingRecord(folderId, - trafficKey.ResourceId, - "ymq.traffic.v1", - fqdn, - now, - trafficValue, - "byte", - tags)); - } - } - - return result; -} - -TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const { - TVector<NSc::TValue> result; - - for (const auto& perFolderData : ReportedRequests) { - const TString& folderId = perFolderData.first; - for (const auto& [requestsKey, requestsValue] : perFolderData.second) { - NSc::TValue tags; - tags.SetDict(); - tags["queue_type"] = ToString(requestsKey.QueueType); - + trafficKey.ResourceId, + "ymq.traffic.v1", + fqdn, + now, + trafficValue, + "byte", + tags)); + } + } + + return result; +} + +TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const { + TVector<NSc::TValue> result; + + for (const auto& perFolderData : ReportedRequests) { + const TString& folderId = perFolderData.first; + for (const auto& [requestsKey, requestsValue] : perFolderData.second) { + NSc::TValue tags; + tags.SetDict(); + tags["queue_type"] = ToString(requestsKey.QueueType); + result.push_back(CreateMeteringBillingRecord(folderId, - requestsKey.ResourceId, - "ymq.requests.v1", - fqdn, - now, - requestsValue, - "request", - tags)); - } - } - - return result; -} - -void TProcessedRequestsAggregator::ResetReportsStorage() { - ReportedTraffic.clear(); - ReportedRequests.clear(); -} - + requestsKey.ResourceId, + "ymq.requests.v1", + fqdn, + now, + requestsValue, + "request", + tags)); + } + } + + return result; +} + +void TProcessedRequestsAggregator::ResetReportsStorage() { + ReportedTraffic.clear(); + ReportedRequests.clear(); +} + void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) { NetClassifier = classifier; } @@ -220,87 +220,87 @@ void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TS AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels)); } -class TMeteringActor +class TMeteringActor : public TActorBootstrapped<TMeteringActor> -{ -public: - TMeteringActor() - : HostFQDN(FQDNHostName()) - { - } - +{ +public: + TMeteringActor() + : HostFQDN(FQDNHostName()) + { + } + void Bootstrap(const TActorContext& ctx) { - Become(&TThis::Work); - + Become(&TThis::Work); + ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); Aggregator = MakeHolder<TProcessedRequestsAggregator>(Cfg()); - + FlushProcessedRequestsAttributes(); - } - - void WriteLogRecord(const NSc::TValue& record, TStringBuilder& records) const { - records << record.ToJsonSafe() << "\n"; - } - + } + + void WriteLogRecord(const NSc::TValue& record, TStringBuilder& records) const { + records << record.ToJsonSafe() << "\n"; + } + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SQS_METERING_ACTOR; - } - + } + void FlushProcessedRequestsAttributes() { - Y_VERIFY(Aggregator); - + Y_VERIFY(Aggregator); + Schedule(TDuration::MilliSeconds(Cfg().GetMeteringFlushingIntervalMs()), new TEvWakeup()); - + const auto reportedTrafficRecords = Aggregator->DumpReportedTrafficAsJsonArray(HostFQDN, TActivationContext::Now()); const auto reportedRequestsRecords = Aggregator->DumpReportedRequestsAsJsonArray(HostFQDN, TActivationContext::Now()); - + if (Cfg().GetMeteringLogFilePath()) { - TStringBuilder records; - - for (const auto& trafficRecord : reportedTrafficRecords) { - WriteLogRecord(trafficRecord, records); - } - for (const auto& requestRecord : reportedRequestsRecords) { - WriteLogRecord(requestRecord, records); - } - - if (const TString packedRecords = records) { - TLoggerOperator<TGlobalLog>::Log().Write(packedRecords.Data(), packedRecords.Size()); - } - } - - Aggregator->ResetReportsStorage(); - } - + TStringBuilder records; + + for (const auto& trafficRecord : reportedTrafficRecords) { + WriteLogRecord(trafficRecord, records); + } + for (const auto& requestRecord : reportedRequestsRecords) { + WriteLogRecord(requestRecord, records); + } + + if (const TString packedRecords = records) { + TLoggerOperator<TGlobalLog>::Log().Write(packedRecords.Data(), packedRecords.Size()); + } + } + + Aggregator->ResetReportsStorage(); + } + void HandleWakeup(TEvWakeup::TPtr&) { FlushProcessedRequestsAttributes(); - } - + } + void HandleReportProcessedRequestAttributes(TSqsEvents::TEvReportProcessedRequestAttributes::TPtr& ev) { - Y_VERIFY(Aggregator); - - Aggregator->Add(ev->Get()->Data); - } - + Y_VERIFY(Aggregator); + + Aggregator->Add(ev->Get()->Data); + } + STATEFN(Work) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate); hFunc(TSqsEvents::TEvReportProcessedRequestAttributes, HandleReportProcessedRequestAttributes); - } - } + } + } -private: +private: void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp); } private: - const TString HostFQDN; - THolder<TProcessedRequestsAggregator> Aggregator; -}; - + const TString HostFQDN; + THolder<TProcessedRequestsAggregator> Aggregator; +}; + IActor* CreateSqsMeteringService() { return new TMeteringActor(); } - -} // namespace NKikimr::NSQS + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/metering.h b/ydb/core/ymq/actor/metering.h index c9d5a58991..e6d5f8da1c 100644 --- a/ydb/core/ymq/actor/metering.h +++ b/ydb/core/ymq/actor/metering.h @@ -1,20 +1,20 @@ -#pragma once +#pragma once #include "defs.h" - + #include <ydb/core/ymq/base/action.h> #include <ydb/core/util/address_classifier.h> #include <ydb/core/ymq/base/processed_request_attributes.h> - + #include <ydb/core/protos/config.pb.h> - + #include <library/cpp/scheme/scheme.h> - -#include <util/datetime/base.h> -#include <util/generic/hash.h> -#include <util/generic/map.h> - -namespace NKikimr::NSQS { - + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/generic/map.h> + +namespace NKikimr::NSQS { + struct TMeteringCounters; NSc::TValue CreateMeteringBillingRecord( @@ -28,60 +28,60 @@ NSc::TValue CreateMeteringBillingRecord( const NSc::TValue& tags ); -class TProcessedRequestsAggregator { -public: +class TProcessedRequestsAggregator { +public: static const THashMap<TString, TString> LabelTransformation; public: - TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config); - - // Do not rename enum members in the enums below (SQS-22) - enum ETrafficType { - ingress = 0, - egress - }; - - enum ENetworkClass { - inet = 0, - cloud, + TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config); + + // Do not rename enum members in the enums below (SQS-22) + enum ETrafficType { + ingress = 0, + egress + }; + + enum ENetworkClass { + inet = 0, + cloud, yandex, unknown - }; - - enum EQueueType { - std = 0, - fifo, - other - }; - - struct TReportedTrafficKey { - TString ResourceId; - ETrafficType TrafficType; + }; + + enum EQueueType { + std = 0, + fifo, + other + }; + + struct TReportedTrafficKey { + TString ResourceId; + ETrafficType TrafficType; TString NetworkClassLabel; - - bool operator<(const TReportedTrafficKey& rhs) const; - }; - - struct TReportedRequestsKey { - TString ResourceId; - EQueueType QueueType; - - bool operator<(const TReportedRequestsKey& rhs) const; - }; - + + bool operator<(const TReportedTrafficKey& rhs) const; + }; + + struct TReportedRequestsKey { + TString ResourceId; + EQueueType QueueType; + + bool operator<(const TReportedRequestsKey& rhs) const; + }; + TString ClassifyNetwork(const TString& address) const; - - ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const; - - bool Add(const TProcessedRequestAttributes& attrs); - - TVector<NSc::TValue> DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const; - TVector<NSc::TValue> DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const; - - void ResetReportsStorage(); - + + ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const; + + bool Add(const TProcessedRequestAttributes& attrs); + + TVector<NSc::TValue> DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const; + TVector<NSc::TValue> DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const; + + void ResetReportsStorage(); + void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp); -private: +private: static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address); void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels); @@ -89,12 +89,12 @@ private: private: bool NetClassifierOnly; - THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic; - THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests; + THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic; + THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests; NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier; NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier; TIntrusivePtr<TMeteringCounters> Counters; -}; - -} // namespace NKikimr::NSQS +}; + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/migration.cpp b/ydb/core/ymq/actor/migration.cpp index e547e97558..aafb5ef35b 100644 --- a/ydb/core/ymq/actor/migration.cpp +++ b/ydb/core/ymq/actor/migration.cpp @@ -350,7 +350,7 @@ void TQueueMigrationActor::StartAltering() { CheckAddColumn("Attributes", "DlqArn", NScheme::NTypeIds::Utf8); CheckAddColumn("Attributes", "MaxReceiveCount", NScheme::NTypeIds::Uint64); CheckAddColumn("Attributes", "ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64); - + CheckAddColumn("State", "InflyVersion", NScheme::NTypeIds::Uint64); if (!IsFifoQueue) { for (ui64 shard = 0; shard < ShardsCount; ++shard) { diff --git a/ydb/core/ymq/actor/modify_permissions.cpp b/ydb/core/ymq/actor/modify_permissions.cpp index 8b9e1cb2d7..87207f63e5 100644 --- a/ydb/core/ymq/actor/modify_permissions.cpp +++ b/ydb/core/ymq/actor/modify_permissions.cpp @@ -1,174 +1,174 @@ -#include "action.h" - +#include "action.h" + #include <google/protobuf/text_format.h> - -#include <util/string/ascii.h> -#include <util/string/vector.h> -#include <util/string/cast.h> -#include <util/string/join.h> - + +#include <util/string/ascii.h> +#include <util/string/vector.h> +#include <util/string/cast.h> +#include <util/string/join.h> + namespace NKikimr::NSQS { -class TModifyPermissionsActor - : public TActionActor<TModifyPermissionsActor> { -public: +class TModifyPermissionsActor + : public TActionActor<TModifyPermissionsActor> { +public: TModifyPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) : TActionActor(sourceSqsRequest, EAction::ModifyPermissions, std::move(cb)) - { - Response_.MutableModifyPermissions()->SetRequestId(RequestId_); + { + Response_.MutableModifyPermissions()->SetRequestId(RequestId_); CopySecurityToken(Request()); - } - - static constexpr bool NeedExistingQueue() { - return false; - } - + } + + static constexpr bool NeedExistingQueue() { + return false; + } + static constexpr bool NeedUserSpecified() { return false; } -private: - bool TryConvertSQSPermissionToYdbACLMask(const TString& permission, ui32& mask) const { - mask = GetACERequiredAccess(permission); - return mask != 0; - } - -#define PROCESS_MODIFY_ACL_ACTION(PERMISSION_NAME, ACCESS_ACTION_TYPE, CLEAR_FLAG) \ - case TPermissionsAction::Y_CAT(k, PERMISSION_NAME): { \ - const auto& SID = action.Y_CAT(Get, PERMISSION_NAME)().GetSubject(); \ - if (CLEAR_FLAG) { \ - ACLDiff_.ClearAccessForSid(SID); \ - } \ - for (const auto& name : action.Y_CAT(Get, PERMISSION_NAME)().GetPermissionNames()) { \ - if (!TryConvertSQSPermissionToYdbACLMask(name, mask)) { \ - TString permissionName(Y_STRINGIZE(PERMISSION_NAME)); \ - permissionName.to_lower(); \ +private: + bool TryConvertSQSPermissionToYdbACLMask(const TString& permission, ui32& mask) const { + mask = GetACERequiredAccess(permission); + return mask != 0; + } + +#define PROCESS_MODIFY_ACL_ACTION(PERMISSION_NAME, ACCESS_ACTION_TYPE, CLEAR_FLAG) \ + case TPermissionsAction::Y_CAT(k, PERMISSION_NAME): { \ + const auto& SID = action.Y_CAT(Get, PERMISSION_NAME)().GetSubject(); \ + if (CLEAR_FLAG) { \ + ACLDiff_.ClearAccessForSid(SID); \ + } \ + for (const auto& name : action.Y_CAT(Get, PERMISSION_NAME)().GetPermissionNames()) { \ + if (!TryConvertSQSPermissionToYdbACLMask(name, mask)) { \ + TString permissionName(Y_STRINGIZE(PERMISSION_NAME)); \ + permissionName.to_lower(); \ const auto errorMsg = Sprintf("ModifyPermissions failed to %s unknown permission %s.", permissionName.c_str(), name.c_str()); \ - MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, errorMsg); \ - return false; \ - } \ - ACLDiff_.Y_CAT(ACCESS_ACTION_TYPE, Access)(NACLib::EAccessType::Allow, mask, SID); \ - } \ - break; \ - } - - bool DoValidate() override { + MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, errorMsg); \ + return false; \ + } \ + ACLDiff_.Y_CAT(ACCESS_ACTION_TYPE, Access)(NACLib::EAccessType::Allow, mask, SID); \ + } \ + break; \ + } + + bool DoValidate() override { if (!Request().GetResource()) { - MakeError(Response_.MutableModifyPermissions(), NErrors::MISSING_PARAMETER, "No Resource parameter."); - return false; - } - + MakeError(Response_.MutableModifyPermissions(), NErrors::MISSING_PARAMETER, "No Resource parameter."); + return false; + } + Resource_ = MakeAbsolutePath(Request().GetResource()); - - if (IsForbiddenPath(Resource_)) { + + if (IsForbiddenPath(Resource_)) { MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Resource_).c_str())); - return false; - } - + return false; + } + if (Request().GetClearACL()) { - ACLDiff_.ClearAccess(); - } - - ui32 mask = 0; + ACLDiff_.ClearAccess(); + } + + ui32 mask = 0; for (const auto& action : Request().GetActions()) { - switch (action.Action_case()) { - PROCESS_MODIFY_ACL_ACTION(Set, Add, true); - PROCESS_MODIFY_ACL_ACTION(Grant, Add, false); - PROCESS_MODIFY_ACL_ACTION(Revoke, Remove, false); - default: { + switch (action.Action_case()) { + PROCESS_MODIFY_ACL_ACTION(Set, Add, true); + PROCESS_MODIFY_ACL_ACTION(Grant, Add, false); + PROCESS_MODIFY_ACL_ACTION(Revoke, Remove, false); + default: { MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Unknown ModifyPermissions action."); - return false; - } - } - } - - return true; - } - - TError* MutableErrorDesc() override { - return Response_.MutableModifyPermissions()->MutableError(); - } - - virtual TString GetCustomACLPath() const override { + return false; + } + } + } + + return true; + } + + TError* MutableErrorDesc() override { + return Response_.MutableModifyPermissions()->MutableError(); + } + + virtual TString GetCustomACLPath() const override { return MakeAbsolutePath(Request().GetResource()); - } - - TString DoGetQueueName() const override { - return {}; - } - + } + + TString DoGetQueueName() const override { + return {}; + } + void DoAction() override { - Become(&TThis::StateFunc); - - TStringBuf workingDir, resource; - TStringBuf(Resource_).RSplit('/', workingDir, resource); - - auto proposeRequest = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; + Become(&TThis::StateFunc); + + TStringBuf workingDir, resource; + TStringBuf(Resource_).RSplit('/', workingDir, resource); + + auto proposeRequest = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpModifyACL); - modifyScheme->SetWorkingDir(TString(workingDir)); - modifyScheme->MutableModifyACL()->SetName(TString(resource)); - - modifyScheme->MutableModifyACL()->SetDiffACL(ACLDiff_.SerializeAsString()); + modifyScheme->SetWorkingDir(TString(workingDir)); + modifyScheme->MutableModifyACL()->SetName(TString(resource)); + + modifyScheme->MutableModifyACL()->SetDiffACL(ACLDiff_.SerializeAsString()); Send(MakeTxProxyID(), proposeRequest.Release()); - } - + } + STATEFN(StateFunc) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleProposeTransactionStatus); - } - } - + } + } + void LogModifyACLRequestResultSafe(bool success) const { - const TString user = (UserToken_ && UserToken_->GetUserSID()) ? UserToken_->GetUserSID() : "Someone"; - - TModifyPermissionsRequest copy; + const TString user = (UserToken_ && UserToken_->GetUserSID()) ? UserToken_->GetUserSID() : "Someone"; + + TModifyPermissionsRequest copy; copy.CopyFrom(Request()); - copy.ClearCredentials(); - copy.ClearResource(); - - if (success) { + copy.ClearCredentials(); + copy.ClearResource(); + + if (success) { RLOG_SQS_WARN(user << " modified ACL for " << Resource_ << " with request " << copy); - } else { + } else { RLOG_SQS_ERROR(user << " failed to modify ACL for " << Resource_ << " with request " << copy); - } - } - + } + } + void HandleProposeTransactionStatus(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) { - const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get(); - const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus()); - switch (status) { - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: { + const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get(); + const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus()); + switch (status) { + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete: + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: { LogModifyACLRequestResultSafe(true); - break; - } - - case TEvTxUserProxy::TResultStatus::ResolveError: - case TEvTxUserProxy::TResultStatus::AccessDenied: - case TEvTxUserProxy::TResultStatus::ExecError: - default: { + break; + } + + case TEvTxUserProxy::TResultStatus::ResolveError: + case TEvTxUserProxy::TResultStatus::AccessDenied: + case TEvTxUserProxy::TResultStatus::ExecError: + default: { LogModifyACLRequestResultSafe(false); - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); - break; - } - } + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + break; + } + } SendReplyAndDie(); - } - + } + const TModifyPermissionsRequest& Request() const { return SourceSqsRequest_.GetModifyPermissions(); } -private: - NACLib::TDiffACL ACLDiff_; - TString Resource_; -}; - +private: + NACLib::TDiffACL ACLDiff_; + TString Resource_; +}; + IActor* CreateModifyPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) { return new TModifyPermissionsActor(sourceSqsRequest, std::move(cb)); -} - +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/params.h b/ydb/core/ymq/actor/params.h index abe1154c8d..cc23308af1 100644 --- a/ydb/core/ymq/actor/params.h +++ b/ydb/core/ymq/actor/params.h @@ -47,12 +47,12 @@ public: return *this; } - TParameters& Uint32(const TString& name, ui32 value) { - DataType(name, NScheme::NTypeIds::Uint32); - Params_->MutableValue()->AddStruct()->SetUint32(value); - return *this; - } - + TParameters& Uint32(const TString& name, ui32 value) { + DataType(name, NScheme::NTypeIds::Uint32); + Params_->MutableValue()->AddStruct()->SetUint32(value); + return *this; + } + TParameters& OptionalBool(const TString& name, const TMaybe<bool>& value) { OptionalDataType(name, NScheme::NTypeIds::Bool); if (value) { @@ -79,17 +79,17 @@ public: return *this; } - TParameters& OptionalUtf8(const TString& name, const TMaybe<TString>& value) { - OptionalDataType(name, NScheme::NTypeIds::Utf8); - if (value) { - Params_->MutableValue()->AddStruct()->MutableOptional()->SetText(*value); - } else { - Params_->MutableValue()->AddStruct(); - } - - return *this; - } - + TParameters& OptionalUtf8(const TString& name, const TMaybe<TString>& value) { + OptionalDataType(name, NScheme::NTypeIds::Utf8); + if (value) { + Params_->MutableValue()->AddStruct()->MutableOptional()->SetText(*value); + } else { + Params_->MutableValue()->AddStruct(); + } + + return *this; + } + private: void DataType(const TString& name, NScheme::TTypeId typeId) { auto* member = Params_->MutableType()->MutableStruct()->AddMember(); diff --git a/ydb/core/ymq/actor/proxy_actor.cpp b/ydb/core/ymq/actor/proxy_actor.cpp index 98d85210a5..cd2b9c0b8b 100644 --- a/ydb/core/ymq/actor/proxy_actor.cpp +++ b/ydb/core/ymq/actor/proxy_actor.cpp @@ -1,4 +1,4 @@ -#include "cfg.h" +#include "cfg.h" #include "error.h" #include "proxy_actor.h" @@ -10,7 +10,7 @@ #include <util/string/builder.h> #include <util/system/defaults.h> - + namespace NKikimr::NSQS { void TProxyActor::Bootstrap() { @@ -21,20 +21,20 @@ void TProxyActor::Bootstrap() { const auto& cfg = Cfg(); if (cfg.GetYandexCloudMode()) { - TString securityToken; -#define SQS_REQUEST_CASE(action) \ - const auto& request = Request_.Y_CAT(Get, action)(); \ - securityToken = ExtractSecurityToken<typename std::remove_reference<decltype(request)>::type, TCredentials>(request); - SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false)); -#undef SQS_REQUEST_CASE - TStringBuf tokenBuf(securityToken); - UserName_ = TString(tokenBuf.NextTok(':')); - FolderId_ = TString(tokenBuf.NextTok(':')); - - // TODO: handle empty cloud id better + TString securityToken; +#define SQS_REQUEST_CASE(action) \ + const auto& request = Request_.Y_CAT(Get, action)(); \ + securityToken = ExtractSecurityToken<typename std::remove_reference<decltype(request)>::type, TCredentials>(request); + SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false)); +#undef SQS_REQUEST_CASE + TStringBuf tokenBuf(securityToken); + UserName_ = TString(tokenBuf.NextTok(':')); + FolderId_ = TString(tokenBuf.NextTok(':')); + + // TODO: handle empty cloud id better RLOG_SQS_DEBUG("Proxy actor: used " << UserName_ << " as an account name and " << QueueName_ << " as a queue name"); - } - + } + if (!UserName_ || !QueueName_) { RLOG_SQS_WARN("Validation error: No " << (!UserName_ ? "user name" : "queue name") << " in proxy actor"); SendErrorAndDie(NErrors::INVALID_PARAMETER_VALUE, "Both account and queue name should be specified."); @@ -107,20 +107,20 @@ void TProxyActor::SendErrorAndDie(const TErrorClass& error, const TString& messa detailedCounters->APIStatuses.AddError(error.ErrorCode); } NKikimrClient::TSqsResponse response; -#define SQS_REQUEST_CASE(action) \ +#define SQS_REQUEST_CASE(action) \ MakeError(response.Y_CAT(Mutable, action)(), error, message); \ response.Y_CAT(Mutable, action)()->SetRequestId(RequestId_); - SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false)); - -#undef SQS_REQUEST_CASE + SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false)); +#undef SQS_REQUEST_CASE + if (Cfg().GetYandexCloudMode()) { - response.SetFolderId(FolderId_); - response.SetIsFifo(false); - response.SetResourceId(QueueName_); - } - + response.SetFolderId(FolderId_); + response.SetIsFifo(false); + response.SetResourceId(QueueName_); + } + SendReplyAndDie(response); } @@ -136,11 +136,11 @@ void TProxyActor::HandleResponse(TSqsEvents::TEvProxySqsResponse::TPtr& ev) { void TProxyActor::HandleWakeup(TEvWakeup::TPtr&) { TString actionName; -#define SQS_REQUEST_CASE(action) actionName = Y_STRINGIZE(action); +#define SQS_REQUEST_CASE(action) actionName = Y_STRINGIZE(action); - SQS_SWITCH_REQUEST(Request_, break;); + SQS_SWITCH_REQUEST(Request_, break;); -#undef SQS_REQUEST_CASE +#undef SQS_REQUEST_CASE RLOG_SQS_ERROR("Proxy request timeout. User [" << UserName_ << "] Queue [" << QueueName_ << "] Action [" << actionName << "]"); @@ -179,11 +179,11 @@ const TErrorClass& TProxyActor::GetErrorClass(TSqsEvents::TEvProxySqsResponse::E } bool TProxyActor::NeedCreateProxyActor(const NKikimrClient::TSqsRequest& req) { -#define SQS_REQUEST_CASE(action) return true; +#define SQS_REQUEST_CASE(action) return true; - SQS_SWITCH_REQUEST(req, return false) + SQS_SWITCH_REQUEST(req, return false) -#undef SQS_REQUEST_CASE +#undef SQS_REQUEST_CASE } bool TProxyActor::NeedCreateProxyActor(EAction action) { @@ -191,14 +191,14 @@ bool TProxyActor::NeedCreateProxyActor(EAction action) { } void TProxyActor::RetrieveUserAndQueueParameters() { -// User name might be changed later in bootstrap for cloud mode -#define SQS_REQUEST_CASE(action) \ +// User name might be changed later in bootstrap for cloud mode +#define SQS_REQUEST_CASE(action) \ UserName_ = Request_.Y_CAT(Get, action)().GetAuth().GetUserName(); \ QueueName_ = Request_.Y_CAT(Get, action)().GetQueueName(); \ - SQS_SWITCH_REQUEST(Request_, throw TSQSException(NErrors::INVALID_ACTION) << "Incorrect request type") + SQS_SWITCH_REQUEST(Request_, throw TSQSException(NErrors::INVALID_ACTION) << "Incorrect request type") - #undef SQS_REQUEST_CASE + #undef SQS_REQUEST_CASE } } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/proxy_actor.h b/ydb/core/ymq/actor/proxy_actor.h index b553716bf0..123185c3ea 100644 --- a/ydb/core/ymq/actor/proxy_actor.h +++ b/ydb/core/ymq/actor/proxy_actor.h @@ -14,24 +14,24 @@ namespace NKikimr::NSQS { -#define SQS_REQUEST_CASE_WRAP(action) \ - case NKikimrClient::TSqsRequest::Y_CAT(k, action): { \ - SQS_REQUEST_CASE(action) \ - break; \ - } - -// DO NOT proxy account creation or queue listing - -#define SQS_SWITCH_REQUEST_CUSTOM(request, enumerate, default_case) \ - switch ((request).GetRequestCase()) { \ - enumerate(SQS_REQUEST_CASE_WRAP) \ - default: \ - default_case; \ - } - -#define SQS_SWITCH_REQUEST(request, default_case) \ - SQS_SWITCH_REQUEST_CUSTOM(request, ENUMERATE_PROXY_ACTIONS, default_case) - +#define SQS_REQUEST_CASE_WRAP(action) \ + case NKikimrClient::TSqsRequest::Y_CAT(k, action): { \ + SQS_REQUEST_CASE(action) \ + break; \ + } + +// DO NOT proxy account creation or queue listing + +#define SQS_SWITCH_REQUEST_CUSTOM(request, enumerate, default_case) \ + switch ((request).GetRequestCase()) { \ + enumerate(SQS_REQUEST_CASE_WRAP) \ + default: \ + default_case; \ + } + +#define SQS_SWITCH_REQUEST(request, default_case) \ + SQS_SWITCH_REQUEST_CUSTOM(request, ENUMERATE_PROXY_ACTIONS, default_case) + class TProxyActor : public TActorBootstrapped<TProxyActor> { @@ -81,7 +81,7 @@ private: NKikimrClient::TSqsRequest Request_; TString QueueName_; TString UserName_; - TString FolderId_; + TString FolderId_; THolder<IReplyCallback> Cb_; bool ErrorResponse_ = false; TInstant StartTs_; diff --git a/ydb/core/ymq/actor/purge_queue.cpp b/ydb/core/ymq/actor/purge_queue.cpp index 72830bdd4a..20ed641fde 100644 --- a/ydb/core/ymq/actor/purge_queue.cpp +++ b/ydb/core/ymq/actor/purge_queue.cpp @@ -21,7 +21,7 @@ public: { CopyAccountName(Request()); Response_.MutablePurgeQueue()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } @@ -121,11 +121,11 @@ private: const auto& entry = Request().GetEntries(i); auto& req = *ret[i].MutablePurgeQueue(); req.MutableAuth()->SetUserName(UserName_); - + if (Request().HasCredentials()) { *req.MutableCredentials() = Request().GetCredentials(); - } - + } + req.SetQueueName(entry.GetQueueName()); req.SetId(entry.GetId()); } diff --git a/ydb/core/ymq/actor/queue_leader.cpp b/ydb/core/ymq/actor/queue_leader.cpp index 2c6e81d827..2840b6ef09 100644 --- a/ydb/core/ymq/actor/queue_leader.cpp +++ b/ydb/core/ymq/actor/queue_leader.cpp @@ -41,7 +41,7 @@ const TString INFLY_INVALIDATION_REASON_DELETED = "MessageDeleted"; TQueueLeader::TQueueLeader(TString userName, TString queueName, TString folderId, TString rootUrl, TIntrusivePtr<TQueueCounters> counters, TIntrusivePtr<TUserCounters> userCounters, const TActorId& schemeCache, const TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions>& quoterResourcesForUser) : UserName_(std::move(userName)) , QueueName_(std::move(queueName)) - , FolderId_(std::move(folderId)) + , FolderId_(std::move(folderId)) , RootUrl_(std::move(rootUrl)) , SchemeCache_(schemeCache) , Counters_(std::move(counters)) @@ -260,13 +260,13 @@ void TQueueLeader::HandleExecuteWhileWorking(TSqsEvents::TEvExecute::TPtr& ev) { void TQueueLeader::Prepare(TSqsEvents::TEvExecute::TPtr& ev) { const TSqsEvents::TEvExecute& req = *ev->Get(); RLOG_SQS_REQ_DEBUG(req.RequestId, "Preparing query(idx=" << req.QueryIdx << ")"); - + TExecutorBuilder(SelfId(), req.RequestId) .User(UserName_) .Queue(QueueName_) .Shard(req.Shard) - .QueueVersion(QueueVersion_) - .Fifo(IsFifoQueue_) + .QueueVersion(QueueVersion_) + .Fifo(IsFifoQueue_) .Mode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE) .QueryId(req.QueryIdx) .RetryOnTimeout(req.RetryOnTimeout) @@ -337,8 +337,8 @@ void TQueueLeader::ExecuteRequest(TSqsEvents::TEvExecute::TPtr& ev, const TStrin .User(UserName_) .Queue(QueueName_) .Shard(req.Shard) - .QueueVersion(QueueVersion_) - .Fifo(IsFifoQueue_) + .QueueVersion(QueueVersion_) + .Fifo(IsFifoQueue_) .QueryId(req.QueryIdx) .Bin(compiled) .RetryOnTimeout(req.RetryOnTimeout) @@ -556,7 +556,7 @@ void TQueueLeader::OnFifoGroupLocked(const TString& requestId, const TSqsEvents: auto& msg = reqInfo.LockedFifoMessages.back(); msg.RandomId = offsets[i]["RandomId"]; msg.Offset = offsets[i]["Head"]; - msg.GroupId = offsets[i]["GroupId"]; + msg.GroupId = offsets[i]["GroupId"]; } if (truncated) { @@ -579,15 +579,15 @@ void TQueueLeader::OnFifoGroupLocked(const TString& requestId, const TSqsEvents: } void TQueueLeader::ReadFifoMessages(TReceiveMessageBatchRequestProcessing& reqInfo) { - ui32 maxReceiveCount = 0; // not set + ui32 maxReceiveCount = 0; // not set if (Cfg().GetEnableDeadLetterQueues() && DlqInfo_) { - const auto& dlqInfo(*DlqInfo_); - if (dlqInfo.DlqName && dlqInfo.QueueId) { - // dlq is set and resolved - maxReceiveCount = dlqInfo.MaxReceiveCount; - } - } - + const auto& dlqInfo(*DlqInfo_); + if (dlqInfo.DlqName && dlqInfo.QueueId) { + // dlq is set and resolved + maxReceiveCount = dlqInfo.MaxReceiveCount; + } + } + TExecutorBuilder builder(SelfId(), reqInfo.Event->Get()->RequestId); builder .User(UserName_) @@ -600,46 +600,46 @@ void TQueueLeader::ReadFifoMessages(TReceiveMessageBatchRequestProcessing& reqIn NClient::TWriteValue params = builder.ParamsValue(); params["NOW"] = ui64(TActivationContext::Now().MilliSeconds()); - ui64 index = 0; - - THashSet<TString> usedGroups; // mitigates extremely rare bug with duplicated groups during locking + ui64 index = 0; + + THashSet<TString> usedGroups; // mitigates extremely rare bug with duplicated groups during locking for (const auto& msg : reqInfo.LockedFifoMessages) { - if (usedGroups.insert(msg.GroupId).second) { - auto key = params["KEYS"].AddListItem(); - - key["RandomId"] = msg.RandomId; - key["Offset"] = msg.Offset; - - if (maxReceiveCount) { - key["GroupId"].Bytes(msg.GroupId); - key["Index"] = index++; - } - } - } - - if (maxReceiveCount) { - // perform heavy read and move transaction (DLQ) - Y_VERIFY(DlqInfo_); - + if (usedGroups.insert(msg.GroupId).second) { + auto key = params["KEYS"].AddListItem(); + + key["RandomId"] = msg.RandomId; + key["Offset"] = msg.Offset; + + if (maxReceiveCount) { + key["GroupId"].Bytes(msg.GroupId); + key["Index"] = index++; + } + } + } + + if (maxReceiveCount) { + // perform heavy read and move transaction (DLQ) + Y_VERIFY(DlqInfo_); + const TQueuePath currentQueuePath = { Cfg().GetRoot(), UserName_, QueueName_, QueueVersion_ }; const TQueuePath deadLetterQueuePath = { Cfg().GetRoot(), UserName_, DlqInfo_->QueueId, DlqInfo_->QueueVersion }; - - const TString transactionText = Sprintf(GetFifoQueryById(READ_OR_REDRIVE_MESSAGE_ID), - currentQueuePath.GetVersionedQueuePath().c_str(), - 0, - deadLetterQueuePath.GetVersionedQueuePath().c_str()); - builder - .Text(transactionText) - .Params() - .Uint32("MAX_RECEIVE_COUNT", maxReceiveCount) - .Uint64("RANDOM_ID", RandomNumber<ui64>()); - } else { - builder - .QueryId(READ_MESSAGE_ID); - } - - const bool usedDLQ = maxReceiveCount > 0; - + + const TString transactionText = Sprintf(GetFifoQueryById(READ_OR_REDRIVE_MESSAGE_ID), + currentQueuePath.GetVersionedQueuePath().c_str(), + 0, + deadLetterQueuePath.GetVersionedQueuePath().c_str()); + builder + .Text(transactionText) + .Params() + .Uint32("MAX_RECEIVE_COUNT", maxReceiveCount) + .Uint64("RANDOM_ID", RandomNumber<ui64>()); + } else { + builder + .QueryId(READ_MESSAGE_ID); + } + + const bool usedDLQ = maxReceiveCount > 0; + builder.OnExecuted([this, requestId = reqInfo.Event->Get()->RequestId, usedDLQ] (const TSqsEvents::TEvExecuted::TRecord& ev) { OnFifoMessagesRead(requestId, ev, usedDLQ); }); @@ -657,24 +657,24 @@ void TQueueLeader::OnFifoMessagesRead(const TString& requestId, const TSqsEvents const TValue value(TValue::Create(ev.GetExecutionEngineEvaluatedResponse())); const TValue list(value["result"]); - if (const ui64 movedMessagesCount = value["movedMessagesCount"]) { - ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount); + if (const ui64 movedMessagesCount = value["movedMessagesCount"]) { + ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount); const i64 newMessagesCount = value["newMessagesCount"]; Y_VERIFY(newMessagesCount >= 0); auto& shardInfo = Shards_[0]; shardInfo.MessagesCount = static_cast<ui64>(newMessagesCount); - } - + } + reqInfo.Answer->Messages.resize(list.Size()); for (size_t i = 0; i < list.Size(); ++i) { - const TValue& data = list[i]["SourceDataFieldsRead"]; - const TValue& msg = list[i]["SourceMessageFieldsRead"]; + const TValue& data = list[i]["SourceDataFieldsRead"]; + const TValue& msg = list[i]["SourceMessageFieldsRead"]; const ui64 receiveTimestamp = msg["FirstReceiveTimestamp"]; auto& msgAnswer = reqInfo.Answer->Messages[i]; msgAnswer.FirstReceiveTimestamp = (receiveTimestamp ? TInstant::MilliSeconds(receiveTimestamp) : reqInfo.LockSendTs); - msgAnswer.ReceiveCount = ui32(msg["ReceiveCount"]) + 1; // since the query returns old receive count value + msgAnswer.ReceiveCount = ui32(msg["ReceiveCount"]) + 1; // since the query returns old receive count value msgAnswer.MessageId = data["MessageId"]; msgAnswer.MessageDeduplicationId = data["DedupId"]; msgAnswer.MessageGroupId = msg["GroupId"]; @@ -701,15 +701,15 @@ void TQueueLeader::OnFifoMessagesRead(const TString& requestId, const TSqsEvents } } } else { - const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(ev.GetStatus()); - if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) { - // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info - DlqInfo_.Clear(); - reqInfo.Answer->Failed = false; - reqInfo.Answer->Messages.clear(); - } else { - reqInfo.Answer->Failed = true; - } + const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(ev.GetStatus()); + if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) { + // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info + DlqInfo_.Clear(); + reqInfo.Answer->Failed = false; + reqInfo.Answer->Messages.clear(); + } else { + reqInfo.Answer->Failed = true; + } } Reply(reqInfo); @@ -756,17 +756,17 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o bool deleted = true; bool deadlineChanged = true; const bool exists = (*messageRecord)["Exists"]; - const auto wasDeadLetterValue = (*messageRecord)["IsDeadLetter"]; - const bool wasDeadLetter = wasDeadLetterValue.HaveValue() ? bool(wasDeadLetterValue) : false; - + const auto wasDeadLetterValue = (*messageRecord)["IsDeadLetter"]; + const bool wasDeadLetter = wasDeadLetterValue.HaveValue() ? bool(wasDeadLetterValue) : false; + const bool valid = (*messageRecord)["Valid"]; - if (exists && !wasDeadLetter) { + if (exists && !wasDeadLetter) { const ui64 visibilityDeadlineMs = (*messageRecord)["VisibilityDeadline"]; - const ui32 receiveCount = (*messageRecord)["ReceiveCount"]; + const ui32 receiveCount = (*messageRecord)["ReceiveCount"]; const TInstant visibilityDeadline = TInstant::MilliSeconds(visibilityDeadlineMs); - // Update actual visibility deadline and receive count even if this message won't be given to user in this request. + // Update actual visibility deadline and receive count even if this message won't be given to user in this request. // It prevents such synchronization errors later. - reqInfo.ReceiveCandidates.SetVisibilityDeadlineAndReceiveCount(offset, visibilityDeadline, receiveCount); + reqInfo.ReceiveCandidates.SetVisibilityDeadlineAndReceiveCount(offset, visibilityDeadline, receiveCount); if (valid && reqInfo.ReceiveCandidates.Has(offset)) { // there may be concurrent successful delete message request (purge) reqInfo.Answer->Messages.emplace_back(); @@ -807,8 +807,8 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o deleted = false; // Success, not invalidated } else { RLOG_SQS_REQ_WARN(requestId, "Attempted to receive message that was deleted. Shard: " << reqInfo.GetCurrentShard() << ". Offset: " << offset); - deleted = true; - } + deleted = true; + } } // else there was concurrent delete (purge) by this leader, => OK } const bool invalidated = deleted || deadlineChanged; @@ -819,8 +819,8 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o MarkInflyReloading(reqInfo.GetCurrentShard(), 1, reason); } } else { - reqInfo.LoadError = !ignoreMessageLoadingErrors; - // there may be other successful loads + reqInfo.LoadError = !ignoreMessageLoadingErrors; + // there may be other successful loads } if (reqInfo.LoadAnswersLeft == 0) { @@ -838,21 +838,21 @@ void TQueueLeader::OnLoadStdMessagesBatchExecuted(ui64 shard, ui64 batchId, cons Y_VERIFY(batchIt != batchingState.BatchesExecuting.end()); auto batch = batchIt->second; auto status = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus(reply.GetStatus()); - bool ignoreMessageLoadingErrors = false; + bool ignoreMessageLoadingErrors = false; if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { using NKikimr::NClient::TValue; const TValue value(TValue::Create(reply.GetExecutionEngineEvaluatedResponse())); const TValue list(value["result"]); Y_VERIFY(list.Size() == batch->Size()); - if (const ui64 movedMessagesCount = value["movedMessagesCount"]) { - ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount); + if (const ui64 movedMessagesCount = value["movedMessagesCount"]) { + ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount); const i64 newMessagesCount = value["newMessagesCount"]; Y_VERIFY(newMessagesCount >= 0); shardInfo.MessagesCount = static_cast<ui64>(newMessagesCount); - } - + } + THashMap<ui64, const TLoadBatchEntry*> offset2entry; offset2entry.reserve(batch->Entries.size()); for (const TLoadBatchEntry& entry : batch->Entries) { @@ -867,13 +867,13 @@ void TQueueLeader::OnLoadStdMessagesBatchExecuted(ui64 shard, ui64 batchId, cons OnLoadStdMessageResult(entry->second->RequestId, offset, reply, &msg, ignoreMessageLoadingErrors); } } else { - const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(reply.GetStatus()); - if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) { - // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info - DlqInfo_.Clear(); - ignoreMessageLoadingErrors = true; - } - + const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(reply.GetStatus()); + if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) { + // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info + DlqInfo_.Clear(); + ignoreMessageLoadingErrors = true; + } + const TString* prevRequestId = nullptr; for (size_t i = 0; i < batch->Size(); ++i) { const TLoadBatchEntry& entry = batch->Entries[i]; @@ -1269,10 +1269,10 @@ void TQueueLeader::OnQueueConfiguration(const TSqsEvents::TEvExecuted::TRecord& ShardsCount_ = data["Shards"]; PartitionsCount_ = data["Partitions"]; QueueId_ = data["QueueId"]; - if (data["Version"].HaveValue()) { - QueueVersion_ = ui64(data["Version"]); - } - IsFifoQueue_ = bool(data["FifoQueue"]); + if (data["Version"].HaveValue()) { + QueueVersion_ = ui64(data["Version"]); + } + IsFifoQueue_ = bool(data["FifoQueue"]); Shards_.resize(ShardsCount_); const auto& cfg = Cfg(); if (IsFifoQueue_) { @@ -1362,8 +1362,8 @@ void TQueueLeader::AskQueueAttributes() { .Queue(QueueName_) .QueueLeader(SelfId()) .QueryId(INTERNAL_GET_QUEUE_ATTRIBUTES_ID) - .QueueVersion(QueueVersion_) - .Fifo(IsFifoQueue_) + .QueueVersion(QueueVersion_) + .Fifo(IsFifoQueue_) .RetryOnTimeout() .OnExecuted([this](const TSqsEvents::TEvExecuted::TRecord& ev) { OnQueueAttributes(ev); }) .Counters(Counters_) @@ -1392,22 +1392,22 @@ void TQueueLeader::OnQueueAttributes(const TSqsEvents::TEvExecuted::TRecord& ev) Counters_->ShowDetailedCounters(TInstant::MilliSeconds(ms)); } - // update dead letter queue info - const auto& dlqNameVal(attrs["DlqName"]); - const auto& maxReceiveCountVal(attrs["MaxReceiveCount"]); - if (dlqNameVal.HaveValue() && maxReceiveCountVal.HaveValue()) { - TTargetDlqInfo info; - info.DlqName = TString(dlqNameVal); - info.MaxReceiveCount = ui64(maxReceiveCountVal); - if (info.DlqName && info.MaxReceiveCount) { - DlqInfo_ = info; - // now we have to discover queue id and version + // update dead letter queue info + const auto& dlqNameVal(attrs["DlqName"]); + const auto& maxReceiveCountVal(attrs["MaxReceiveCount"]); + if (dlqNameVal.HaveValue() && maxReceiveCountVal.HaveValue()) { + TTargetDlqInfo info; + info.DlqName = TString(dlqNameVal); + info.MaxReceiveCount = ui64(maxReceiveCountVal); + if (info.DlqName && info.MaxReceiveCount) { + DlqInfo_ = info; + // now we have to discover queue id and version Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueId("DLQ", UserName_, info.DlqName, FolderId_)); - } else { - DlqInfo_.Clear(); - } - } - + } else { + DlqInfo_.Clear(); + } + } + QueueAttributes_ = attributes; AttributesUpdateTime_ = TActivationContext::Now(); for (auto& req : GetConfigurationRequests_) { @@ -1423,26 +1423,26 @@ void TQueueLeader::OnQueueAttributes(const TSqsEvents::TEvExecuted::TRecord& ev) } void TQueueLeader::HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) { - if (!DlqInfo_) { - return; - } - - if (ev->Get()->Failed) { + if (!DlqInfo_) { + return; + } + + if (ev->Get()->Failed) { LOG_SQS_DEBUG("Dlq discovering failed"); - } else { - if (ev->Get()->Exists) { - DlqInfo_->QueueId = ev->Get()->QueueId; - DlqInfo_->QueueVersion = ev->Get()->Version; - DlqInfo_->ShardsCount = ev->Get()->ShardsCount; - + } else { + if (ev->Get()->Exists) { + DlqInfo_->QueueId = ev->Get()->QueueId; + DlqInfo_->QueueVersion = ev->Get()->Version; + DlqInfo_->ShardsCount = ev->Get()->ShardsCount; + LOG_SQS_DEBUG("Discovered DLQ: name: " << DlqInfo_->DlqName << ", maxReceiveCount: " << DlqInfo_->MaxReceiveCount << ", queueId: " << DlqInfo_->QueueId << ", version: " << DlqInfo_->QueueVersion << ", shards count: " << DlqInfo_->ShardsCount); - return; - } - } - - DlqInfo_.Clear(); // something is off -} - + return; + } + } + + DlqInfo_.Clear(); // something is off +} + void TQueueLeader::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { ev->Get()->Call(); } @@ -1457,11 +1457,11 @@ void TQueueLeader::StartGatheringMetrics() { if (IsDlqQueue_) { LOG_SQS_INFO("Stopped periodic message counting for queue " << TLogQueueName(UserName_, QueueName_) << ". Latest dlq notification was at " << LatestDlqNotificationTs_); - } - + } + IsDlqQueue_ = false; - } - + } + for (ui64 shard = 0; shard < ShardsCount_; ++shard) { if (IsFifoQueue_ || IsDlqQueue_) { RequestMessagesCountMetrics(shard); @@ -1746,7 +1746,7 @@ void TQueueLeader::OnInflyLoaded(ui64 shard, const TSqsEvents::TEvExecuted::TRec DelayStatistics_.AddDelayedMessage(delayDeadline, now); } const ui64 offset = message["Offset"]; - const ui32 receiveCount = message["ReceiveCount"]; + const ui32 receiveCount = message["ReceiveCount"]; const TInstant maxVisibilityDeadline = TInstant::MilliSeconds(Max(visibilityDeadlineMs, delayDeadlineMs)); LOG_SQS_TRACE("Adding message to infly struct for shard " << TLogQueueName(UserName_, QueueName_, shard) << ": { Offset: " << offset << ", VisibilityDeadline: " << maxVisibilityDeadline << ", ReceiveCount: " << receiveCount << " }"); shardInfo.Infly->Add(MakeHolder<TInflyMessage>(offset, message["RandomId"], maxVisibilityDeadline, receiveCount)); @@ -1866,7 +1866,7 @@ void TQueueLeader::OnAddedMessagesToInfly(ui64 shard, const TSqsEvents::TEvExecu const ui64 delayDeadlineMs = delayDeadlineValue.HaveValue() ? ui64(delayDeadlineValue) : 0; const TInstant delayDeadline = TInstant::MilliSeconds(delayDeadlineMs); const ui64 offset = message["Offset"]; - const ui32 receiveCount = 0; // as in transaction + const ui32 receiveCount = 0; // as in transaction LOG_SQS_TRACE("Adding message to infly struct for shard " << TLogQueueName(UserName_, QueueName_, shard) << ": { Offset: " << offset << ", DelayDeadline: " << delayDeadline << ", ReceiveCount: " << receiveCount << " }"); shardInfo.Infly->Add(MakeHolder<TInflyMessage>(offset, message["RandomId"], delayDeadline, receiveCount)); } @@ -2118,17 +2118,17 @@ void TQueueLeader::HandleGetRuntimeQueueAttributesWhileWorking(TSqsEvents::TEvGe void TQueueLeader::HandleDeadLetterQueueNotification(TSqsEvents::TEvDeadLetterQueueNotification::TPtr&) { LatestDlqNotificationTs_ = TActivationContext::Now(); - + if (!IsFifoQueue_ && !IsDlqQueue_) { - // we need to start the process only once + // we need to start the process only once IsDlqQueue_ = true; LOG_SQS_INFO("Started periodic message counting for queue " << TLogQueueName(UserName_, QueueName_) << ". Latest dlq notification was at " << LatestDlqNotificationTs_); - + StartGatheringMetrics(); - } -} - + } +} + void TQueueLeader::ProcessGetRuntimeQueueAttributes(TGetRuntimeQueueAttributesRequestProcessing& reqInfo) { if (reqInfo.ShardProcessFlags.empty()) { Y_VERIFY(ShardsCount_ > 0); @@ -2521,7 +2521,7 @@ void TQueueLeader::TLoadBatchingState::AddRequest(TReceiveMessageBatchRequestPro void TQueueLeader::TLoadBatch::Execute(TQueueLeader* leader) { RLOG_SQS_DEBUG(TLogQueueName(leader->UserName_, leader->QueueName_, Shard) << " Executing load batch. BatchId: " << BatchId << ". Size: " << Size()); - + TExecutorBuilder builder(SelfId(), RequestId_); const auto now = TActivationContext::Now(); builder @@ -2538,63 +2538,63 @@ void TQueueLeader::TLoadBatch::Execute(TQueueLeader* leader) { .Uint64("READ_ID", RandomNumber<ui64>()) .Uint64("SHARD", Shard); - ui32 maxReceiveCount = 0; // not set + ui32 maxReceiveCount = 0; // not set if (Cfg().GetEnableDeadLetterQueues() && leader->DlqInfo_) { const auto& dlqInfo(*leader->DlqInfo_); - if (dlqInfo.DlqName && dlqInfo.QueueId) { - // dlq is set and resolved - maxReceiveCount = dlqInfo.MaxReceiveCount; - } - } - + if (dlqInfo.DlqName && dlqInfo.QueueId) { + // dlq is set and resolved + maxReceiveCount = dlqInfo.MaxReceiveCount; + } + } + NClient::TWriteValue params = builder.ParamsValue(); const TString* prevRequestId = nullptr; - size_t deadLettersCounter = 0; - THashSet<ui64> offsets; // check for duplicates + size_t deadLettersCounter = 0; + THashSet<ui64> offsets; // check for duplicates for (const TLoadBatchEntry& entry : Entries) { - Y_VERIFY(offsets.insert(entry.Offset).second); - + Y_VERIFY(offsets.insert(entry.Offset).second); + auto item = params["KEYS"].AddListItem(); item["RandomId"] = entry.RandomId; item["Offset"] = entry.Offset; item["CurrentVisibilityDeadline"] = ui64(entry.CurrentVisibilityDeadline.MilliSeconds()); item["VisibilityDeadline"] = ui64((now + entry.VisibilityTimeout).MilliSeconds()); - if (maxReceiveCount && entry.ReceiveCount >= maxReceiveCount) { - item["DlqIndex"] = ui64(deadLettersCounter); - ++deadLettersCounter; - item["IsDeadLetter"] = true; - } else { - item["DlqIndex"] = ui64(0); - item["IsDeadLetter"] = false; - } - + if (maxReceiveCount && entry.ReceiveCount >= maxReceiveCount) { + item["DlqIndex"] = ui64(deadLettersCounter); + ++deadLettersCounter; + item["IsDeadLetter"] = true; + } else { + item["DlqIndex"] = ui64(0); + item["IsDeadLetter"] = false; + } + if (!prevRequestId || *prevRequestId != entry.RequestId) { prevRequestId = &entry.RequestId; RLOG_SQS_REQ_DEBUG(entry.RequestId, "Send batch transaction to database. BatchId: " << BatchId); } } - if (deadLettersCounter) { - // perform heavy read and move transaction (DLQ) + if (deadLettersCounter) { + // perform heavy read and move transaction (DLQ) Y_VERIFY(leader->DlqInfo_); const auto& dlqInfo(*leader->DlqInfo_); - + const TQueuePath currentQueuePath = { Cfg().GetRoot(), leader->UserName_, leader->QueueName_, leader->QueueVersion_ }; const TQueuePath deadLetterQueuePath = { Cfg().GetRoot(), leader->UserName_, dlqInfo.QueueId, dlqInfo.QueueVersion }; - - const TString transactionText = Sprintf(GetStdQueryById(LOAD_OR_REDRIVE_MESSAGE_ID), - currentQueuePath.GetVersionedQueuePath().c_str(), Shard, - deadLetterQueuePath.GetVersionedQueuePath().c_str(), Shard % dlqInfo.ShardsCount); // TODO: shard inserts aren't balanced - - builder.Text(transactionText) - .Params() - .Uint64("DEAD_LETTERS_COUNT", deadLettersCounter); - } else { - // perform simple read transaction - builder.QueryId(LOAD_MESSAGES_ID); - } - - const bool usedDLQ = deadLettersCounter; + + const TString transactionText = Sprintf(GetStdQueryById(LOAD_OR_REDRIVE_MESSAGE_ID), + currentQueuePath.GetVersionedQueuePath().c_str(), Shard, + deadLetterQueuePath.GetVersionedQueuePath().c_str(), Shard % dlqInfo.ShardsCount); // TODO: shard inserts aren't balanced + + builder.Text(transactionText) + .Params() + .Uint64("DEAD_LETTERS_COUNT", deadLettersCounter); + } else { + // perform simple read transaction + builder.QueryId(LOAD_MESSAGES_ID); + } + + const bool usedDLQ = deadLettersCounter; builder.OnExecuted([leader, shard = Shard, batchId = BatchId, usedDLQ] (const TSqsEvents::TEvExecuted::TRecord& ev) { leader->OnLoadStdMessagesBatchExecuted(shard, batchId, usedDLQ, ev); }); diff --git a/ydb/core/ymq/actor/queue_leader.h b/ydb/core/ymq/actor/queue_leader.h index 3504e4ddb1..adf6edadef 100644 --- a/ydb/core/ymq/actor/queue_leader.h +++ b/ydb/core/ymq/actor/queue_leader.h @@ -153,13 +153,13 @@ private: // const info TString UserName_; TString QueueName_; - TString FolderId_; + TString FolderId_; TString RootUrl_; ui64 ShardsCount_ = 0; ui64 PartitionsCount_ = 0; - bool IsFifoQueue_ = false; + bool IsFifoQueue_ = false; TString QueueId_; - ui64 QueueVersion_ = 0; + ui64 QueueVersion_ = 0; TActorId SchemeCache_; TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_; TLocalRateLimiterResource SendMessageQuoterResource_; @@ -177,20 +177,20 @@ private: TIntrusivePtr<TUserCounters> UserCounters_; size_t MetricsQueriesInfly_ = 0; - // dead letter queue params - struct TTargetDlqInfo { - // from attributes - TString DlqName; - ui64 MaxReceiveCount = 0; - // discovered via service - TString QueueId; // unique name or resource id in cloud - ui64 QueueVersion = 0; - ui64 ShardsCount = 0; - }; - TMaybe<TTargetDlqInfo> DlqInfo_; + // dead letter queue params + struct TTargetDlqInfo { + // from attributes + TString DlqName; + ui64 MaxReceiveCount = 0; + // discovered via service + TString QueueId; // unique name or resource id in cloud + ui64 QueueVersion = 0; + ui64 ShardsCount = 0; + }; + TMaybe<TTargetDlqInfo> DlqInfo_; bool IsDlqQueue_ = false; - TInstant LatestDlqNotificationTs_ = TInstant::Zero(); - + TInstant LatestDlqNotificationTs_ = TInstant::Zero(); + // shards enum class EQueryState { Empty, @@ -294,7 +294,7 @@ private: : RequestId(requestId) , RandomId(msg->GetRandomId()) , Offset(msg->GetOffset()) - , ReceiveCount(msg->GetReceiveCount()) + , ReceiveCount(msg->GetReceiveCount()) , CurrentVisibilityDeadline(msg->GetVisibilityDeadline()) , VisibilityTimeout(visibilityTimeout) { @@ -303,7 +303,7 @@ private: TString RequestId; ui64 RandomId = 0; ui64 Offset = 0; - ui32 ReceiveCount = 0; + ui32 ReceiveCount = 0; TInstant CurrentVisibilityDeadline; TDuration VisibilityTimeout; }; @@ -455,7 +455,7 @@ private: struct TLockedFifoMessage { ui64 RandomId = 0; ui64 Offset = 0; - TString GroupId; + TString GroupId; }; std::vector<TLockedFifoMessage> LockedFifoMessages; }; diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp index 91d3c5d3df..1cde9b0fe3 100644 --- a/ydb/core/ymq/actor/queue_schema.cpp +++ b/ydb/core/ymq/actor/queue_schema.cpp @@ -1,22 +1,22 @@ #include "cfg.h" -#include "executor.h" -#include "log.h" -#include "params.h" -#include "queue_schema.h" -#include "serviceid.h" - +#include "executor.h" +#include "log.h" +#include "params.h" +#include "queue_schema.h" +#include "serviceid.h" + #include <ydb/core/ymq/base/limits.h> #include <ydb/core/ymq/queues/fifo/schema.h> #include <ydb/core/ymq/queues/std/schema.h> - + #include <util/digest/city.h> #include <util/generic/utility.h> #include <util/string/join.h> -using NKikimr::NClient::TValue; - +using NKikimr::NClient::TValue; + namespace NKikimr::NSQS { - + constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO"; constexpr TStringBuf STD_TABLES_DIR = ".STD"; @@ -25,145 +25,145 @@ ui64 GetHash(TArgs&&... args) { return CityHash64(Join("#", args...)); } -static bool IsGoodStatusCode(ui32 code) { - switch (NTxProxy::TResultStatus::EStatus(code)) { - case NTxProxy::TResultStatus::EStatus::ExecComplete: - case NTxProxy::TResultStatus::EStatus::ExecAlready: - return true; - default: - return false; - } -} - -TCreateQueueSchemaActorV2::TCreateQueueSchemaActorV2(const TQueuePath& path, - const TCreateQueueRequest& req, +static bool IsGoodStatusCode(ui32 code) { + switch (NTxProxy::TResultStatus::EStatus(code)) { + case NTxProxy::TResultStatus::EStatus::ExecComplete: + case NTxProxy::TResultStatus::EStatus::ExecAlready: + return true; + default: + return false; + } +} + +TCreateQueueSchemaActorV2::TCreateQueueSchemaActorV2(const TQueuePath& path, + const TCreateQueueRequest& req, const TActorId& sender, - const TString& requestId, - const TString& customQueueName, - const TString& folderId, - const bool isCloudMode, - const bool enableQueueAttributesValidation, + const TString& requestId, + const TString& customQueueName, + const TString& folderId, + const bool isCloudMode, + const bool enableQueueAttributesValidation, TIntrusivePtr<TUserCounters> userCounters, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResources) - : QueuePath_(path) - , Request_(req) - , Sender_(sender) - , CustomQueueName_(customQueueName) - , FolderId_(folderId) - , RequestId_(requestId) - , GeneratedQueueId_(CreateGuidAsString()) - , QueueCreationTimestamp_(TInstant::Now()) - , IsCloudMode_(isCloudMode) - , EnableQueueAttributesValidation_(enableQueueAttributesValidation) + : QueuePath_(path) + , Request_(req) + , Sender_(sender) + , CustomQueueName_(customQueueName) + , FolderId_(folderId) + , RequestId_(requestId) + , GeneratedQueueId_(CreateGuidAsString()) + , QueueCreationTimestamp_(TInstant::Now()) + , IsCloudMode_(isCloudMode) + , EnableQueueAttributesValidation_(enableQueueAttributesValidation) , UserCounters_(std::move(userCounters)) , QuoterResources_(std::move(quoterResources)) -{ - IsFifo_ = AsciiHasSuffixIgnoreCase(IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName, ".fifo"); - - if (IsFifo_) { - RequiredShardsCount_ = 1; - RequiredTables_ = GetFifoTables(); - } else { - RequiredShardsCount_ = Request_.GetShards(); +{ + IsFifo_ = AsciiHasSuffixIgnoreCase(IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName, ".fifo"); + + if (IsFifo_) { + RequiredShardsCount_ = 1; + RequiredTables_ = GetFifoTables(); + } else { + RequiredShardsCount_ = Request_.GetShards(); RequiredTables_ = GetStandardTables(RequiredShardsCount_, req.GetPartitions(), req.GetEnableAutosplit(), req.GetSizeToSplit()); - } -} - -TCreateQueueSchemaActorV2::~TCreateQueueSchemaActorV2() = default; - + } +} + +TCreateQueueSchemaActorV2::~TCreateQueueSchemaActorV2() = default; + static THolder<TSqsEvents::TEvQueueCreated> MakeErrorResponse(const TErrorClass& errorClass) { - auto resp = MakeHolder<TSqsEvents::TEvQueueCreated>(); - resp->Success = false; - resp->State = EQueueState::Active; + auto resp = MakeHolder<TSqsEvents::TEvQueueCreated>(); + resp->Success = false; + resp->State = EQueueState::Active; resp->ErrorClass = &errorClass; - - return resp; -} - -template<typename TMaybeAttribute, typename TValueType> -static void SetDefaultIfMissing(TMaybeAttribute& attribute, const TValueType& defaultVal) { - if (!attribute) { - attribute = defaultVal; - } -} - -static ui64 SecondsToMs(ui64 seconds) { - return TDuration::Seconds(seconds).MilliSeconds(); -} - -void TCreateQueueSchemaActorV2::InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config) { - // https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html - - SetDefaultIfMissing(ValidatedAttributes_.DelaySeconds, 0); - SetDefaultIfMissing(ValidatedAttributes_.MaximumMessageSize, 256 * 1024); // 256 KB - SetDefaultIfMissing(ValidatedAttributes_.MessageRetentionPeriod, - Max(TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), TDuration::Days(4).Seconds())); - SetDefaultIfMissing(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds, 0); // seconds - SetDefaultIfMissing(ValidatedAttributes_.VisibilityTimeout, 30); // seconds - SetDefaultIfMissing(ValidatedAttributes_.ContentBasedDeduplication, false); // bool - - // RedrivePolicy could be unspecified -} - + + return resp; +} + +template<typename TMaybeAttribute, typename TValueType> +static void SetDefaultIfMissing(TMaybeAttribute& attribute, const TValueType& defaultVal) { + if (!attribute) { + attribute = defaultVal; + } +} + +static ui64 SecondsToMs(ui64 seconds) { + return TDuration::Seconds(seconds).MilliSeconds(); +} + +void TCreateQueueSchemaActorV2::InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config) { + // https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html + + SetDefaultIfMissing(ValidatedAttributes_.DelaySeconds, 0); + SetDefaultIfMissing(ValidatedAttributes_.MaximumMessageSize, 256 * 1024); // 256 KB + SetDefaultIfMissing(ValidatedAttributes_.MessageRetentionPeriod, + Max(TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), TDuration::Days(4).Seconds())); + SetDefaultIfMissing(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds, 0); // seconds + SetDefaultIfMissing(ValidatedAttributes_.VisibilityTimeout, 30); // seconds + SetDefaultIfMissing(ValidatedAttributes_.ContentBasedDeduplication, false); // bool + + // RedrivePolicy could be unspecified +} + void TCreateQueueSchemaActorV2::Bootstrap() { - Become(&TCreateQueueSchemaActorV2::Preamble); - - THashMap<TString, TString> attributes; - for (const auto& attr : Request_.attributes()) { - attributes[attr.GetName()] = attr.GetValue(); - } - - const bool clampValues = !EnableQueueAttributesValidation_; + Become(&TCreateQueueSchemaActorV2::Preamble); + + THashMap<TString, TString> attributes; + for (const auto& attr : Request_.attributes()) { + attributes[attr.GetName()] = attr.GetValue(); + } + + const bool clampValues = !EnableQueueAttributesValidation_; ValidatedAttributes_ = TQueueAttributes::FromAttributesAndConfig(attributes, Cfg(), IsFifo_, clampValues); - - if (!ValidatedAttributes_.Validate()) { + + if (!ValidatedAttributes_.Validate()) { auto resp = MakeErrorResponse(NErrors::VALIDATION_ERROR); - resp->Error = ValidatedAttributes_.ErrorText; - + resp->Error = ValidatedAttributes_.ErrorText; + Send(Sender_, std::move(resp)); PassAway(); - return; - } - - if (ValidatedAttributes_.HasClampedAttributes()) { + return; + } + + if (ValidatedAttributes_.HasClampedAttributes()) { RLOG_SQS_WARN("Clamped some queue attribute values for account " << QueuePath_.UserName << " and queue name " << QueuePath_.QueueName); - } - + } + InitMissingQueueAttributes(Cfg()); - - if (ValidatedAttributes_.RedrivePolicy.TargetQueueName) { - const TString createdQueueName = IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName; + + if (ValidatedAttributes_.RedrivePolicy.TargetQueueName) { + const TString createdQueueName = IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName; auto resp = MakeErrorResponse(NErrors::VALIDATION_ERROR); - if (ValidatedAttributes_.RedrivePolicy.TargetQueueName->empty()) { + if (ValidatedAttributes_.RedrivePolicy.TargetQueueName->empty()) { resp->Error = "Empty target dead letter queue name."; - } else if (*ValidatedAttributes_.RedrivePolicy.TargetQueueName == createdQueueName) { - resp->Error = "Using the queue itself as a dead letter queue is not allowed."; - } else { + } else if (*ValidatedAttributes_.RedrivePolicy.TargetQueueName == createdQueueName) { + resp->Error = "Using the queue itself as a dead letter queue is not allowed."; + } else { Send(MakeSqsServiceID(SelfId().NodeId()), - new TSqsEvents::TEvGetQueueId(RequestId_, QueuePath_.UserName, - *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_)); - return; - } - + new TSqsEvents::TEvGetQueueId(RequestId_, QueuePath_.UserName, + *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_)); + return; + } + Send(Sender_, std::move(resp)); PassAway(); - - return; - } - + + return; + } + RequestQueueParams(); -} - -static const char* const ReadQueueParamsQueryCloud = R"__( - ( - (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String))) - (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) +} + +static const char* const ReadQueueParamsQueryCloud = R"__( + ( + (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String))) + (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) (let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - + (let queuesTable '%1$s/.Queues) (let settingsTable '%1$s/.Settings) - + (let maxQueuesCountSettingRow '( '('Account userName) '('Name (Utf8String '"MaxQueuesCount")))) @@ -174,49 +174,49 @@ static const char* const ReadQueueParamsQueryCloud = R"__( (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) - (let queues + '('QueueName (Utf8String '"") (Void)))) + (let queues (Member (SelectRange queuesTable queuesRange '('QueueName 'CustomQueueName 'Version 'FolderId) '()) 'List)) - (let overLimit + (let overLimit (LessOrEqual maxQueuesCountSetting (Length queues))) - - (let existingQueuesWithSameNameAndFolderId - (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (Equal (Member item 'CustomQueueName) customName) - (Equal (Member item 'FolderId) folderId)) - (Bool 'false)))))) - ) - ) - - (let queueExists (NotEqual (Uint64 '0) (Length existingQueuesWithSameNameAndFolderId))) - (let currentVersion (Coalesce - (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version) - (Uint64 '0) - )) - (let existingResourceId (Coalesce - (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName) - (Utf8String '"") - )) - - (return (AsList - (SetResult 'exists queueExists) - (SetResult 'resourceId existingResourceId) - (SetResult 'overLimit overLimit) - (SetResult 'version currentVersion))) - ) -)__"; - -static const char* const ReadQueueParamsQueryYandex = R"__( - ( - (let name (Parameter 'NAME (DataType 'Utf8String))) + + (let existingQueuesWithSameNameAndFolderId + (Filter queues (lambda '(item) (block '( + (return (Coalesce + (And + (Equal (Member item 'CustomQueueName) customName) + (Equal (Member item 'FolderId) folderId)) + (Bool 'false)))))) + ) + ) + + (let queueExists (NotEqual (Uint64 '0) (Length existingQueuesWithSameNameAndFolderId))) + (let currentVersion (Coalesce + (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version) + (Uint64 '0) + )) + (let existingResourceId (Coalesce + (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName) + (Utf8String '"") + )) + + (return (AsList + (SetResult 'exists queueExists) + (SetResult 'resourceId existingResourceId) + (SetResult 'overLimit overLimit) + (SetResult 'version currentVersion))) + ) +)__"; + +static const char* const ReadQueueParamsQueryYandex = R"__( + ( + (let name (Parameter 'NAME (DataType 'Utf8String))) (let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - + (let queuesTable '%1$s/.Queues) (let settingsTable '%1$s/.Settings) - + (let maxQueuesCountSettingRow '( '('Account userName) '('Name (Utf8String '"MaxQueuesCount")))) @@ -227,131 +227,131 @@ static const char* const ReadQueueParamsQueryYandex = R"__( (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) - (let queues + '('QueueName (Utf8String '"") (Void)))) + (let queues (Member (SelectRange queuesTable queuesRange '('QueueState) '()) 'List)) - (let overLimit + (let overLimit (LessOrEqual maxQueuesCountSetting (Length queues))) - + (let queuesRow '( '('Account userName) - '('QueueName name))) + '('QueueName name))) (let queuesSelect '( - 'QueueState - 'Version)) + 'QueueState + 'Version)) (let queuesRead (SelectRow queuesTable queuesRow queuesSelect)) - - (let queueExists - (Coalesce - (Or + + (let queueExists + (Coalesce + (Or (Equal (Uint64 '1) (Member queuesRead 'QueueState)) (Equal (Uint64 '3) (Member queuesRead 'QueueState)) - ) - (Bool 'false))) - - (let currentVersion - (Coalesce + ) + (Bool 'false))) + + (let currentVersion + (Coalesce (Member queuesRead 'Version) - (Uint64 '0) - ) - ) - - (return (AsList - (SetResult 'exists queueExists) - (SetResult 'overLimit overLimit) - (SetResult 'version currentVersion))) - ) -)__"; - + (Uint64 '0) + ) + ) + + (return (AsList + (SetResult 'exists queueExists) + (SetResult 'overLimit overLimit) + (SetResult 'version currentVersion))) + ) +)__"; + void TCreateQueueSchemaActorV2::RequestQueueParams() { - if (IsCloudMode_) { + if (IsCloudMode_) { auto ev = MakeExecuteEvent(Sprintf(ReadQueueParamsQueryCloud, Cfg().GetRoot().c_str())); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - TParameters(trans->MutableParams()->MutableProto()) - .Utf8("CUSTOMNAME", CustomQueueName_) + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + TParameters(trans->MutableParams()->MutableProto()) + .Utf8("CUSTOMNAME", CustomQueueName_) .Utf8("FOLDERID", FolderId_) .Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount()) .Utf8("USER_NAME", QueuePath_.UserName); - + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); - } else { + } else { auto ev = MakeExecuteEvent(Sprintf(ReadQueueParamsQueryYandex, Cfg().GetRoot().c_str())); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - TParameters(trans->MutableParams()->MutableProto()) + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + TParameters(trans->MutableParams()->MutableProto()) .Utf8("NAME", QueuePath_.QueueName) .Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount()) .Utf8("USER_NAME", QueuePath_.UserName); - + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); - } -} - + } +} + STATEFN(TCreateQueueSchemaActorV2::Preamble) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvQueueId, HandleQueueId); hFunc(TSqsEvents::TEvExecuted, OnReadQueueParams); hFunc(TEvQuota::TEvClearance, OnCreateQueueQuota); hFunc(TSqsEvents::TEvAtomicCounterIncrementResult, OnAtomicCounterIncrement); cFunc(TEvPoisonPill::EventType, PassAway); - } -} - + } +} + void TCreateQueueSchemaActorV2::HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) { THolder<TSqsEvents::TEvQueueCreated> resp; - if (ev->Get()->Failed) { + if (ev->Get()->Failed) { RLOG_SQS_WARN("Get queue id failed"); resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - } else if (!ev->Get()->Exists) { + } else if (!ev->Get()->Exists) { resp = MakeErrorResponse(NErrors::VALIDATION_ERROR); resp->Error = "Target DLQ does not exist"; - } else { + } else { RequestQueueParams(); - return; - } - + return; + } + Y_VERIFY(resp); Send(Sender_, std::move(resp)); PassAway(); -} - +} + void TCreateQueueSchemaActorV2::OnReadQueueParams(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); - + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); + THolder<TSqsEvents::TEvQueueCreated> resp; - - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - if (bool(val["exists"])) { - if (IsCloudMode_) { - ExistingQueueResourceId_ = TString(val["resourceId"]); - } - const ui64 currentVersion = ui64(val["version"]); + + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + if (bool(val["exists"])) { + if (IsCloudMode_) { + ExistingQueueResourceId_ = TString(val["resourceId"]); + } + const ui64 currentVersion = ui64(val["version"]); MatchQueueAttributes(currentVersion); - return; - } else { - if (bool(val["overLimit"])) { + return; + } else { + if (bool(val["overLimit"])) { resp = MakeErrorResponse(NErrors::OVER_LIMIT); resp->Error = "Too many queues."; - } else { + } else { if (Cfg().GetQuotingConfig().GetEnableQuoting() && QuoterResources_) { RequestCreateQueueQuota(); } else { RunAtomicCounterIncrement(); } - return; - } - } - } else { + return; + } + } + } else { resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); resp->Error = "Failed to read queue params."; - } - + } + Y_VERIFY(resp); Send(Sender_, std::move(resp)); PassAway(); -} - +} + void TCreateQueueSchemaActorV2::RequestCreateQueueQuota() { TDuration deadline = TDuration::Max(); const auto& quotingConfig = Cfg().GetQuotingConfig(); @@ -393,24 +393,24 @@ void TCreateQueueSchemaActorV2::RunAtomicCounterIncrement() { } void TCreateQueueSchemaActorV2::OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev) { - auto event = ev->Get(); - if (event->Success) { - Become(&TCreateQueueSchemaActorV2::CreateComponentsState); - Version_ = event->NewValue; - VersionName_ = "v" + ToString(Version_); // add "v" prefix to provide the difference with deprecated version shards - VersionedQueueFullPath_ = TString::Join(QueuePath_.GetQueuePath(), '/', VersionName_); + auto event = ev->Get(); + if (event->Success) { + Become(&TCreateQueueSchemaActorV2::CreateComponentsState); + Version_ = event->NewValue; + VersionName_ = "v" + ToString(Version_); // add "v" prefix to provide the difference with deprecated version shards + VersionedQueueFullPath_ = TString::Join(QueuePath_.GetQueuePath(), '/', VersionName_); CreateComponents(); - return; - } else { + return; + } else { auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); resp->Error = "Failed to create unique id."; - resp->State = EQueueState::Creating; + resp->State = EQueueState::Creating; Send(Sender_, std::move(resp)); - } - + } + PassAway(); -} - +} + static const char* const GetTablesFormatQuery = R"__( ( (let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32))) @@ -446,126 +446,126 @@ void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accou } void TCreateQueueSchemaActorV2::RegisterMakeDirActor(const TString& workingDir, const TString& dirName) { - auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); - - trans->SetWorkingDir(workingDir); + auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); + + trans->SetWorkingDir(workingDir); trans->SetOperationType(NKikimrSchemeOp::ESchemeOpMkDir); - trans->MutableMkDir()->SetName(dirName); - + trans->MutableMkDir()->SetName(dirName); + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); -} - +} + void TCreateQueueSchemaActorV2::RequestLeaderTabletId() { RLOG_SQS_TRACE("Requesting leader tablet id for path id " << TableWithLeaderPathId_.second); - THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate()); + THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate()); request->Record.MutableDescribePath()->SetSchemeshardId(TableWithLeaderPathId_.first); request->Record.MutableDescribePath()->SetPathId(TableWithLeaderPathId_.second); Send(MakeTxProxyID(), std::move(request)); -} - +} + void TCreateQueueSchemaActorV2::CreateComponents() { - switch (CurrentCreationStep_) { + switch (CurrentCreationStep_) { case ECreateComponentsStep::GetTablesFormatSetting: { RequestTablesFormatSettings(QueuePath_.UserName); break; } - case ECreateComponentsStep::MakeQueueDir: { + case ECreateComponentsStep::MakeQueueDir: { RegisterMakeDirActor(QueuePath_.GetUserPath(), QueuePath_.QueueName); - break; - } - case ECreateComponentsStep::MakeQueueVersionDir: { + break; + } + case ECreateComponentsStep::MakeQueueVersionDir: { RegisterMakeDirActor(QueuePath_.GetQueuePath(), VersionName_); - break; - } - case ECreateComponentsStep::MakeShards: { - for (ui64 shardIdx = 0; shardIdx < RequiredShardsCount_; ++shardIdx) { + break; + } + case ECreateComponentsStep::MakeShards: { + for (ui64 shardIdx = 0; shardIdx < RequiredShardsCount_; ++shardIdx) { RegisterMakeDirActor(VersionedQueueFullPath_, ToString(shardIdx)); - } - break; - } - case ECreateComponentsStep::MakeTables: { - for (const TTable& table : RequiredTables_) { - auto ev = MakeCreateTableEvent(VersionedQueueFullPath_, table, RequiredShardsCount_); - auto* cmd = ev->Record.MutableTransaction()->MutableModifyScheme()->MutableCreateTable(); - cmd->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(Request_.GetEnableOutOfOrderTransactionsExecution()); - + } + break; + } + case ECreateComponentsStep::MakeTables: { + for (const TTable& table : RequiredTables_) { + auto ev = MakeCreateTableEvent(VersionedQueueFullPath_, table, RequiredShardsCount_); + auto* cmd = ev->Record.MutableTransaction()->MutableModifyScheme()->MutableCreateTable(); + cmd->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(Request_.GetEnableOutOfOrderTransactionsExecution()); + const TActorId actorId = Register(new TMiniKqlExecutionActor( SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); - + if (table.HasLeaderTablet && !CreateTableWithLeaderTabletActorId_) { CreateTableWithLeaderTabletActorId_ = actorId; - } - } - - break; - } + } + } + + break; + } case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { SendDescribeTable(); break; } case ECreateComponentsStep::DiscoverLeaderTabletId: { RequestLeaderTabletId(); - break; - } + break; + } case ECreateComponentsStep::AddQuoterResource: { AddRPSQuota(); break; } - } -} - + } +} + STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, OnExecuted); hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, OnDescribeSchemeResult); hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription); hFunc(NKesus::TEvKesus::TEvAddQuoterResourceResult, HandleAddQuoterResource); cFunc(TEvPoisonPill::EventType, PassAway); - } -} - + } +} + void TCreateQueueSchemaActorV2::Step() { RLOG_SQS_TRACE("Next step. Step: " << (int)CurrentCreationStep_); - switch (CurrentCreationStep_) { + switch (CurrentCreationStep_) { case ECreateComponentsStep::GetTablesFormatSetting: { CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir; break; } - case ECreateComponentsStep::MakeQueueDir: { + case ECreateComponentsStep::MakeQueueDir: { if (TablesFormat_ == 0) { CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir; } else { CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId; } - break; - } - case ECreateComponentsStep::MakeQueueVersionDir: { - if (IsFifo_) { - CurrentCreationStep_ = ECreateComponentsStep::MakeTables; - } else { - CurrentCreationStep_ = ECreateComponentsStep::MakeShards; - } - break; - } - case ECreateComponentsStep::MakeShards: { - if (++CreatedShardsCount_ != RequiredShardsCount_) { - return; // do not progress - } - - CurrentCreationStep_ = ECreateComponentsStep::MakeTables; - break; - } - - case ECreateComponentsStep::MakeTables: { - if (++CreatedTablesCount_ != RequiredTables_.size()) { - return; // do not progress - } - + break; + } + case ECreateComponentsStep::MakeQueueVersionDir: { + if (IsFifo_) { + CurrentCreationStep_ = ECreateComponentsStep::MakeTables; + } else { + CurrentCreationStep_ = ECreateComponentsStep::MakeShards; + } + break; + } + case ECreateComponentsStep::MakeShards: { + if (++CreatedShardsCount_ != RequiredShardsCount_) { + return; // do not progress + } + + CurrentCreationStep_ = ECreateComponentsStep::MakeTables; + break; + } + + case ECreateComponentsStep::MakeTables: { + if (++CreatedTablesCount_ != RequiredTables_.size()) { + return; // do not progress + } + Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second); CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId; - break; - } + break; + } case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { Y_VERIFY(TablesFormat_ == 1); Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second); @@ -578,28 +578,28 @@ void TCreateQueueSchemaActorV2::Step() { break; } case ECreateComponentsStep::AddQuoterResource: { - Y_VERIFY(false); // unreachable + Y_VERIFY(false); // unreachable break; - } - } - + } + } + CreateComponents(); -} - +} + void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); RLOG_SQS_TRACE("OnExecuted: " << ev->Get()->Record); - + if (ev->Sender == CreateTableWithLeaderTabletActorId_) { CreateTableWithLeaderTabletTxId_ = record.GetTxId(); TableWithLeaderPathId_ = std::make_pair(record.GetSchemeShardTabletId(), record.GetPathId()); RLOG_SQS_TRACE("Handle executed transaction with leader tablet: " << record); - } - - // Note: - // SS finishes transaction immediately if the specified path already exists - // DO NOT add any special logic based on the result type (except for an error) + } + + // Note: + // SS finishes transaction immediately if the specified path already exists + // DO NOT add any special logic based on the result type (except for an error) if (IsGoodStatusCode(status)) { if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) { const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); @@ -624,49 +624,49 @@ void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { } Step(); - } else { + } else { RLOG_SQS_WARN("Component creation request execution error: " << record); - + auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - resp->State = EQueueState::Creating; - - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest) { + resp->State = EQueueState::Creating; + + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest) { resp->Error = TStringBuilder() << "Missing account: " << QueuePath_.UserName << "."; - } else { - resp->Error = record.GetMiniKQLErrors(); - } - + } else { + resp->Error = record.GetMiniKQLErrors(); + } + Send(Sender_, std::move(resp)); - + PassAway(); - } -} - + } +} + void TCreateQueueSchemaActorV2::OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { RLOG_SQS_TRACE("OnDescribeSchemeResult for leader tablet: " << ev->Get()->GetRecord()); - const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription(); - + const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription(); + if (ev->Get()->GetRecord().GetStatus() != NKikimrScheme::StatusSuccess || pathDescription.TablePartitionsSize() == 0 || !pathDescription.GetTablePartitions(0).GetDatashardId()) { - // fail + // fail auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - resp->State = EQueueState::Creating; + resp->State = EQueueState::Creating; resp->Error = "Failed to discover leader."; - + Send(Sender_, std::move(resp)); - + PassAway(); - return; - } - + return; + } + LeaderTabletId_ = pathDescription.GetTablePartitions(0).GetDatashardId(); - + if (Cfg().GetQuotingConfig().GetEnableQuoting() && Cfg().GetQuotingConfig().HasKesusQuoterConfig()) { Step(); } else { CommitNewVersion(); } -} - +} + void TCreateQueueSchemaActorV2::SendDescribeTable() { auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); @@ -718,38 +718,38 @@ void TCreateQueueSchemaActorV2::HandleAddQuoterResource(NKesus::TEvKesus::TEvAdd } } -static const char* const CommitQueueParamsQuery = R"__( - ( - (let name (Parameter 'NAME (DataType 'Utf8String))) - (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String))) - (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) - (let id (Parameter 'ID (DataType 'String))) - (let fifo (Parameter 'FIFO (DataType 'Bool))) +static const char* const CommitQueueParamsQuery = R"__( + ( + (let name (Parameter 'NAME (DataType 'Utf8String))) + (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String))) + (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) + (let id (Parameter 'ID (DataType 'String))) + (let fifo (Parameter 'FIFO (DataType 'Bool))) (let contentBasedDeduplication (Parameter 'CONTENT_BASED_DEDUPLICATION (DataType 'Bool))) - (let now (Parameter 'NOW (DataType 'Uint64))) - (let shards (Parameter 'SHARDS (DataType 'Uint64))) - (let partitions (Parameter 'PARTITIONS (DataType 'Uint64))) + (let now (Parameter 'NOW (DataType 'Uint64))) + (let shards (Parameter 'SHARDS (DataType 'Uint64))) + (let partitions (Parameter 'PARTITIONS (DataType 'Uint64))) (let masterTabletId (Parameter 'MASTER_TABLET_ID (DataType 'Uint64))) (let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32))) - (let version (Parameter 'VERSION (DataType 'Uint64))) + (let version (Parameter 'VERSION (DataType 'Uint64))) (let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64))) - (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64))) - (let delay (Parameter 'DELAY (DataType 'Uint64))) - (let visibility (Parameter 'VISIBILITY (DataType 'Uint64))) - (let retention (Parameter 'RETENTION (DataType 'Uint64))) + (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64))) + (let delay (Parameter 'DELAY (DataType 'Uint64))) + (let visibility (Parameter 'VISIBILITY (DataType 'Uint64))) + (let retention (Parameter 'RETENTION (DataType 'Uint64))) (let receiveMessageWaitTime (Parameter 'RECEIVE_MESSAGE_WAIT_TIME (DataType 'Uint64))) - (let dlqArn (Parameter 'DLQ_TARGET_ARN (DataType 'Utf8String))) - (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String))) - (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64))) + (let dlqArn (Parameter 'DLQ_TARGET_ARN (DataType 'Utf8String))) + (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String))) + (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64))) (let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - + (let attrsTable '%1$s/Attributes) (let stateTable '%1$s/State) (let settingsTable '%2$s/.Settings) (let queuesTable '%2$s/.Queues) (let eventsTable '%2$s/.Events) - + (let maxQueuesCountSettingRow '( '('Account userName) '('Name (Utf8String '"MaxQueuesCount")))) @@ -760,15 +760,15 @@ static const char* const CommitQueueParamsQuery = R"__( (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) - (let queues + '('QueueName (Utf8String '"") (Void)))) + (let queues (Member (SelectRange queuesTable queuesRange '('QueueName 'CustomQueueName 'Version 'FolderId 'QueueState) '()) 'List)) - (let overLimit + (let overLimit (LessOrEqual maxQueuesCountSetting (Length queues))) - + (let queuesRow '( '('Account userName) - '('QueueName name))) + '('QueueName name))) (let eventsRow '( '('Account userName) @@ -776,66 +776,66 @@ static const char* const CommitQueueParamsQuery = R"__( '('EventType (Uint64 '1)))) (let queuesSelect '( - 'QueueState - 'QueueId - 'FifoQueue - 'Shards - 'Partitions - 'Version)) + 'QueueState + 'QueueId + 'FifoQueue + 'Shards + 'Partitions + 'Version)) (let queuesRead (SelectRow queuesTable queuesRow queuesSelect)) - - (let existingQueuesWithSameNameAndFolderId + + (let existingQueuesWithSameNameAndFolderId (If (Equal (Utf8String '"") customName) (List (TypeOf queues)) - (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (Equal (Member item 'CustomQueueName) customName) - (Equal (Member item 'FolderId) folderId)) - (Bool 'false)))))) - ) - ) - ) - - (let existingResourceId (Coalesce - (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName) - (Utf8String '"") - )) - - (let queueExists - (Coalesce - (Or - (Or + (Filter queues (lambda '(item) (block '( + (return (Coalesce + (And + (Equal (Member item 'CustomQueueName) customName) + (Equal (Member item 'FolderId) folderId)) + (Bool 'false)))))) + ) + ) + ) + + (let existingResourceId (Coalesce + (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName) + (Utf8String '"") + )) + + (let queueExists + (Coalesce + (Or + (Or (Equal (Uint64 '1) (Member queuesRead 'QueueState)) (Equal (Uint64 '3) (Member queuesRead 'QueueState)) - ) - (NotEqual (Utf8String '"") existingResourceId) - ) - (Bool 'false))) - - (let currentVersion - (Coalesce - (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version) + ) + (NotEqual (Utf8String '"") existingResourceId) + ) + (Bool 'false))) + + (let currentVersion + (Coalesce + (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version) (Member queuesRead 'Version) - (Uint64 '0) - ) - ) - + (Uint64 '0) + ) + ) + (let queuesUpdate '( - '('QueueId id) - '('CustomQueueName customName) - '('FolderId folderId) - '('QueueState (Uint64 '3)) - '('FifoQueue fifo) - '('DeadLetterQueue (Bool 'false)) - '('CreatedTimestamp now) - '('Shards shards) - '('Partitions partitions) - '('Version version) - '('DlqName dlqName) + '('QueueId id) + '('CustomQueueName customName) + '('FolderId folderId) + '('QueueState (Uint64 '3)) + '('FifoQueue fifo) + '('DeadLetterQueue (Bool 'false)) + '('CreatedTimestamp now) + '('Shards shards) + '('Partitions partitions) + '('Version version) + '('DlqName dlqName) '('MasterTabletId masterTabletId) '('TablesFormat tablesFormat))) - + (let eventsUpdate '( '('CustomQueueName customName) '('EventTimestamp now) @@ -843,23 +843,23 @@ static const char* const CommitQueueParamsQuery = R"__( (let attrRow '(%3$s)) - (let attrUpdate '( + (let attrUpdate '( '('ContentBasedDeduplication contentBasedDeduplication) - '('DelaySeconds delay) - '('FifoQueue fifo) - '('MaximumMessageSize maxSize) - '('MessageRetentionPeriod retention) + '('DelaySeconds delay) + '('FifoQueue fifo) + '('MaximumMessageSize maxSize) + '('MessageRetentionPeriod retention) '('ReceiveMessageWaitTime receiveMessageWaitTime) - '('MaxReceiveCount maxReceiveCount) - '('DlqArn dlqArn) - '('DlqName dlqName) - '('VisibilityTimeout visibility))) - - (let willCommit - (And - (Not queueExists) - (Not overLimit))) - + '('MaxReceiveCount maxReceiveCount) + '('DlqArn dlqArn) + '('DlqName dlqName) + '('VisibilityTimeout visibility))) + + (let willCommit + (And + (Not queueExists) + (Not overLimit))) + (let stateUpdate '( '('CleanupTimestamp now) '('CreatedTimestamp now) @@ -873,19 +873,19 @@ static const char* const CommitQueueParamsQuery = R"__( (let queueIdNumberAndShardHashes (AsList %4$s)) - (return (Extend - (AsList - (SetResult 'exists queueExists) - (SetResult 'overLimit overLimit) - (SetResult 'version currentVersion) - (SetResult 'resourceId existingResourceId) - (SetResult 'commited willCommit)) - + (return (Extend + (AsList + (SetResult 'exists queueExists) + (SetResult 'overLimit overLimit) + (SetResult 'version currentVersion) + (SetResult 'resourceId existingResourceId) + (SetResult 'commited willCommit)) + (ListIf queueExists (SetResult 'meta queuesRead)) - + (ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate)) (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate)) - (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate)) + (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate)) (If (Not willCommit) (AsList (Void)) (Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '( @@ -894,21 +894,21 @@ static const char* const CommitQueueParamsQuery = R"__( (let queueIdNumberAndShardHash (Nth item '1)) (let row '(%5$s)) - (let update '( - '('CleanupTimestamp now) - '('CreatedTimestamp now) - '('LastModifiedTimestamp now) - '('InflyCount (Int64 '0)) - '('MessageCount (Int64 '0)) - '('RetentionBoundary (Uint64 '0)) - '('ReadOffset (Uint64 '0)) - '('WriteOffset (Uint64 '0)) + (let update '( + '('CleanupTimestamp now) + '('CreatedTimestamp now) + '('LastModifiedTimestamp now) + '('InflyCount (Int64 '0)) + '('MessageCount (Int64 '0)) + '('RetentionBoundary (Uint64 '0)) + '('ReadOffset (Uint64 '0)) + '('WriteOffset (Uint64 '0)) '('CleanupVersion (Uint64 '0)))) - (return (UpdateRow stateTable row update))))))) - )) - ) -)__"; - + (return (UpdateRow stateTable row update))))))) + )) + ) +)__"; + TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) { if (tablesFormat == 1) { if (isFifo) { @@ -946,8 +946,8 @@ TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) { } void TCreateQueueSchemaActorV2::CommitNewVersion() { - Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit); - + Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit); + TString queuePath; if (TablesFormat_ == 0) { queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_); @@ -965,271 +965,271 @@ void TCreateQueueSchemaActorV2::CommitNewVersion() { ); auto ev = MakeExecuteEvent(query); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); Y_VERIFY(LeaderTabletId_ != 0); - TParameters(trans->MutableParams()->MutableProto()) - .Utf8("NAME", QueuePath_.QueueName) - .Utf8("CUSTOMNAME", CustomQueueName_) - .Utf8("FOLDERID", FolderId_) - .String("ID", GeneratedQueueId_) - .Bool("FIFO", IsFifo_) - .Bool("CONTENT_BASED_DEDUPLICATION", *ValidatedAttributes_.ContentBasedDeduplication) - .Uint64("NOW", QueueCreationTimestamp_.MilliSeconds()) - .Uint64("SHARDS", RequiredShardsCount_) - .Uint64("PARTITIONS", Request_.GetPartitions()) + TParameters(trans->MutableParams()->MutableProto()) + .Utf8("NAME", QueuePath_.QueueName) + .Utf8("CUSTOMNAME", CustomQueueName_) + .Utf8("FOLDERID", FolderId_) + .String("ID", GeneratedQueueId_) + .Bool("FIFO", IsFifo_) + .Bool("CONTENT_BASED_DEDUPLICATION", *ValidatedAttributes_.ContentBasedDeduplication) + .Uint64("NOW", QueueCreationTimestamp_.MilliSeconds()) + .Uint64("SHARDS", RequiredShardsCount_) + .Uint64("PARTITIONS", Request_.GetPartitions()) .Uint64("MASTER_TABLET_ID", LeaderTabletId_) .Uint32("TABLES_FORMAT", TablesFormat_) - .Uint64("VERSION", Version_) + .Uint64("VERSION", Version_) .Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_)) - .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize) - .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds)) - .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout)) - .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod)) - .Uint64("RECEIVE_MESSAGE_WAIT_TIME", SecondsToMs(*ValidatedAttributes_.ReceiveMessageWaitTimeSeconds)) - .Utf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn ? *ValidatedAttributes_.RedrivePolicy.TargetArn : "") - .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "") + .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize) + .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds)) + .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout)) + .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod)) + .Uint64("RECEIVE_MESSAGE_WAIT_TIME", SecondsToMs(*ValidatedAttributes_.ReceiveMessageWaitTimeSeconds)) + .Utf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn ? *ValidatedAttributes_.RedrivePolicy.TargetArn : "") + .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "") .Uint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount ? *ValidatedAttributes_.RedrivePolicy.MaxReceiveCount : 0) .Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount()) .Utf8("USER_NAME", QueuePath_.UserName); - + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); -} - +} + STATEFN(TCreateQueueSchemaActorV2::FinalizeAndCommit) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, OnCommit); cFunc(TEvPoisonPill::EventType, PassAway); - } -} - + } +} + void TCreateQueueSchemaActorV2::OnCommit(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); - + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); + auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - if (bool(val["commited"])) { - // a new born queue is here! - resp->QueueId = GeneratedQueueId_; - resp->Success = true; + + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + if (bool(val["commited"])) { + // a new born queue is here! + resp->QueueId = GeneratedQueueId_; + resp->Success = true; resp->ErrorClass = nullptr; - } else { - // something is off - if (bool(val["overLimit"])) { + } else { + // something is off + if (bool(val["overLimit"])) { resp->ErrorClass = &NErrors::OVER_LIMIT; resp->Error = "Too many queues."; - } else if (bool(val["exists"])) { - if (IsCloudMode_) { - ExistingQueueResourceId_ = TString(val["resourceId"]); - } - const ui64 currentVersion = ui64(val["version"]); + } else if (bool(val["exists"])) { + if (IsCloudMode_) { + ExistingQueueResourceId_ = TString(val["resourceId"]); + } + const ui64 currentVersion = ui64(val["version"]); MatchQueueAttributes(currentVersion); - return; - } else { - Y_VERIFY(false); // unreachable - } - } - } else { + return; + } else { + Y_VERIFY(false); // unreachable + } + } + } else { resp->Error = "Failed to commit new queue version."; - } - + } + Send(Sender_, std::move(resp)); PassAway(); -} - -static const char* const MatchQueueAttributesQuery = R"__( - ( - (let name (Parameter 'NAME (DataType 'Utf8String))) - (let fifo (Parameter 'FIFO (DataType 'Bool))) - (let shards (Parameter 'SHARDS (DataType 'Uint64))) - (let partitions (Parameter 'PARTITIONS (DataType 'Uint64))) - (let expectedVersion (Parameter 'EXPECTED_VERSION (DataType 'Uint64))) - (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64))) - (let delay (Parameter 'DELAY (DataType 'Uint64))) - (let visibility (Parameter 'VISIBILITY (DataType 'Uint64))) - (let retention (Parameter 'RETENTION (DataType 'Uint64))) - (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String))) - (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64))) +} + +static const char* const MatchQueueAttributesQuery = R"__( + ( + (let name (Parameter 'NAME (DataType 'Utf8String))) + (let fifo (Parameter 'FIFO (DataType 'Bool))) + (let shards (Parameter 'SHARDS (DataType 'Uint64))) + (let partitions (Parameter 'PARTITIONS (DataType 'Uint64))) + (let expectedVersion (Parameter 'EXPECTED_VERSION (DataType 'Uint64))) + (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64))) + (let delay (Parameter 'DELAY (DataType 'Uint64))) + (let visibility (Parameter 'VISIBILITY (DataType 'Uint64))) + (let retention (Parameter 'RETENTION (DataType 'Uint64))) + (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String))) + (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - - (let attrsTable '%1$s/%2$s/Attributes) + + (let attrsTable '%1$s/%2$s/Attributes) (let queuesTable '%3$s/.Queues) - + (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) - (let queues + '('QueueName (Utf8String '"") (Void)))) + (let queues (Member (SelectRange queuesTable queuesRange '('QueueState) '()) 'List)) - + (let queuesRow '( '('Account userName) - '('QueueName name))) + '('QueueName name))) (let queuesSelect '( - 'QueueState - 'QueueId - 'FifoQueue - 'Shards - 'Partitions - 'DlqName - 'Version)) + 'QueueState + 'QueueId + 'FifoQueue + 'Shards + 'Partitions + 'DlqName + 'Version)) (let queuesRead (SelectRow queuesTable queuesRow queuesSelect)) - - (let queueExists - (Coalesce - (Or + + (let queueExists + (Coalesce + (Or (Equal (Uint64 '1) (Member queuesRead 'QueueState)) (Equal (Uint64 '3) (Member queuesRead 'QueueState)) - ) - (Bool 'false))) - - (let currentVersion - (Coalesce + ) + (Bool 'false))) + + (let currentVersion + (Coalesce (Member queuesRead 'Version) - (Uint64 '0) - ) - ) - - (let sameParams - (Coalesce - (And - (And + (Uint64 '0) + ) + ) + + (let sameParams + (Coalesce + (And + (And (And (Equal (Member queuesRead 'Shards) shards) (Equal (Member queuesRead 'Partitions) partitions)) (Equal (Member queuesRead 'FifoQueue) fifo)) (Equal (Coalesce (Member queuesRead 'DlqName) (Utf8String '"")) dlqName)) - (Bool 'true))) - - (let attrRow '( - '('State (Uint64 '0)))) - (let attrSelect '( - 'DelaySeconds - 'MaximumMessageSize - 'MessageRetentionPeriod - 'MaxReceiveCount - 'VisibilityTimeout)) - (let attrRead (SelectRow attrsTable attrRow attrSelect)) - - (let sameAttributes - (Coalesce - (And - (And - (And (Equal (Member attrRead 'DelaySeconds) delay) - (And (Equal (Member attrRead 'MaximumMessageSize) maxSize) - (Equal (Member attrRead 'MessageRetentionPeriod) retention))) - (Equal (Member attrRead 'VisibilityTimeout) visibility)) - (Equal (Coalesce (Member attrRead 'MaxReceiveCount) (Uint64 '0)) maxReceiveCount)) - (Bool 'true))) - - (let sameVersion - (Equal currentVersion expectedVersion)) - - (let isSame - (And - queueExists - (And - sameVersion - (And - sameAttributes - sameParams)))) - - (let existingQueueId - (Coalesce + (Bool 'true))) + + (let attrRow '( + '('State (Uint64 '0)))) + (let attrSelect '( + 'DelaySeconds + 'MaximumMessageSize + 'MessageRetentionPeriod + 'MaxReceiveCount + 'VisibilityTimeout)) + (let attrRead (SelectRow attrsTable attrRow attrSelect)) + + (let sameAttributes + (Coalesce + (And + (And + (And (Equal (Member attrRead 'DelaySeconds) delay) + (And (Equal (Member attrRead 'MaximumMessageSize) maxSize) + (Equal (Member attrRead 'MessageRetentionPeriod) retention))) + (Equal (Member attrRead 'VisibilityTimeout) visibility)) + (Equal (Coalesce (Member attrRead 'MaxReceiveCount) (Uint64 '0)) maxReceiveCount)) + (Bool 'true))) + + (let sameVersion + (Equal currentVersion expectedVersion)) + + (let isSame + (And + queueExists + (And + sameVersion + (And + sameAttributes + sameParams)))) + + (let existingQueueId + (Coalesce (Member queuesRead 'QueueId) - (String '""))) - - (return (AsList - (SetResult 'exists queueExists) - (SetResult 'sameVersion sameVersion) - (SetResult 'id existingQueueId) - (SetResult 'isSame isSame))) - ) -)__"; - + (String '""))) + + (return (AsList + (SetResult 'exists queueExists) + (SetResult 'sameVersion sameVersion) + (SetResult 'id existingQueueId) + (SetResult 'isSame isSame))) + ) +)__"; + void TCreateQueueSchemaActorV2::MatchQueueAttributes(const ui64 currentVersion) { - Become(&TCreateQueueSchemaActorV2::MatchAttributes); - - TString versionedQueuePath = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName; - if (currentVersion != 0) { - // modern-way constructed queue requires version suffix - versionedQueuePath = TString::Join(versionedQueuePath, "/v", ToString(currentVersion)); - } - auto ev = MakeExecuteEvent(Sprintf( + Become(&TCreateQueueSchemaActorV2::MatchAttributes); + + TString versionedQueuePath = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName; + if (currentVersion != 0) { + // modern-way constructed queue requires version suffix + versionedQueuePath = TString::Join(versionedQueuePath, "/v", ToString(currentVersion)); + } + auto ev = MakeExecuteEvent(Sprintf( MatchQueueAttributesQuery, QueuePath_.GetUserPath().c_str(), versionedQueuePath.c_str(), Cfg().GetRoot().c_str() - )); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - TParameters(trans->MutableParams()->MutableProto()) - .Utf8("NAME", IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName) - .Bool("FIFO", IsFifo_) - .Uint64("SHARDS", RequiredShardsCount_) - .Uint64("PARTITIONS", Request_.GetPartitions()) - .Uint64("EXPECTED_VERSION", currentVersion) - .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize) - .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds)) - .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout)) - .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod)) - .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "") + )); + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + TParameters(trans->MutableParams()->MutableProto()) + .Utf8("NAME", IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName) + .Bool("FIFO", IsFifo_) + .Uint64("SHARDS", RequiredShardsCount_) + .Uint64("PARTITIONS", Request_.GetPartitions()) + .Uint64("EXPECTED_VERSION", currentVersion) + .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize) + .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds)) + .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout)) + .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod)) + .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "") .Uint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount ? *ValidatedAttributes_.RedrivePolicy.MaxReceiveCount : 0) .Utf8("USER_NAME", QueuePath_.UserName); - + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); -} - +} + STATEFN(TCreateQueueSchemaActorV2::MatchAttributes) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, OnAttributesMatch); cFunc(TEvPoisonPill::EventType, PassAway); - } -} - + } +} + void TCreateQueueSchemaActorV2::OnAttributesMatch(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); - + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); + auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - - if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - if (bool(val["exists"])) { - resp->AlreadyExists = true; + + if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + if (bool(val["exists"])) { + resp->AlreadyExists = true; resp->ErrorClass = nullptr; - resp->ExistingQueueResourceId = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName; - const bool isSame = bool(val["isSame"]); - if (isSame || !EnableQueueAttributesValidation_) { - resp->Success = true; + resp->ExistingQueueResourceId = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName; + const bool isSame = bool(val["isSame"]); + if (isSame || !EnableQueueAttributesValidation_) { + resp->Success = true; resp->ErrorClass = nullptr; - resp->QueueId = TString(val["id"]); - - if (!isSame) { + resp->QueueId = TString(val["id"]); + + if (!isSame) { RLOG_SQS_WARN("Queue attributes do not match for account " << QueuePath_.UserName << " and queue name " << QueuePath_.QueueName); - } - } else { + } + } else { resp->Error = "queue with specified name already exists and has different attributes."; resp->ErrorClass = &NErrors::VALIDATION_ERROR; - } - + } + if (CurrentCreationStep_ == ECreateComponentsStep::DiscoverLeaderTabletId) { - // call the special version of cleanup actor + // call the special version of cleanup actor RLOG_SQS_WARN("Removing redundant queue version: " << Version_ << " for queue " << - QueuePath_.GetQueuePath() << ". Shards: " << RequiredShardsCount_ << " IsFifo: " << IsFifo_); + QueuePath_.GetQueuePath() << ". Shards: " << RequiredShardsCount_ << " IsFifo: " << IsFifo_); Register(new TDeleteQueueSchemaActorV2(QueuePath_, SelfId(), RequestId_, UserCounters_, - Version_, RequiredShardsCount_, IsFifo_)); - } - - } else { + Version_, RequiredShardsCount_, IsFifo_)); + } + + } else { resp->Error = "Queue was removed recently."; resp->ErrorClass = &NErrors::QUEUE_DELETED_RECENTLY; resp->State = EQueueState::Deleting; - } - } else { + } + } else { resp->Error = "Failed to compare queue attributes."; - } - + } + Send(Sender_, std::move(resp)); PassAway(); -} - +} + void TCreateQueueSchemaActorV2::PassAway() { if (AddQuoterResourceActor_) { Send(AddQuoterResourceActor_, new TEvPoisonPill()); @@ -1238,97 +1238,97 @@ void TCreateQueueSchemaActorV2::PassAway() { TActorBootstrapped<TCreateQueueSchemaActorV2>::PassAway(); } -TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path, +TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path, const TActorId& sender, - const TString& requestId, + const TString& requestId, TIntrusivePtr<TUserCounters> userCounters) - : QueuePath_(path) - , Sender_(sender) - , SI_(0) - , RequestId_(requestId) + : QueuePath_(path) + , Sender_(sender) + , SI_(0) + , RequestId_(requestId) , UserCounters_(std::move(userCounters)) -{ -} - -TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path, +{ +} + +TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path, const TActorId& sender, - const TString& requestId, + const TString& requestId, TIntrusivePtr<TUserCounters> userCounters, - const ui64 advisedQueueVersion, - const ui64 advisedShardCount, - const bool advisedIsFifoFlag) - : QueuePath_(path) - , Sender_(sender) - , SI_(static_cast<ui32>(EDeleting::RemoveTables)) - , RequestId_(requestId) + const ui64 advisedQueueVersion, + const ui64 advisedShardCount, + const bool advisedIsFifoFlag) + : QueuePath_(path) + , Sender_(sender) + , SI_(static_cast<ui32>(EDeleting::RemoveTables)) + , RequestId_(requestId) , UserCounters_(std::move(userCounters)) -{ - Y_VERIFY(advisedQueueVersion > 0); - - Version_ = advisedQueueVersion; - - PrepareCleanupPlan(advisedIsFifoFlag, advisedShardCount); -} - +{ + Y_VERIFY(advisedQueueVersion > 0); + + Version_ = advisedQueueVersion; + + PrepareCleanupPlan(advisedIsFifoFlag, advisedShardCount); +} + void TDeleteQueueSchemaActorV2::Bootstrap() { NextAction(); - Become(&TThis::StateFunc); -} - -void TDeleteQueueSchemaActorV2::PrepareCleanupPlan(const bool isFifo, const ui64 shardCount) { - if (isFifo) { - Tables_ = GetFifoTables(); - } else { - Tables_ = GetStandardTableNames(shardCount); - - for (ui64 i = 0; i < shardCount; ++i) { - Shards_.push_back(i); - } - } -} - -static TString GetVersionedQueueDir(const TString& baseQueueDir, const ui64 version) { - if (!version) { - return baseQueueDir; - } - - return TString::Join(baseQueueDir, "/v", ToString(version)); -} - -static const char* EraseQueueRecordQuery = R"__( - ( - (let name (Parameter 'NAME (DataType 'Utf8String))) + Become(&TThis::StateFunc); +} + +void TDeleteQueueSchemaActorV2::PrepareCleanupPlan(const bool isFifo, const ui64 shardCount) { + if (isFifo) { + Tables_ = GetFifoTables(); + } else { + Tables_ = GetStandardTableNames(shardCount); + + for (ui64 i = 0; i < shardCount; ++i) { + Shards_.push_back(i); + } + } +} + +static TString GetVersionedQueueDir(const TString& baseQueueDir, const ui64 version) { + if (!version) { + return baseQueueDir; + } + + return TString::Join(baseQueueDir, "/v", ToString(version)); +} + +static const char* EraseQueueRecordQuery = R"__( + ( + (let name (Parameter 'NAME (DataType 'Utf8String))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) (let now (Parameter 'NOW (DataType 'Uint64))) - + (let queuesTable '%2$s/.Queues) (let eventsTable '%2$s/.Events) - + (let queuesRow '( '('Account userName) - '('QueueName name))) + '('QueueName name))) (let eventsRow '( '('Account userName) '('QueueName name) '('EventType (Uint64 '0)))) (let queuesSelect '( - 'QueueState - 'Version - 'FifoQueue + 'QueueState + 'Version + 'FifoQueue 'Shards 'CustomQueueName 'CreatedTimestamp 'FolderId)) (let queuesRead (SelectRow queuesTable queuesRow queuesSelect)) - - (let currentVersion - (Coalesce + + (let currentVersion + (Coalesce (Member queuesRead 'Version) - (Uint64 '0) - ) - ) - + (Uint64 '0) + ) + ) + (let queueCreateTs (Coalesce (Member queuesRead 'CreatedTimestamp) @@ -1351,129 +1351,129 @@ static const char* EraseQueueRecordQuery = R"__( (let eventTs (Max now (Add queueCreateTs (Uint64 '2)))) - (let queueExists - (Coalesce - (Or + (let queueExists + (Coalesce + (Or (Equal (Uint64 '1) (Member queuesRead 'QueueState)) (Equal (Uint64 '3) (Member queuesRead 'QueueState)) - ) - (Bool 'false))) - + ) + (Bool 'false))) + (let eventsUpdate '( '('CustomQueueName customName) '('EventTimestamp eventTs) '('FolderId folderId))) - (return (AsList - (SetResult 'exists queueExists) - (SetResult 'version currentVersion) + (return (AsList + (SetResult 'exists queueExists) + (SetResult 'version currentVersion) (SetResult 'fields queuesRead) (If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void)) (If queueExists (EraseRow queuesTable queuesRow) (Void)))) - ) -)__"; - + ) +)__"; + void TDeleteQueueSchemaActorV2::NextAction() { - switch (EDeleting(SI_)) { - case EDeleting::EraseQueueRecord: { + switch (EDeleting(SI_)) { + case EDeleting::EraseQueueRecord: { auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str())); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); auto nowMs = TInstant::Now().MilliSeconds(); - TParameters(trans->MutableParams()->MutableProto()) + TParameters(trans->MutableParams()->MutableProto()) .Utf8("NAME", QueuePath_.QueueName) .Utf8("USER_NAME", QueuePath_.UserName) .Uint64("NOW", nowMs); - + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); - break; - } - case EDeleting::RemoveTables: { + break; + } + case EDeleting::RemoveTables: { Y_VERIFY(!Tables_.empty()); - + Register(new TMiniKqlExecutionActor( SelfId(), RequestId_, MakeDeleteTableEvent(GetVersionedQueueDir(QueuePath_, Version_), Tables_.back()), false, QueuePath_, GetTransactionCounters(UserCounters_)) - ); - break; - } - case EDeleting::RemoveShards: { + ); + break; + } + case EDeleting::RemoveShards: { Register(new TMiniKqlExecutionActor( SelfId(), RequestId_, MakeRemoveDirectoryEvent(GetVersionedQueueDir(QueuePath_, Version_), ToString(Shards_.back())), false, QueuePath_, GetTransactionCounters(UserCounters_)) - ); - break; - } - case EDeleting::RemoveQueueVersionDirectory: { + ); + break; + } + case EDeleting::RemoveQueueVersionDirectory: { Register(new TMiniKqlExecutionActor( SelfId(), RequestId_, MakeRemoveDirectoryEvent(QueuePath_, TString::Join("v", ToString(Version_))), false, QueuePath_, GetTransactionCounters(UserCounters_)) - ); - break; - } - case EDeleting::RemoveQueueDirectory: { - // this may silently fail for versioned queues + ); + break; + } + case EDeleting::RemoveQueueDirectory: { + // this may silently fail for versioned queues Register(new TMiniKqlExecutionActor( SelfId(), RequestId_, MakeRemoveDirectoryEvent(QueuePath_.GetUserPath(), QueuePath_.QueueName), false, QueuePath_, GetTransactionCounters(UserCounters_)) - ); - break; - } + ); + break; + } case EDeleting::DeleteQuoterResource: { DeleteRPSQuota(); break; } - case EDeleting::Finish: { + case EDeleting::Finish: { Send(Sender_, MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, true)); PassAway(); - break; - } - } -} - + break; + } + } +} + void TDeleteQueueSchemaActorV2::DoSuccessOperation() { - if (EDeleting(SI_) == EDeleting::RemoveTables) { - Tables_.pop_back(); - - if (Tables_.empty()) { - if (Shards_.empty()) { - SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory); - } else { - SI_ = ui32(EDeleting::RemoveShards); - } - } - } else if (EDeleting(SI_) == EDeleting::RemoveShards) { - Shards_.pop_back(); - - if (Shards_.empty()) { - SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory); - } - } else { - SI_++; + if (EDeleting(SI_) == EDeleting::RemoveTables) { + Tables_.pop_back(); + + if (Tables_.empty()) { + if (Shards_.empty()) { + SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory); + } else { + SI_ = ui32(EDeleting::RemoveShards); + } + } + } else if (EDeleting(SI_) == EDeleting::RemoveShards) { + Shards_.pop_back(); + + if (Shards_.empty()) { + SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory); + } + } else { + SI_++; if ((!Cfg().GetQuotingConfig().GetEnableQuoting() || !Cfg().GetQuotingConfig().HasKesusQuoterConfig()) && EDeleting(SI_) == EDeleting::DeleteQuoterResource) { SI_++; } - } - + } + NextAction(); -} - +} + void TDeleteQueueSchemaActorV2::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; + const auto& record = ev->Get()->Record; if (IsGoodStatusCode(record.GetStatus())) { - if (EDeleting(SI_) == EDeleting::EraseQueueRecord) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - if (!bool(val["exists"])) { + if (EDeleting(SI_) == EDeleting::EraseQueueRecord) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + if (!bool(val["exists"])) { Send(Sender_, - MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Queue does not exist.")); + MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Queue does not exist.")); PassAway(); - return; - } else { - Version_ = ui64(val["version"]); - - PrepareCleanupPlan(bool(val["fields"]["FifoQueue"]), ui64(val["fields"]["Shards"])); - } - } - + return; + } else { + Version_ = ui64(val["version"]); + + PrepareCleanupPlan(bool(val["fields"]["FifoQueue"]), ui64(val["fields"]["Shards"])); + } + } + DoSuccessOperation(); - } else { + } else { RLOG_SQS_WARN("request execution error: " << record); - + if (EDeleting(SI_) == EDeleting::EraseQueueRecord) { Send(Sender_, MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Failed to erase queue record.")); @@ -1481,11 +1481,11 @@ void TDeleteQueueSchemaActorV2::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev return; } - // we don't really care if some components are already deleted + // we don't really care if some components are already deleted DoSuccessOperation(); - } -} - + } +} + void TDeleteQueueSchemaActorV2::DeleteRPSQuota() { NKikimrKesus::TEvDeleteQuoterResource cmd; cmd.SetResourcePath(TStringBuilder() << RPS_QUOTA_NAME << "/" << QueuePath_.QueueName); diff --git a/ydb/core/ymq/actor/queue_schema.h b/ydb/core/ymq/actor/queue_schema.h index 793ba1c06a..54cc41530d 100644 --- a/ydb/core/ymq/actor/queue_schema.h +++ b/ydb/core/ymq/actor/queue_schema.h @@ -1,85 +1,85 @@ -#pragma once +#pragma once #include "defs.h" - -#include "schema.h" + +#include "schema.h" #include <ydb/core/base/quoter.h> #include <ydb/core/kesus/tablet/events.h> #include <ydb/core/protos/config.pb.h> #include <ydb/public/lib/value/value.h> #include <ydb/core/ymq/base/queue_attributes.h> - -#include <util/generic/maybe.h> - + +#include <util/generic/maybe.h> + namespace NKikimr::NSQS { - -class TCreateQueueSchemaActorV2 + +class TCreateQueueSchemaActorV2 : public TActorBootstrapped<TCreateQueueSchemaActorV2> -{ -public: - TCreateQueueSchemaActorV2(const TQueuePath& path, - const TCreateQueueRequest& req, +{ +public: + TCreateQueueSchemaActorV2(const TQueuePath& path, + const TCreateQueueRequest& req, const TActorId& sender, - const TString& requestId, - const TString& customQueueName, - const TString& folderId, - const bool isCloudMode, - const bool enableQueueAttributesValidation, + const TString& requestId, + const TString& customQueueName, + const TString& folderId, + const bool isCloudMode, + const bool enableQueueAttributesValidation, TIntrusivePtr<TUserCounters> userCounters, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResources); - - ~TCreateQueueSchemaActorV2(); - - void InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config); - + + ~TCreateQueueSchemaActorV2(); + + void InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config); + void Bootstrap(); - + void RequestQueueParams(); - + STATEFN(Preamble); - + void HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev); - + void OnReadQueueParams(TSqsEvents::TEvExecuted::TPtr& ev); - + void RunAtomicCounterIncrement(); void OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev); - + void RequestCreateQueueQuota(); void OnCreateQueueQuota(TEvQuota::TEvClearance::TPtr& ev); void RequestTablesFormatSettings(const TString& accountName); void RegisterMakeDirActor(const TString& workingDir, const TString& dirName); - + void RequestLeaderTabletId(); - + void CreateComponents(); - + STATEFN(CreateComponentsState); - + void Step(); - + void OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev); - + void OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev); - + void SendDescribeTable(); void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); - template <typename T> - T GetAttribute(const TStringBuf name, const T& defaultValue) const; - + template <typename T> + T GetAttribute(const TStringBuf name, const T& defaultValue) const; + void CommitNewVersion(); - + STATEFN(FinalizeAndCommit); - + void OnCommit(TSqsEvents::TEvExecuted::TPtr& ev); - + void MatchQueueAttributes(const ui64 currentVersion); - + STATEFN(MatchAttributes); - + void OnAttributesMatch(TSqsEvents::TEvExecuted::TPtr& ev); - + void AddRPSQuota(); void HandleAddQuoterResource(NKesus::TEvKesus::TEvAddQuoterResourceResult::TPtr& ev); @@ -88,122 +88,122 @@ public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SQS_ACTOR; - } - -private: - enum class ECreateComponentsStep : ui32 { + } + +private: + enum class ECreateComponentsStep : ui32 { GetTablesFormatSetting, MakeQueueDir, - MakeQueueVersionDir, - MakeShards, - MakeTables, + MakeQueueVersionDir, + MakeShards, + MakeTables, DescribeTableForSetSchemeShardId, DiscoverLeaderTabletId, AddQuoterResource, - }; - - const TQueuePath QueuePath_; - const TCreateQueueRequest Request_; + }; + + const TQueuePath QueuePath_; + const TCreateQueueRequest Request_; const TActorId Sender_; - const TString CustomQueueName_; - const TString FolderId_; - const TString RequestId_; - const TString GeneratedQueueId_; - const TInstant QueueCreationTimestamp_; - - bool IsFifo_ = false; - bool IsCloudMode_ = false; - bool EnableQueueAttributesValidation_ = true; - + const TString CustomQueueName_; + const TString FolderId_; + const TString RequestId_; + const TString GeneratedQueueId_; + const TInstant QueueCreationTimestamp_; + + bool IsFifo_ = false; + bool IsCloudMode_ = false; + bool EnableQueueAttributesValidation_ = true; + ui32 TablesFormat_ = 0; ui64 Version_ = 0; - - TString VersionName_; - TString VersionedQueueFullPath_; - TString ExistingQueueResourceId_; + + TString VersionName_; + TString VersionedQueueFullPath_; + TString ExistingQueueResourceId_; TIntrusivePtr<TUserCounters> UserCounters_; TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_; - ui64 RequiredShardsCount_ = 0; - ui64 CreatedShardsCount_ = 0; - TVector<TTable> RequiredTables_; - ui64 CreatedTablesCount_ = 0; - TQueueAttributes ValidatedAttributes_; - + ui64 RequiredShardsCount_ = 0; + ui64 CreatedShardsCount_ = 0; + TVector<TTable> RequiredTables_; + ui64 CreatedTablesCount_ = 0; + TQueueAttributes ValidatedAttributes_; + ui64 LeaderTabletId_ = 0; TActorId CreateTableWithLeaderTabletActorId_; ui64 CreateTableWithLeaderTabletTxId_ = 0; std::pair<ui64, ui64> TableWithLeaderPathId_ = std::make_pair(0, 0); // (scheme shard, path id) are required for describing table - + ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting; TActorId AddQuoterResourceActor_; -}; - -class TDeleteQueueSchemaActorV2 +}; + +class TDeleteQueueSchemaActorV2 : public TActorBootstrapped<TDeleteQueueSchemaActorV2> -{ -public: - TDeleteQueueSchemaActorV2(const TQueuePath& path, +{ +public: + TDeleteQueueSchemaActorV2(const TQueuePath& path, const TActorId& sender, - const TString& requestId, + const TString& requestId, TIntrusivePtr<TUserCounters> userCounters); - - TDeleteQueueSchemaActorV2(const TQueuePath& path, + + TDeleteQueueSchemaActorV2(const TQueuePath& path, const TActorId& sender, - const TString& requestId, + const TString& requestId, TIntrusivePtr<TUserCounters> userCounters, - const ui64 advisedQueueVersion, - const ui64 advisedShardCount, - const bool advisedIsFifoFlag); - + const ui64 advisedQueueVersion, + const ui64 advisedShardCount, + const bool advisedIsFifoFlag); + void Bootstrap(); - + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SQS_ACTOR; - } - -private: - void PrepareCleanupPlan(const bool isFifo, const ui64 shardCount); - + } + +private: + void PrepareCleanupPlan(const bool isFifo, const ui64 shardCount); + void NextAction(); - + void DoSuccessOperation(); - + void DeleteRPSQuota(); -private: +private: STATEFN(StateFunc) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, HandleExecuted); hFunc(NKesus::TEvKesus::TEvDeleteQuoterResourceResult, HandleDeleteQuoterResource); cFunc(TEvPoisonPill::EventType, PassAway); - } - } - + } + } + void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev); void HandleDeleteQuoterResource(NKesus::TEvKesus::TEvDeleteQuoterResourceResult::TPtr& ev); void PassAway() override; - -private: - enum class EDeleting : ui32 { - EraseQueueRecord, - RemoveTables, - RemoveShards, - RemoveQueueVersionDirectory, - RemoveQueueDirectory, + +private: + enum class EDeleting : ui32 { + EraseQueueRecord, + RemoveTables, + RemoveShards, + RemoveQueueVersionDirectory, + RemoveQueueDirectory, DeleteQuoterResource, - Finish, - }; - - const TQueuePath QueuePath_; + Finish, + }; + + const TQueuePath QueuePath_; const TActorId Sender_; - TVector<TTable> Tables_; - TVector<int> Shards_; - ui32 SI_; - const TString RequestId_; + TVector<TTable> Tables_; + TVector<int> Shards_; + ui32 SI_; + const TString RequestId_; TIntrusivePtr<TUserCounters> UserCounters_; - ui64 Version_ = 0; + ui64 Version_ = 0; TActorId DeleteQuoterResourceActor_; -}; - +}; + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/queues_list_reader.cpp b/ydb/core/ymq/actor/queues_list_reader.cpp index 37b8b73426..865475bd60 100644 --- a/ydb/core/ymq/actor/queues_list_reader.cpp +++ b/ydb/core/ymq/actor/queues_list_reader.cpp @@ -24,8 +24,8 @@ STATEFN(TQueuesListReader::StateFunc) { } void TQueuesListReader::HandleReadQueuesList(TSqsEvents::TEvReadQueuesList::TPtr& ev) { - Recipients.insert(ev->Sender); - + Recipients.insert(ev->Sender); + if (!ListingQueues) { ListingQueues = true; Result = MakeHolder<TSqsEvents::TEvQueuesList>(); @@ -60,7 +60,7 @@ void TQueuesListReader::OnRequestCompiled(const TSqsEvents::TEvExecuted::TRecord NextRequest(); } else { LOG_SQS_WARN("Get queues list request compilation failed: " << record); - Fail(); + Fail(); } } @@ -126,7 +126,7 @@ void TQueuesListReader::OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& rec } rec.FolderId = row["FolderId"]; rec.ShardsCount = row["Shards"]; - rec.DlqName = row["DlqName"]; + rec.DlqName = row["DlqName"]; rec.CreatedTimestamp = TInstant::MilliSeconds(ui64(row["CreatedTimestamp"])); } @@ -137,38 +137,38 @@ void TQueuesListReader::OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& rec NextRequest(); } else { std::sort(Result->SortedQueues.begin(), Result->SortedQueues.end()); // If .Queues table consists of many shards, result is possibly not sorted. - Success(); + Success(); } } else { LOG_SQS_WARN("Get queues list request failed: " << record); - Fail(); + Fail(); } } -void TQueuesListReader::Success() { - if (Recipients.size() == 1) { - Send(*Recipients.begin(), std::move(Result)); - } else { - for (const auto& recipientId : Recipients) { - auto result = MakeHolder<TSqsEvents::TEvQueuesList>(); - result->SortedQueues = Result->SortedQueues; - result->Success = Result->Success; - Send(recipientId, result.Release()); - } - } - - Recipients.clear(); - +void TQueuesListReader::Success() { + if (Recipients.size() == 1) { + Send(*Recipients.begin(), std::move(Result)); + } else { + for (const auto& recipientId : Recipients) { + auto result = MakeHolder<TSqsEvents::TEvQueuesList>(); + result->SortedQueues = Result->SortedQueues; + result->Success = Result->Success; + Send(recipientId, result.Release()); + } + } + + Recipients.clear(); + ListingQueues = false; } -void TQueuesListReader::Fail() { - for (const auto& recipientId : Recipients) { - Send(recipientId, new TSqsEvents::TEvQueuesList(false)); - } - - Recipients.clear(); - +void TQueuesListReader::Fail() { + for (const auto& recipientId : Recipients) { + Send(recipientId, new TSqsEvents::TEvQueuesList(false)); + } + + Recipients.clear(); + ListingQueues = false; } diff --git a/ydb/core/ymq/actor/queues_list_reader.h b/ydb/core/ymq/actor/queues_list_reader.h index 6562ad4036..a8924be8f8 100644 --- a/ydb/core/ymq/actor/queues_list_reader.h +++ b/ydb/core/ymq/actor/queues_list_reader.h @@ -31,8 +31,8 @@ private: void NextRequest(); void OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& record); - void Success(); - void Fail(); + void Success(); + void Fail(); private: TString CompiledQuery; diff --git a/ydb/core/ymq/actor/receive_message.cpp b/ydb/core/ymq/actor/receive_message.cpp index 2b5bf0e517..0060168208 100644 --- a/ydb/core/ymq/actor/receive_message.cpp +++ b/ydb/core/ymq/actor/receive_message.cpp @@ -31,7 +31,7 @@ public: { CopyAccountName(Request()); Response_.MutableReceiveMessage()->SetRequestId(RequestId_); - + CopySecurityToken(Request()); } @@ -185,8 +185,8 @@ private: item->SetSentTimestamp(message.SentTimestamp.MilliSeconds()); if (message.SenderId) { item->SetSenderId(message.SenderId); - } - + } + if (message.MessageAttributes) { TMessageAttributeList attrs; if (attrs.ParseFromString(message.MessageAttributes)) { diff --git a/ydb/core/ymq/actor/schema.cpp b/ydb/core/ymq/actor/schema.cpp index 20e84581fe..6e3f2bbd67 100644 --- a/ydb/core/ymq/actor/schema.cpp +++ b/ydb/core/ymq/actor/schema.cpp @@ -27,26 +27,26 @@ extern const TString RPS_QUOTA_NAME = "RPSQuota"; namespace { -static const char* const GetNextAtomicValueQuery = R"__( - ( - (let counterTable '%1$s/.AtomicCounter) - (let counterRow '( - '('counter_key (Uint64 '0)))) - (let counterSelect '( - 'value)) - (let counterRead - (SelectRow counterTable counterRow counterSelect)) - (let newValue - (Add (Member counterRead 'value) (Uint64 '1))) - (let counterUpdate '( - '('value newValue))) - (return (Extend - (AsList (SetResult 'value newValue)) - (AsList (UpdateRow counterTable counterRow counterUpdate)) - )) - ) -)__"; - +static const char* const GetNextAtomicValueQuery = R"__( + ( + (let counterTable '%1$s/.AtomicCounter) + (let counterRow '( + '('counter_key (Uint64 '0)))) + (let counterSelect '( + 'value)) + (let counterRead + (SelectRow counterTable counterRow counterSelect)) + (let newValue + (Add (Member counterRead 'value) (Uint64 '1))) + (let counterUpdate '( + '('value newValue))) + (return (Extend + (AsList (SetResult 'value newValue)) + (AsList (UpdateRow counterTable counterRow counterUpdate)) + )) + ) +)__"; + static bool SuccessStatusCode(ui32 code) { switch (NTxProxy::TResultStatus::EStatus(code)) { case NTxProxy::TResultStatus::EStatus::ExecComplete: @@ -110,31 +110,31 @@ static void SetOnePartitionPerShardSettings(NKikimrSchemeOp::TTableDescription& } } -} // namespace - -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeExecuteEvent(const TString& query) -{ - auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - trans->SetMode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC); - trans->SetFlatMKQL(true); - trans->MutableProgram()->SetText(query); - - return ev; -} - -THolder<TEvTxUserProxy::TEvProposeTransaction> +} // namespace + +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeExecuteEvent(const TString& query) +{ + auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + trans->SetMode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC); + trans->SetFlatMKQL(true); + trans->MutableProgram()->SetText(query); + + return ev; +} + +THolder<TEvTxUserProxy::TEvProposeTransaction> MakeCreateTableEvent(const TString& root, const TTable& table, - size_t queueShardsCount) + size_t queueShardsCount) { auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); // Transaction info auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); if (table.Shard != -1) { - trans->SetWorkingDir(TString::Join(root, "/", ToString(table.Shard))); + trans->SetWorkingDir(TString::Join(root, "/", ToString(table.Shard))); } else { trans->SetWorkingDir(root); } @@ -182,37 +182,37 @@ THolder<TEvTxUserProxy::TEvProposeTransaction> return ev; } -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeDeleteTableEvent(const TString& root, - const TTable& table) -{ - auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - // Transaction info - auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); - if (table.Shard == -1) { - trans->SetWorkingDir(root); - } else { - trans->SetWorkingDir(root + "/" + ToString(table.Shard)); - } +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeDeleteTableEvent(const TString& root, + const TTable& table) +{ + auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + // Transaction info + auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); + if (table.Shard == -1) { + trans->SetWorkingDir(root); + } else { + trans->SetWorkingDir(root + "/" + ToString(table.Shard)); + } trans->SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); - trans->MutableDrop()->SetName(table.Name); - - return ev; -} - -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeRemoveDirectoryEvent(const TString& root, const TString& name) -{ - auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - // Transaction info - auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); - trans->SetWorkingDir(root); + trans->MutableDrop()->SetName(table.Name); + + return ev; +} + +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeRemoveDirectoryEvent(const TString& root, const TString& name) +{ + auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + // Transaction info + auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); + trans->SetWorkingDir(root); trans->SetOperationType(NKikimrSchemeOp::ESchemeOpRmDir); - trans->MutableDrop()->SetName(name); - - return ev; -} - + trans->MutableDrop()->SetName(name); + + return ev; +} + THolder<TEvTxUserProxy::TEvProposeTransaction> MakeCreateKesusEvent(const TString& root, const TString& kesusName) @@ -248,12 +248,12 @@ THolder<TEvTxUserProxy::TEvProposeTransaction> } TCreateUserSchemaActor::TCreateUserSchemaActor(const TString& root, - const TString& userName, + const TString& userName, const TActorId& sender, const TString& requestId, TIntrusivePtr<TUserCounters> userCounters) : Root_(root) - , UserName_(userName) + , UserName_(userName) , Sender_(sender) , SI_(static_cast<int>(ECreating::MakeDirectory)) , RequestId_(requestId) @@ -459,36 +459,36 @@ void TDeleteUserSchemaActor::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { } TAtomicCounterActor::TAtomicCounterActor(const TActorId& sender, const TString& rootPath, const TString& requestId) - : Sender_(sender) - , RootPath_(rootPath) - , RequestId_(requestId) -{ -} - -TAtomicCounterActor::~TAtomicCounterActor() = default; - + : Sender_(sender) + , RootPath_(rootPath) + , RequestId_(requestId) +{ +} + +TAtomicCounterActor::~TAtomicCounterActor() = default; + void TAtomicCounterActor::Bootstrap() { - Become(&TThis::StateFunc); - auto ev = MakeExecuteEvent(Sprintf(GetNextAtomicValueQuery, RootPath_.c_str())); + Become(&TThis::StateFunc); + auto ev = MakeExecuteEvent(Sprintf(GetNextAtomicValueQuery, RootPath_.c_str())); Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), true, TQueuePath(), nullptr)); -} - +} + void TAtomicCounterActor::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { - const auto& record = ev->Get()->Record; - const auto status = record.GetStatus(); - - if (SuccessStatusCode(status)) { - const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + const auto& record = ev->Get()->Record; + const auto status = record.GetStatus(); + + if (SuccessStatusCode(status)) { + const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); Send(Sender_, - MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(true, "ok", val["value"])); - } else { + MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(true, "ok", val["value"])); + } else { RLOG_SQS_ERROR("Failed to increment the atomic counter: bad status code"); Send(Sender_, - MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(false)); - } + MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(false)); + } PassAway(); -} - +} + template <class TEvCmd, class TEvCmdResult> class TQuoterCmdRunner : public TActorBootstrapped<TQuoterCmdRunner<TEvCmd, TEvCmdResult>> { public: diff --git a/ydb/core/ymq/actor/schema.h b/ydb/core/ymq/actor/schema.h index c88face921..76a5e624c0 100644 --- a/ydb/core/ymq/actor/schema.h +++ b/ydb/core/ymq/actor/schema.h @@ -21,21 +21,21 @@ namespace NKikimr::NSQS { extern const TString QUOTER_KESUS_NAME; extern const TString RPS_QUOTA_NAME; -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeExecuteEvent(const TString& query); - -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeCreateTableEvent(const TString& root, - const TTable& table, - size_t queueShardsCount = 0); - -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeDeleteTableEvent(const TString& root, - const TTable& table); - -THolder<TEvTxUserProxy::TEvProposeTransaction> - MakeRemoveDirectoryEvent(const TString& root, const TString& name); - +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeExecuteEvent(const TString& query); + +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeCreateTableEvent(const TString& root, + const TTable& table, + size_t queueShardsCount = 0); + +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeDeleteTableEvent(const TString& root, + const TTable& table); + +THolder<TEvTxUserProxy::TEvProposeTransaction> + MakeRemoveDirectoryEvent(const TString& root, const TString& name); + // Create actor that calls AddQuoterResource and handles pipe errors and retries TActorId RunAddQuoterResource(ui64 quoterSchemeShardId, ui64 quoterPathId, const NKikimrKesus::TEvAddQuoterResource& cmd, const TString& requestId); TActorId RunAddQuoterResource(const TString& quoterPath, const NKikimrKesus::TEvAddQuoterResource& cmd, const TString& requestId); @@ -94,7 +94,7 @@ private: }; const TString Root_; - const TString UserName_; + const TString UserName_; const TActorId Sender_; int SI_; const TString RequestId_; @@ -148,32 +148,32 @@ private: TIntrusivePtr<TUserCounters> UserCounters_; }; -class TAtomicCounterActor +class TAtomicCounterActor : public TActorBootstrapped<TAtomicCounterActor> -{ -public: +{ +public: TAtomicCounterActor(const TActorId& sender, const TString& rootPath, const TString& requestId); - ~TAtomicCounterActor(); - + ~TAtomicCounterActor(); + void Bootstrap(); - + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SQS_ACTOR; - } - -private: + } + +private: STATEFN(StateFunc) { - switch (ev->GetTypeRewrite()) { + switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, HandleExecuted); - } - } - + } + } + void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev); - -private: + +private: const TActorId Sender_; - const TString RootPath_; - const TString RequestId_; -}; - + const TString RootPath_; + const TString RequestId_; +}; + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp index 34c9a4ae46..caf26a509e 100644 --- a/ydb/core/ymq/actor/service.cpp +++ b/ydb/core/ymq/actor/service.cpp @@ -1,9 +1,9 @@ #include "service.h" - + #include "auth_factory.h" #include "cfg.h" #include "executor.h" -#include "garbage_collector.h" +#include "garbage_collector.h" #include "local_rate_limiter_allocator.h" #include "params.h" #include "proxy_service.h" @@ -78,8 +78,8 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> { , QueueName_(std::move(queueName)) , CustomName_(std::move(customName)) , FolderId_(std::move(folderId)) - , Version_(version) - , ShardsCount_(shardsCount) + , Version_(version) + , ShardsCount_(shardsCount) , RootUrl_(std::move(rootUrl)) , LeaderTabletId_(leaderTabletId) , Counters_(userCounters->CreateQueueCounters(QueueName_, FolderId_, insertCounters)) @@ -182,8 +182,8 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> { TString QueueName_; TString CustomName_; TString FolderId_; - ui64 Version_; - ui64 ShardsCount_; + ui64 Version_; + ui64 ShardsCount_; TString RootUrl_; ui64 LeaderTabletId_ = 0; TIntrusivePtr<TQueueCounters> Counters_; @@ -239,14 +239,14 @@ struct TSqsService::TUserInfo : public TAtomicRefCount<TUserInfo> { } } - size_t CountQueuesInFolder(const TString& folderId) const { - if (!folderId) { - return QueueByNameAndFolder_.size(); // for YaSQS - } - - return std::count_if(QueueByNameAndFolder_.begin(), QueueByNameAndFolder_.end(), [&folderId](const auto& p) { return p.first.second == folderId; }); - } - + size_t CountQueuesInFolder(const TString& folderId) const { + if (!folderId) { + return QueueByNameAndFolder_.size(); // for YaSQS + } + + return std::count_if(QueueByNameAndFolder_.begin(), QueueByNameAndFolder_.end(), [&folderId](const auto& p) { return p.first.second == folderId; }); + } + TString UserName_; std::shared_ptr<const std::map<TString, TString>> Settings_ = std::make_shared<const std::map<TString, TString>>(); TIntrusivePtr<TUserCounters> Counters_; @@ -262,8 +262,8 @@ struct TSqsService::TUserInfo : public TAtomicRefCount<TUserInfo> { THashMultiMap<TString, TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr> GetLeaderNodeRequests_; // queue name -> request THashMultiMap<TString, TSqsEvents::TEvGetConfiguration::TPtr> GetConfigurationRequests_; // queue name -> request THashMultiMap<std::pair<TString, TString>, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // <queue custom name, folder id> -> request - THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // queue name -> request - THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // folder id -> request + THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // queue name -> request + THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // folder id -> request }; static TString GetEndpoint(const NKikimrConfig::TSqsConfig& config) { @@ -310,8 +310,8 @@ void TSqsService::Bootstrap() { Register(new TUserSettingsReader(AggregatedUserCounters_->GetTransactionCounters())); QueuesListReader_ = Register(new TQueuesListReader(AggregatedUserCounters_->GetTransactionCounters())); - Register(CreateGarbageCollector(SchemeCache_, QueuesListReader_)); - + Register(CreateGarbageCollector(SchemeCache_, QueuesListReader_)); + RequestSqsUsersList(); RequestSqsQueuesList(); @@ -541,8 +541,8 @@ void TSqsService::AnswerNotExists(TSqsEvents::TEvGetQueueId::TPtr& ev, const TUs void TSqsService::AnswerNotExists(TSqsEvents::TEvCountQueues::TPtr& ev, const TUserInfoPtr&) { RLOG_SQS_REQ_DEBUG(ev->Get()->RequestId, "No user [" << ev->Get()->UserName << "] found while counting queues"); Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(false)); -} - +} + void TSqsService::AnswerFailed(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr& ev, const TUserInfoPtr&) { Send(ev->Sender, new TSqsEvents::TEvGetLeaderNodeForQueueResponse(ev->Get()->RequestId, ev->Get()->UserName, ev->Get()->QueueName, TSqsEvents::TEvGetLeaderNodeForQueueResponse::EStatus::Error)); } @@ -568,8 +568,8 @@ void TSqsService::AnswerFailed(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPt void TSqsService::AnswerFailed(TSqsEvents::TEvCountQueues::TPtr& ev, const TUserInfoPtr&) { Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(true)); -} - +} + void TSqsService::Answer(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr& ev, const TQueueInfoPtr& queueInfo) { Send(ev->Sender, new TSqsEvents::TEvQueueFolderIdAndCustomName(queueInfo->FolderId_, queueInfo->CustomName_)); } @@ -699,13 +699,13 @@ void TSqsService::HandleGetQueueFolderIdAndCustomName(TSqsEvents::TEvGetQueueFol void TSqsService::HandleCountQueues(TSqsEvents::TEvCountQueues::TPtr& ev) { TUserInfoPtr user = GetUserOrWait(ev); - if (!user) { - return; - } - + if (!user) { + return; + } + Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(false, true, user->CountQueuesInFolder(ev->Get()->FolderId))); -} - +} + template <class TEvent> TSqsService::TUserInfoPtr TSqsService::GetUserOrWait(TAutoPtr<TEvent>& ev) { const TString& reqId = ev->Get()->RequestId; @@ -861,7 +861,7 @@ void TSqsService::HandleQueuesList(TSqsEvents::TEvQueuesList::TPtr& ev) { ++usersIt; } } - + NotifyLocalDeadLetterQueuesLeaders(ev->Get()->SortedQueues); } else { for (const auto& [userName, user] : Users_) { @@ -869,45 +869,45 @@ void TSqsService::HandleQueuesList(TSqsEvents::TEvQueuesList::TPtr& ev) { } } } - + void TSqsService::NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const { - using TKnownDeadLetterQueues = THashMap<TString, THashSet<std::pair<TString, TString>>>; - - TKnownDeadLetterQueues knownDlqs; - for (const auto& queueRecord : sortedQueues) { - if (queueRecord.DlqName) { - knownDlqs[queueRecord.UserName].insert(std::make_pair(queueRecord.DlqName, queueRecord.FolderId)); // account -> custom name + folder id pair - } - } - - for (const auto& [account, dlqs] : knownDlqs) { - auto accountIt = Users_.find(account); - if (accountIt != Users_.end()) { - for (const auto& customNameAndFolderPair : dlqs) { - auto queueInfoIt = accountIt->second->QueueByNameAndFolder_.find(customNameAndFolderPair); - if (queueInfoIt != accountIt->second->QueueByNameAndFolder_.end()) { - const auto& queueInfo = *queueInfoIt->second; + using TKnownDeadLetterQueues = THashMap<TString, THashSet<std::pair<TString, TString>>>; + + TKnownDeadLetterQueues knownDlqs; + for (const auto& queueRecord : sortedQueues) { + if (queueRecord.DlqName) { + knownDlqs[queueRecord.UserName].insert(std::make_pair(queueRecord.DlqName, queueRecord.FolderId)); // account -> custom name + folder id pair + } + } + + for (const auto& [account, dlqs] : knownDlqs) { + auto accountIt = Users_.find(account); + if (accountIt != Users_.end()) { + for (const auto& customNameAndFolderPair : dlqs) { + auto queueInfoIt = accountIt->second->QueueByNameAndFolder_.find(customNameAndFolderPair); + if (queueInfoIt != accountIt->second->QueueByNameAndFolder_.end()) { + const auto& queueInfo = *queueInfoIt->second; if (queueInfo.LocalLeader_) { Send(queueInfo.LocalLeader_, new TSqsEvents::TEvDeadLetterQueueNotification); - } - } - } - } - } -} - + } + } + } + } + } +} + void TSqsService::AnswerCountQueuesRequests(const TUserInfoPtr& user) { while (!user->CountQueuesRequests_.empty()) { const TString folderId = user->CountQueuesRequests_.begin()->first; const auto queuesCount = user->CountQueuesInFolder(folderId); - + auto requests = user->CountQueuesRequests_.equal_range(folderId); - + for (auto i = requests.first; i != requests.second; ++i) { auto& req = i->second; Send(req->Sender, new TSqsEvents::TEvCountQueuesResponse(false, true, queuesCount)); - } - + } + user->CountQueuesRequests_.erase(requests.first, requests.second); } } @@ -1034,8 +1034,8 @@ std::map<TString, TSqsService::TQueueInfoPtr>::iterator TSqsService::AddQueue(co ui64 leaderTabletId, const TString& customName, const TString& folderId, - const ui64 version, - const ui64 shardsCount, + const ui64 version, + const ui64 shardsCount, const TInstant createdTimestamp) { auto user = MutableUser(userName, false); // don't move requests because they are already moved in our caller const TInstant now = TActivationContext::Now(); @@ -1079,12 +1079,12 @@ std::map<TString, TSqsService::TQueueInfoPtr>::iterator TSqsService::AddQueue(co } { - auto requests = user->GetQueueFolderIdAndCustomNameRequests_.equal_range(queue); + auto requests = user->GetQueueFolderIdAndCustomNameRequests_.equal_range(queue); for (auto i = requests.first; i != requests.second; ++i) { auto& req = i->second; Answer(req, queueInfo); } - user->GetQueueFolderIdAndCustomNameRequests_.erase(requests.first, requests.second); + user->GetQueueFolderIdAndCustomNameRequests_.erase(requests.first, requests.second); } queueInfo->ConnectToLeaderTablet(); @@ -1209,8 +1209,8 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev) { GetQueueIdRequests_.emplace(ev->Get()->UserName, std::move(ev)); } -void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev) { - GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->UserName, std::move(ev)); +void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev) { + GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->UserName, std::move(ev)); } void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev) { @@ -1221,16 +1221,16 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueReque GetLeaderNodeRequests_.emplace(ev->Get()->UserName, std::move(ev)); } -void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev) { - CountQueuesRequests_.emplace(ev->Get()->UserName, std::move(ev)); -} - +void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev) { + CountQueuesRequests_.emplace(ev->Get()->UserName, std::move(ev)); +} + void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev, const TUserInfoPtr& userInfo) { userInfo->GetQueueIdRequests_.emplace(std::make_pair(ev->Get()->CustomQueueName, ev->Get()->FolderId), std::move(ev)); } -void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo) { - userInfo->GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->QueueName, std::move(ev)); +void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo) { + userInfo->GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->QueueName, std::move(ev)); } void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev, const TUserInfoPtr& userInfo) { @@ -1241,10 +1241,10 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueReque userInfo->GetLeaderNodeRequests_.emplace(ev->Get()->QueueName, std::move(ev)); } -void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo) { - userInfo->CountQueuesRequests_.emplace(ev->Get()->FolderId, std::move(ev)); -} - +void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo) { + userInfo->CountQueuesRequests_.emplace(ev->Get()->FolderId, std::move(ev)); +} + template <class TMultimap> size_t TSqsService::MoveUserRequests(const TUserInfoPtr& userInfo, TMultimap& map) { size_t moved = 0; diff --git a/ydb/core/ymq/actor/service.h b/ydb/core/ymq/actor/service.h index caf091c24b..5de1b7f2f0 100644 --- a/ydb/core/ymq/actor/service.h +++ b/ydb/core/ymq/actor/service.h @@ -72,7 +72,7 @@ private: TUserInfoPtr MutableUser(const TString& userName, bool moveUserRequestsToUserRecord = true, bool* requestsWereMoved = nullptr); void RemoveUser(const TString& userName); std::map<TString, TQueueInfoPtr>::iterator AddQueue(const TString& userName, const TString& queue, ui64 leaderTabletId, - const TString& customName, const TString& folderId, const ui64 version, + const TString& customName, const TString& folderId, const ui64 version, const ui64 shardsCount, const TInstant createdTimestamp); void AnswerNoUserToRequests(); @@ -91,16 +91,16 @@ private: TUserInfoPtr GetUserOrWait(TAutoPtr<TEvent>& ev); void InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev); - void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev); + void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev); void InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev); void InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr&& ev); - void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev); + void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev); void InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev, const TUserInfoPtr& userInfo); - void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo); + void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo); void InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev, const TUserInfoPtr& userInfo); void InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr&& ev, const TUserInfoPtr& userInfo); - void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo); + void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo); template <class TMultimap> size_t MoveUserRequests(const TUserInfoPtr& userInfo, TMultimap& map); // returns moved requests count @@ -128,7 +128,7 @@ private: void AnswerCountQueuesRequests(const TUserInfoPtr& user); void NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const; - + void MakeAndRegisterYcEventsProcessor(); private: @@ -156,8 +156,8 @@ private: THashMultiMap<TString, TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr> GetLeaderNodeRequests_; // user name -> request THashMultiMap<TString, TSqsEvents::TEvGetConfiguration::TPtr> GetConfigurationRequests_; // user name -> request THashMultiMap<TString, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // user name -> request - THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request - THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request + THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request + THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request struct TYcSearchEventsConfig { diff --git a/ydb/core/ymq/actor/serviceid.h b/ydb/core/ymq/actor/serviceid.h index 7219d918ba..a757bb69c1 100644 --- a/ydb/core/ymq/actor/serviceid.h +++ b/ydb/core/ymq/actor/serviceid.h @@ -4,7 +4,7 @@ #include <library/cpp/actors/core/actor.h> #include <library/cpp/logger/file.h> - + #include <util/generic/strbuf.h> namespace NKikimr::NSQS { @@ -21,16 +21,16 @@ inline TActorId MakeSqsProxyServiceID(ui32 nodeId) { inline TActorId MakeSqsAccessServiceID() { return TActorId(0, TStringBuf("SQS_ACCESS")); -} - +} + inline TActorId MakeSqsFolderServiceID() { return TActorId(0, TStringBuf("SQS_FOLDER")); -} - +} + inline TActorId MakeSqsMeteringServiceID() { return TActorId(0, TStringBuf("SQS_METER")); -} - +} + IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing()); IActor* CreateSqsProxyService(); IActor* CreateSqsAccessService(const TString& address, const TString& pathToRootCA); diff --git a/ydb/core/ymq/actor/set_queue_attributes.cpp b/ydb/core/ymq/actor/set_queue_attributes.cpp index df33ad99e6..b3e0f8a8a1 100644 --- a/ydb/core/ymq/actor/set_queue_attributes.cpp +++ b/ydb/core/ymq/actor/set_queue_attributes.cpp @@ -11,7 +11,7 @@ #include <ydb/public/lib/value/value.h> #include <library/cpp/scheme/scheme.h> - + #include <util/generic/maybe.h> #include <util/generic/utility.h> #include <util/string/cast.h> @@ -33,7 +33,7 @@ public: for (const auto& attr : Request().attributes()) { Attributes_[attr.GetName()] = attr.GetValue(); } - + CopySecurityToken(Request()); } @@ -53,9 +53,9 @@ private: const bool clampValues = !Cfg().GetEnableQueueAttributesValidation(); ValidatedAttributes_ = TQueueAttributes::FromAttributesAndConfig(Attributes_, Cfg(), IsFifoQueue(), clampValues); - if (!ValidatedAttributes_.Validate()) { - MakeError(Response_.MutableSetQueueAttributes(), *ValidatedAttributes_.Error, ValidatedAttributes_.ErrorText); - return false; + if (!ValidatedAttributes_.Validate()) { + MakeError(Response_.MutableSetQueueAttributes(), *ValidatedAttributes_.Error, ValidatedAttributes_.ErrorText); + return false; } return true; @@ -75,18 +75,18 @@ private: .Counters(QueueCounters_) .RetryOnTimeout(); - builder.Params().OptionalUint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount); - builder.Params().OptionalUtf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn); - builder.Params().OptionalUtf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName); - - builder.Params().OptionalUint64("VISIBILITY", ToMilliSeconds(ValidatedAttributes_.VisibilityTimeout)); - builder.Params().OptionalUint64("DELAY", ToMilliSeconds(ValidatedAttributes_.DelaySeconds)); - builder.Params().OptionalUint64("RETENTION", ToMilliSeconds(ValidatedAttributes_.MessageRetentionPeriod)); - builder.Params().OptionalUint64("WAIT", ToMilliSeconds(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds)); - builder.Params().OptionalUint64("MAX_MESSAGE_SIZE", ValidatedAttributes_.MaximumMessageSize); + builder.Params().OptionalUint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount); + builder.Params().OptionalUtf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn); + builder.Params().OptionalUtf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName); + + builder.Params().OptionalUint64("VISIBILITY", ToMilliSeconds(ValidatedAttributes_.VisibilityTimeout)); + builder.Params().OptionalUint64("DELAY", ToMilliSeconds(ValidatedAttributes_.DelaySeconds)); + builder.Params().OptionalUint64("RETENTION", ToMilliSeconds(ValidatedAttributes_.MessageRetentionPeriod)); + builder.Params().OptionalUint64("WAIT", ToMilliSeconds(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds)); + builder.Params().OptionalUint64("MAX_MESSAGE_SIZE", ValidatedAttributes_.MaximumMessageSize); if (IsFifoQueue()) { - builder.Params().OptionalBool("CONTENT_BASED_DEDUPLICATION", ValidatedAttributes_.ContentBasedDeduplication); + builder.Params().OptionalBool("CONTENT_BASED_DEDUPLICATION", ValidatedAttributes_.ContentBasedDeduplication); } builder.Params().Utf8("USER_NAME", UserName_); @@ -95,19 +95,19 @@ private: } void DoAction() override { - Become(&TThis::StateFunc); - - if (ValidatedAttributes_.HasClampedAttributes()) { + Become(&TThis::StateFunc); + + if (ValidatedAttributes_.HasClampedAttributes()) { RLOG_SQS_WARN("Clamped some queue attribute values for account " << UserName_ << " and queue name " << GetQueueName()); - } - - if (ValidatedAttributes_.RedrivePolicy.TargetQueueName && *ValidatedAttributes_.RedrivePolicy.TargetQueueName) { + } + + if (ValidatedAttributes_.RedrivePolicy.TargetQueueName && *ValidatedAttributes_.RedrivePolicy.TargetQueueName) { Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueId(RequestId_, UserName_, *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_)); - } else { + } else { RequestAttributesChange(); - } - } - + } + } + TString DoGetQueueName() const override { return Request().GetQueueName(); } @@ -121,21 +121,21 @@ private: } void HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) { - if (ev->Get()->Failed) { + if (ev->Get()->Failed) { RLOG_SQS_WARN("Get queue id failed"); - MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); - } else if (!ev->Get()->Exists) { + MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE); + } else if (!ev->Get()->Exists) { MakeError(MutableErrorDesc(), NErrors::NON_EXISTENT_QUEUE, "Target DLQ does not exist."); - } else if (ev->Get()->QueueId == GetQueueName()) { - MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Using the queue itself as a dead letter queue is not allowed."); - } else { + } else if (ev->Get()->QueueId == GetQueueName()) { + MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Using the queue itself as a dead letter queue is not allowed."); + } else { RequestAttributesChange(); - return; - } - + return; + } + SendReplyAndDie(); - } - + } + void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { const auto& record = ev->Get()->Record; const ui32 status = record.GetStatus(); @@ -159,8 +159,8 @@ private: private: THashMap<TString, TString> Attributes_; - - TQueueAttributes ValidatedAttributes_; + + TQueueAttributes ValidatedAttributes_; }; IActor* CreateSetQueueAttributesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) { diff --git a/ydb/core/ymq/actor/ut/infly_ut.cpp b/ydb/core/ymq/actor/ut/infly_ut.cpp index f6fed85aff..bab1c4ae99 100644 --- a/ydb/core/ymq/actor/ut/infly_ut.cpp +++ b/ydb/core/ymq/actor/ut/infly_ut.cpp @@ -8,20 +8,20 @@ Y_UNIT_TEST_SUITE(InflyTest) { Y_UNIT_TEST(AddMessage) { TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>(); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(42)), 0); - infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); + infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 1); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(42)), 1); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(43)), 0); - infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); + infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(1)), 2); } Y_UNIT_TEST(DeleteMessage) { TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>(); - infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); - infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); + infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); + infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2); UNIT_ASSERT(!infly->Delete(5)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2); @@ -35,9 +35,9 @@ Y_UNIT_TEST_SUITE(InflyTest) { Y_UNIT_TEST(ChangeMesageVisibility) { TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>(); - infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); - infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); - infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(150), 0)); + infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0)); + infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0)); + infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(150), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 3); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(50)), 1); @@ -82,11 +82,11 @@ Y_UNIT_TEST_SUITE(InflyTest) { Y_UNIT_TEST(ReceiveMessages) { TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>(); - infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0)); - infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0)); - infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0)); - infly->Add(MakeHolder<TInflyMessage>(4ull, 0ull, TInstant::Seconds(4), 0)); - infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(5), 0)); + infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0)); + infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0)); + infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0)); + infly->Add(MakeHolder<TInflyMessage>(4ull, 0ull, TInstant::Seconds(4), 0)); + infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(5), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 5); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(50)), 0); { @@ -123,9 +123,9 @@ Y_UNIT_TEST_SUITE(InflyTest) { Y_UNIT_TEST(DeleteReceivedMessage) { TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>(); - infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0)); - infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0)); - infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0)); + infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0)); + infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0)); + infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0)); UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 3); UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(2)), 2); { diff --git a/ydb/core/ymq/actor/ya.make b/ydb/core/ymq/actor/ya.make index 070653bb57..3599c3756a 100644 --- a/ydb/core/ymq/actor/ya.make +++ b/ydb/core/ymq/actor/ya.make @@ -11,7 +11,7 @@ SRCS( attributes_md5.cpp cfg.cpp change_visibility.cpp - count_queues.cpp + count_queues.cpp create_queue.cpp create_user.cpp delete_message.cpp @@ -20,21 +20,21 @@ SRCS( error.cpp executor.cpp fifo_cleanup.cpp - garbage_collector.cpp + garbage_collector.cpp get_queue_attributes.cpp get_queue_url.cpp index_events_processor.cpp infly.cpp log.cpp - list_dead_letter_source_queues.cpp - list_permissions.cpp + list_dead_letter_source_queues.cpp + list_permissions.cpp list_queues.cpp list_users.cpp local_rate_limiter_allocator.cpp message_delay_stats.cpp - metering.cpp + metering.cpp migration.cpp - modify_permissions.cpp + modify_permissions.cpp proxy_actor.cpp purge.cpp purge_queue.cpp @@ -48,14 +48,14 @@ SRCS( set_queue_attributes.cpp proxy_service.cpp queues_list_reader.cpp - queue_schema.cpp + queue_schema.cpp user_settings_names.cpp user_settings_reader.cpp ) PEERDIR( contrib/libs/openssl - contrib/libs/protobuf + contrib/libs/protobuf library/cpp/actors/core library/cpp/containers/intrusive_rb_tree library/cpp/digest/md5 @@ -96,6 +96,6 @@ YQL_LAST_ABI_VERSION() GENERATE_ENUM_SERIALIZATION(events.h) -GENERATE_ENUM_SERIALIZATION(metering.h) +GENERATE_ENUM_SERIALIZATION(metering.h) END() diff --git a/ydb/core/ymq/base/acl.cpp b/ydb/core/ymq/base/acl.cpp index 683611e88c..0a088aebea 100644 --- a/ydb/core/ymq/base/acl.cpp +++ b/ydb/core/ymq/base/acl.cpp @@ -1,141 +1,141 @@ -#include "acl.h" - -#include <util/generic/hash.h> -#include <util/generic/singleton.h> -#include <util/generic/string.h> - -#include <map> - +#include "acl.h" + +#include <util/generic/hash.h> +#include <util/generic/singleton.h> +#include <util/generic/string.h> + +#include <map> + namespace NKikimr::NSQS { -class TSQSACLMappings { -public: - TSQSACLMappings() { - static const TStringBuf createQueue = "CreateQueue"; - static const TStringBuf createUser = "CreateUser"; - static const TStringBuf sendMessage = "SendMessage"; - static const TStringBuf receiveMessage = "ReceiveMessage"; - static const TStringBuf deleteMessage = "DeleteMessage"; - static const TStringBuf modifyPermissions = "ModifyPermissions"; - static const TStringBuf alterQueue = "AlterQueue"; - static const TStringBuf describePath = "DescribePath"; - static const TStringBuf deleteUser = "DeleteUser"; - static const TStringBuf deleteQueue = "DeleteQueue"; - - ACE2AccessIndex = { - {createQueue, NACLib::EAccessRights::CreateQueue}, - {createUser, NACLib::EAccessRights::CreateTable}, - {sendMessage, NACLib::EAccessRights::UpdateRow}, - {receiveMessage, NACLib::EAccessRights::SelectRow}, - {deleteMessage, NACLib::EAccessRights::EraseRow}, - {modifyPermissions, NACLib::EAccessRights::WriteAttributes}, - {alterQueue, NACLib::EAccessRights::AlterSchema}, - {describePath, NACLib::EAccessRights::DescribeSchema}, - {deleteUser, NACLib::EAccessRights::RemoveSchema}, - {deleteQueue, NACLib::EAccessRights::ReadAttributes}}; // as intended - - Action2ACEIndex = { - {"CreateQueue", {EACLSourceType::AccountDir, createQueue}}, - {"CreateUser", {EACLSourceType::RootDir, createUser}}, - {"SendMessage", {EACLSourceType::QueueDir, sendMessage}}, - {"SendMessageBatch", {EACLSourceType::QueueDir, sendMessage}}, - {"ChangeMessageVisibility", {EACLSourceType::QueueDir, receiveMessage}}, - {"ChangeMessageVisibilityBatch", {EACLSourceType::QueueDir, receiveMessage}}, - {"ReceiveMessage", {EACLSourceType::QueueDir, receiveMessage}}, - {"DeleteMessage", {EACLSourceType::QueueDir, deleteMessage}}, - {"DeleteMessageBatch", {EACLSourceType::QueueDir, deleteMessage}}, - {"PurgeQueue", {EACLSourceType::QueueDir, deleteMessage}}, - {"ModifyPermissions", {EACLSourceType::Custom, modifyPermissions}}, - {"SetQueueAttributes", {EACLSourceType::QueueDir, alterQueue}}, - {"TagQueue", {EACLSourceType::QueueDir, alterQueue}}, - {"UntagQueue", {EACLSourceType::QueueDir, alterQueue}}, - {"GetQueueAttributes", {EACLSourceType::QueueDir, describePath}}, - {"ListQueueTags", {EACLSourceType::QueueDir, describePath}}, - {"GetQueueUrl", {EACLSourceType::QueueDir, describePath}}, - {"ListQueues", {EACLSourceType::AccountDir, describePath}}, - {"ListDeadLetterSourceQueues", {EACLSourceType::QueueDir, describePath}}, - {"ListPermissions", {EACLSourceType::Custom, describePath}}, - {"CountQueues", {EACLSourceType::AccountDir, describePath}}, - {"DeleteUser", {EACLSourceType::AccountDir, deleteUser}}, - {"DeleteQueue", {EACLSourceType::QueueDir, deleteQueue}}}; - - for (const auto& pair : ACE2AccessIndex) { - Access2ACEIndex[pair.second] = pair.first; - } - - Y_VERIFY(Access2ACEIndex.size() == ACE2AccessIndex.size()); - } - - EACLSourceType GetActionACLSourceTypeImpl(const TStringBuf action) const { - auto it = Action2ACEIndex.find(action); - if (it != Action2ACEIndex.end()) { - return it->second.first; - } - - return EACLSourceType::Unknown; - } - - ui32 GetActionRequiredAccessImpl(const TStringBuf action) const { - return GetACERequiredAccessImpl(GetActionMatchingACEImpl(action)); - } - - ui32 GetACERequiredAccessImpl(const TStringBuf aceName) const { - auto accessIt = ACE2AccessIndex.find(aceName); - if (accessIt != ACE2AccessIndex.end()) { - return accessIt->second; - } - - return 0; - } - - TString GetActionMatchingACEImpl(const TStringBuf actionName) const { - auto it = Action2ACEIndex.find(actionName); - if (it != Action2ACEIndex.end()) { - return TString(it->second.second); - } - - return {}; - } - - TVector<TStringBuf> GetAccessMatchingACEImpl(const ui32 access) const { - TVector<TStringBuf> result; - for (const auto& pair : Access2ACEIndex) { - if (pair.first & access) { - result.push_back(pair.second); - } - } - - return result; - } - -private: - THashMap<TStringBuf, ui32> ACE2AccessIndex; - THashMap<ui32, TStringBuf> Access2ACEIndex; - THashMap<TStringBuf, std::pair<EACLSourceType, TStringBuf>> Action2ACEIndex; -}; - -static const TSQSACLMappings& Mappings() { - return *Singleton<TSQSACLMappings>(); -} - -EACLSourceType GetActionACLSourceType(const TString& actionName) { - return Mappings().GetActionACLSourceTypeImpl(actionName); -} - -ui32 GetActionRequiredAccess(const TString& actionName) { - return Mappings().GetActionRequiredAccessImpl(actionName); -} - -ui32 GetACERequiredAccess(const TString& aceName) { - return Mappings().GetACERequiredAccessImpl(aceName); -} - -TString GetActionMatchingACE(const TString& actionName) { - return Mappings().GetActionMatchingACEImpl(actionName); -} - -TVector<TStringBuf> GetAccessMatchingACE(const ui32 access) { - return Mappings().GetAccessMatchingACEImpl(access); -} +class TSQSACLMappings { +public: + TSQSACLMappings() { + static const TStringBuf createQueue = "CreateQueue"; + static const TStringBuf createUser = "CreateUser"; + static const TStringBuf sendMessage = "SendMessage"; + static const TStringBuf receiveMessage = "ReceiveMessage"; + static const TStringBuf deleteMessage = "DeleteMessage"; + static const TStringBuf modifyPermissions = "ModifyPermissions"; + static const TStringBuf alterQueue = "AlterQueue"; + static const TStringBuf describePath = "DescribePath"; + static const TStringBuf deleteUser = "DeleteUser"; + static const TStringBuf deleteQueue = "DeleteQueue"; + + ACE2AccessIndex = { + {createQueue, NACLib::EAccessRights::CreateQueue}, + {createUser, NACLib::EAccessRights::CreateTable}, + {sendMessage, NACLib::EAccessRights::UpdateRow}, + {receiveMessage, NACLib::EAccessRights::SelectRow}, + {deleteMessage, NACLib::EAccessRights::EraseRow}, + {modifyPermissions, NACLib::EAccessRights::WriteAttributes}, + {alterQueue, NACLib::EAccessRights::AlterSchema}, + {describePath, NACLib::EAccessRights::DescribeSchema}, + {deleteUser, NACLib::EAccessRights::RemoveSchema}, + {deleteQueue, NACLib::EAccessRights::ReadAttributes}}; // as intended + + Action2ACEIndex = { + {"CreateQueue", {EACLSourceType::AccountDir, createQueue}}, + {"CreateUser", {EACLSourceType::RootDir, createUser}}, + {"SendMessage", {EACLSourceType::QueueDir, sendMessage}}, + {"SendMessageBatch", {EACLSourceType::QueueDir, sendMessage}}, + {"ChangeMessageVisibility", {EACLSourceType::QueueDir, receiveMessage}}, + {"ChangeMessageVisibilityBatch", {EACLSourceType::QueueDir, receiveMessage}}, + {"ReceiveMessage", {EACLSourceType::QueueDir, receiveMessage}}, + {"DeleteMessage", {EACLSourceType::QueueDir, deleteMessage}}, + {"DeleteMessageBatch", {EACLSourceType::QueueDir, deleteMessage}}, + {"PurgeQueue", {EACLSourceType::QueueDir, deleteMessage}}, + {"ModifyPermissions", {EACLSourceType::Custom, modifyPermissions}}, + {"SetQueueAttributes", {EACLSourceType::QueueDir, alterQueue}}, + {"TagQueue", {EACLSourceType::QueueDir, alterQueue}}, + {"UntagQueue", {EACLSourceType::QueueDir, alterQueue}}, + {"GetQueueAttributes", {EACLSourceType::QueueDir, describePath}}, + {"ListQueueTags", {EACLSourceType::QueueDir, describePath}}, + {"GetQueueUrl", {EACLSourceType::QueueDir, describePath}}, + {"ListQueues", {EACLSourceType::AccountDir, describePath}}, + {"ListDeadLetterSourceQueues", {EACLSourceType::QueueDir, describePath}}, + {"ListPermissions", {EACLSourceType::Custom, describePath}}, + {"CountQueues", {EACLSourceType::AccountDir, describePath}}, + {"DeleteUser", {EACLSourceType::AccountDir, deleteUser}}, + {"DeleteQueue", {EACLSourceType::QueueDir, deleteQueue}}}; + + for (const auto& pair : ACE2AccessIndex) { + Access2ACEIndex[pair.second] = pair.first; + } + + Y_VERIFY(Access2ACEIndex.size() == ACE2AccessIndex.size()); + } + + EACLSourceType GetActionACLSourceTypeImpl(const TStringBuf action) const { + auto it = Action2ACEIndex.find(action); + if (it != Action2ACEIndex.end()) { + return it->second.first; + } + + return EACLSourceType::Unknown; + } + + ui32 GetActionRequiredAccessImpl(const TStringBuf action) const { + return GetACERequiredAccessImpl(GetActionMatchingACEImpl(action)); + } + + ui32 GetACERequiredAccessImpl(const TStringBuf aceName) const { + auto accessIt = ACE2AccessIndex.find(aceName); + if (accessIt != ACE2AccessIndex.end()) { + return accessIt->second; + } + + return 0; + } + + TString GetActionMatchingACEImpl(const TStringBuf actionName) const { + auto it = Action2ACEIndex.find(actionName); + if (it != Action2ACEIndex.end()) { + return TString(it->second.second); + } + + return {}; + } + + TVector<TStringBuf> GetAccessMatchingACEImpl(const ui32 access) const { + TVector<TStringBuf> result; + for (const auto& pair : Access2ACEIndex) { + if (pair.first & access) { + result.push_back(pair.second); + } + } + + return result; + } + +private: + THashMap<TStringBuf, ui32> ACE2AccessIndex; + THashMap<ui32, TStringBuf> Access2ACEIndex; + THashMap<TStringBuf, std::pair<EACLSourceType, TStringBuf>> Action2ACEIndex; +}; + +static const TSQSACLMappings& Mappings() { + return *Singleton<TSQSACLMappings>(); +} + +EACLSourceType GetActionACLSourceType(const TString& actionName) { + return Mappings().GetActionACLSourceTypeImpl(actionName); +} + +ui32 GetActionRequiredAccess(const TString& actionName) { + return Mappings().GetActionRequiredAccessImpl(actionName); +} + +ui32 GetACERequiredAccess(const TString& aceName) { + return Mappings().GetACERequiredAccessImpl(aceName); +} + +TString GetActionMatchingACE(const TString& actionName) { + return Mappings().GetActionMatchingACEImpl(actionName); +} + +TVector<TStringBuf> GetAccessMatchingACE(const ui32 access) { + return Mappings().GetAccessMatchingACEImpl(access); +} } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/acl.h b/ydb/core/ymq/base/acl.h index 18c3f07a39..494881156c 100644 --- a/ydb/core/ymq/base/acl.h +++ b/ydb/core/ymq/base/acl.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include <ydb/library/aclib/aclib.h> - + namespace NKikimr::NSQS { -enum class EACLSourceType : ui32 { - Unknown, - RootDir, - AccountDir, - QueueDir, - Custom -}; - -EACLSourceType GetActionACLSourceType(const TString& actionName); -ui32 GetActionRequiredAccess(const TString& actionName); -ui32 GetACERequiredAccess(const TString& aceName); -TString GetActionMatchingACE(const TString& actionName); -TVector<TStringBuf> GetAccessMatchingACE(const ui32 access); +enum class EACLSourceType : ui32 { + Unknown, + RootDir, + AccountDir, + QueueDir, + Custom +}; + +EACLSourceType GetActionACLSourceType(const TString& actionName); +ui32 GetActionRequiredAccess(const TString& actionName); +ui32 GetACERequiredAccess(const TString& aceName); +TString GetActionMatchingACE(const TString& actionName); +TVector<TStringBuf> GetAccessMatchingACE(const ui32 access); } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/action.h b/ydb/core/ymq/base/action.h index ede7922dd1..decbb1775c 100644 --- a/ydb/core/ymq/base/action.h +++ b/ydb/core/ymq/base/action.h @@ -27,9 +27,9 @@ enum EAction { SendMessageBatch, SetQueueAttributes, ModifyPermissions, - ListPermissions, - ListDeadLetterSourceQueues, - CountQueues, + ListPermissions, + ListDeadLetterSourceQueues, + CountQueues, ActionsArraySize, }; @@ -63,7 +63,7 @@ EAction GetNonBatchAction(EAction action); macro(ReceiveMessage) \ macro(SendMessage) \ macro(SendMessageBatch) \ - macro(ListDeadLetterSourceQueues) \ + macro(ListDeadLetterSourceQueues) \ macro(SetQueueAttributes) } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/cloud_enums.h b/ydb/core/ymq/base/cloud_enums.h index d0adc427e1..2282ea049b 100644 --- a/ydb/core/ymq/base/cloud_enums.h +++ b/ydb/core/ymq/base/cloud_enums.h @@ -1,16 +1,16 @@ -#pragma once - -namespace NKikimr::NSQS::NCloudAuth { - enum EActionType { - Authenticate = 0, - Authorize, - GetCloudId, - ActionTypesCount - }; - - enum ECredentialType { - IamToken = 0, - Signature, - CredentialTypesCount - }; -} // namespace NKikimr::NSQS::NCloudAuth +#pragma once + +namespace NKikimr::NSQS::NCloudAuth { + enum EActionType { + Authenticate = 0, + Authorize, + GetCloudId, + ActionTypesCount + }; + + enum ECredentialType { + IamToken = 0, + Signature, + CredentialTypesCount + }; +} // namespace NKikimr::NSQS::NCloudAuth diff --git a/ydb/core/ymq/base/constants.h b/ydb/core/ymq/base/constants.h index 670e427ff8..c33488e62c 100644 --- a/ydb/core/ymq/base/constants.h +++ b/ydb/core/ymq/base/constants.h @@ -1,8 +1,8 @@ #pragma once #include <cstddef> -#include <util/generic/string.h> - +#include <util/generic/string.h> + #define INFLY_LIMIT 120000 namespace NKikimr::NSQS { @@ -12,5 +12,5 @@ constexpr size_t MAX_PARTITIONS_COUNT = 128; static const TString yaSqsArnPrefix = "yrn:ya:sqs"; static const TString cloudArnPrefix = "yrn:yc:ymq"; - + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp index 90a9f5000f..9b7db90966 100644 --- a/ydb/core/ymq/base/counters.cpp +++ b/ydb/core/ymq/base/counters.cpp @@ -298,8 +298,8 @@ static_assert(AbsDiffLessThanCounter(HttpUserCountersDescriptor.SizeOfCounters() sizeof(NKikimrConfig::TSqsConfig*) + sizeof(TIntrusivePtr<THttpUserCounters>), sizeof(THttpUserCounters))); -TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) { - return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup); +TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) { + return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup); } TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) { // Remove subgroup and don't have subsystem (is this correct - ?) @@ -605,7 +605,7 @@ void TQueueCounters::InitCounters(bool forLeaderNode) { ); INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart); - + INIT_COUNTERS_COUPLE( QueueCounters, SendMessage_DeduplicationCount, deduplicated_count_per_second, @@ -881,63 +881,63 @@ void THttpActionCounters::SetAggregatedParent(THttpActionCounters* parent) { HttpActionCountersDescriptor.SetAggregatedParent(this, parent); } -static const TString& StringifyGrpcStatus(int grpcStatus) { - if (grpcStatus < 0 || grpcStatus > TCloudAuthCounters::GRPC_STATUSES_COUNT - 2) { - grpcStatus = TCloudAuthCounters::GRPC_STATUSES_COUNT - 1; - } - - static const TString statusStrings[] = { - "Ok", - "Cancelled", - "Unknown", - "InvalidArgument", - "DeadlineExceeded", - "NotFound", - "AlreadyExists", - "PermissionDenied", - "ResourceExhausted", - "FailedPrecondition", - "Aborted", - "OutOfRange", - "Unimplemented", - "Internal", - "Unavailable", - "DataLoss", - "Unauthenticated", - "Misc", - }; - - static_assert(Y_ARRAY_SIZE(statusStrings) == TCloudAuthCounters::GRPC_STATUSES_COUNT); - - return statusStrings[grpcStatus]; -} - -void TCloudAuthCounters::IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus) { - if (grpcStatus < 0 || grpcStatus > GRPC_STATUSES_COUNT - 2) { - grpcStatus = GRPC_STATUSES_COUNT - 1; - } - - ++*CloudAuthCounters[actionType][credentialType][grpcStatus]; -} - -void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters) { - for (size_t actionType = 0; actionType < NCloudAuth::EActionType::ActionTypesCount; ++actionType) { - const auto actionTypeStr = ToString(static_cast<NCloudAuth::EActionType>(actionType)); - const auto actionCounters = cloudAuthCounters->GetSubgroup("action_type", actionTypeStr); - for (size_t credentialType = 0; credentialType < NCloudAuth::ECredentialType::CredentialTypesCount; ++credentialType) { - const auto credentialTypeStr = ToString(static_cast<NCloudAuth::ECredentialType>(credentialType)); - const auto actionAndCredentialCounters = actionCounters->GetSubgroup("credential_type", credentialTypeStr); - for (size_t grpcStatus = 0; grpcStatus < GRPC_STATUSES_COUNT; ++grpcStatus) { +static const TString& StringifyGrpcStatus(int grpcStatus) { + if (grpcStatus < 0 || grpcStatus > TCloudAuthCounters::GRPC_STATUSES_COUNT - 2) { + grpcStatus = TCloudAuthCounters::GRPC_STATUSES_COUNT - 1; + } + + static const TString statusStrings[] = { + "Ok", + "Cancelled", + "Unknown", + "InvalidArgument", + "DeadlineExceeded", + "NotFound", + "AlreadyExists", + "PermissionDenied", + "ResourceExhausted", + "FailedPrecondition", + "Aborted", + "OutOfRange", + "Unimplemented", + "Internal", + "Unavailable", + "DataLoss", + "Unauthenticated", + "Misc", + }; + + static_assert(Y_ARRAY_SIZE(statusStrings) == TCloudAuthCounters::GRPC_STATUSES_COUNT); + + return statusStrings[grpcStatus]; +} + +void TCloudAuthCounters::IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus) { + if (grpcStatus < 0 || grpcStatus > GRPC_STATUSES_COUNT - 2) { + grpcStatus = GRPC_STATUSES_COUNT - 1; + } + + ++*CloudAuthCounters[actionType][credentialType][grpcStatus]; +} + +void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters) { + for (size_t actionType = 0; actionType < NCloudAuth::EActionType::ActionTypesCount; ++actionType) { + const auto actionTypeStr = ToString(static_cast<NCloudAuth::EActionType>(actionType)); + const auto actionCounters = cloudAuthCounters->GetSubgroup("action_type", actionTypeStr); + for (size_t credentialType = 0; credentialType < NCloudAuth::ECredentialType::CredentialTypesCount; ++credentialType) { + const auto credentialTypeStr = ToString(static_cast<NCloudAuth::ECredentialType>(credentialType)); + const auto actionAndCredentialCounters = actionCounters->GetSubgroup("credential_type", credentialTypeStr); + for (size_t grpcStatus = 0; grpcStatus < GRPC_STATUSES_COUNT; ++grpcStatus) { INIT_COUNTER_WITH_NAME(actionAndCredentialCounters, CloudAuthCounters[actionType][credentialType][grpcStatus], StringifyGrpcStatus(grpcStatus), ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg)); - } - } - } + } + } + } INIT_HISTOGRAM_COUNTER(cloudAuthCounters, AuthenticateDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg)); INIT_HISTOGRAM_COUNTER(cloudAuthCounters, AuthorizeDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg)); INIT_HISTOGRAM_COUNTER(cloudAuthCounters, GetFolderIdDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg)); -} - +} + void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) { const TString classifierRequestsLabel("ClassifierRequests_"); const TString idleClassifierRequestsLabel("IdleClassifierRequests_"); diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h index 3c2667b241..841afdc99f 100644 --- a/ydb/core/ymq/base/counters.h +++ b/ydb/core/ymq/base/counters.h @@ -1,5 +1,5 @@ #pragma once - + #include <ydb/core/protos/config.pb.h> #include <ydb/core/ymq/base/action.h> @@ -22,7 +22,7 @@ namespace NKikimr::NSQS { struct TUserCounters; struct TQueueCounters; struct THttpCounters; -struct TCloudAuthCounters; +struct TCloudAuthCounters; #define INC_COUNTER(countersPack, counter) \ if (countersPack) { \ @@ -587,7 +587,7 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> { TLazyCachedCounter received_bytes_per_second; TLazyCachedCounter MessagesMovedToDLQ; // Count of messages that were moved to DLQ. - + TLazyCachedCounter SendMessage_DeduplicationCount; // Count of messages that were deduplicated (for fifo only). TLazyCachedCounter deduplicated_count_per_second; TLazyCachedCounter SendMessage_Count; // Count of messages that were sent. // User metric for cloud console. @@ -762,7 +762,7 @@ private: }; // Cloud specific counters. -struct TCloudAuthCounters { +struct TCloudAuthCounters { // Durations for different security actions types. TLazyCachedHistogram AuthenticateDuration; // Histogram with buckets for durations (== 18 counters). TLazyCachedHistogram AuthorizeDuration; // Histogram with buckets for durations (== 18 counters). @@ -771,23 +771,23 @@ struct TCloudAuthCounters { explicit TCloudAuthCounters(const NKikimrConfig::TSqsConfig& cfg, TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCountersRoot) : Cfg(&cfg) { - InitCounters(std::move(cloudAuthCountersRoot)); - } - - void IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus); - - static constexpr int GRPC_STATUSES_COUNT = 18; - -private: - void InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters); - -private: + InitCounters(std::move(cloudAuthCountersRoot)); + } + + void IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus); + + static constexpr int GRPC_STATUSES_COUNT = 18; + +private: + void InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters); + +private: const NKikimrConfig::TSqsConfig* Cfg = nullptr; TLazyCachedCounter CloudAuthCounters[NCloudAuth::EActionType::ActionTypesCount] // 3 types. [NCloudAuth::ECredentialType::CredentialTypesCount] // 2 types. [GRPC_STATUSES_COUNT]; // 18 types. -}; - +}; + // Metering counters in SQS core subsystem. struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> { THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults; diff --git a/ydb/core/ymq/base/dlq_helpers.cpp b/ydb/core/ymq/base/dlq_helpers.cpp index a2ba9fac42..a59dcb417c 100644 --- a/ydb/core/ymq/base/dlq_helpers.cpp +++ b/ydb/core/ymq/base/dlq_helpers.cpp @@ -1,105 +1,105 @@ -#include "dlq_helpers.h" - +#include "dlq_helpers.h" + #include <ydb/core/ymq/base/constants.h> #include <ydb/core/ymq/base/limits.h> #include <library/cpp/scheme/scheme.h> - + #include <util/string/split.h> -#include <util/string/join.h> - -namespace NKikimr::NSQS { - -static const TStringBuf maxReceiveCount = "maxReceiveCount"; -static const TStringBuf deadLetterTargetArn = "deadLetterTargetArn"; - -static bool IsValidArn(const TString& arn, const NKikimrConfig::TSqsConfig& config) { - TVector<TStringBuf> items; - StringSplitter(arn).Split(':').AddTo(&items); - if (items.size() != 6) { - return false; - } - - // prefix - const TString& requiredPrefix = config.GetYandexCloudMode() ? cloudArnPrefix : yaSqsArnPrefix; - if (!arn.StartsWith(requiredPrefix)) { - return false; - } - - // region - if (items[3] != config.GetYandexCloudServiceRegion()) { - return false; - } - - return items[4] && items[5]; // account | folder id and queue name -} - -TRedrivePolicy TRedrivePolicy::FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config) { - - TRedrivePolicy policy; - const NSc::TValue validatedRedrivePolicy = NSc::TValue::FromJson(json); - if (validatedRedrivePolicy.IsDict() && - validatedRedrivePolicy.Has(maxReceiveCount) && (validatedRedrivePolicy[maxReceiveCount].IsString() || validatedRedrivePolicy[maxReceiveCount].IsIntNumber()) && - validatedRedrivePolicy.Has(deadLetterTargetArn) && validatedRedrivePolicy[deadLetterTargetArn].IsString()) - { - size_t maxReceiveCountValue = 0; - if (TryFromString(validatedRedrivePolicy[maxReceiveCount].ForceString(), maxReceiveCountValue)) { - if (maxReceiveCountValue < TLimits::MinMaxReceiveCount || maxReceiveCountValue > TLimits::MaxMaxReceiveCount) { +#include <util/string/join.h> + +namespace NKikimr::NSQS { + +static const TStringBuf maxReceiveCount = "maxReceiveCount"; +static const TStringBuf deadLetterTargetArn = "deadLetterTargetArn"; + +static bool IsValidArn(const TString& arn, const NKikimrConfig::TSqsConfig& config) { + TVector<TStringBuf> items; + StringSplitter(arn).Split(':').AddTo(&items); + if (items.size() != 6) { + return false; + } + + // prefix + const TString& requiredPrefix = config.GetYandexCloudMode() ? cloudArnPrefix : yaSqsArnPrefix; + if (!arn.StartsWith(requiredPrefix)) { + return false; + } + + // region + if (items[3] != config.GetYandexCloudServiceRegion()) { + return false; + } + + return items[4] && items[5]; // account | folder id and queue name +} + +TRedrivePolicy TRedrivePolicy::FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config) { + + TRedrivePolicy policy; + const NSc::TValue validatedRedrivePolicy = NSc::TValue::FromJson(json); + if (validatedRedrivePolicy.IsDict() && + validatedRedrivePolicy.Has(maxReceiveCount) && (validatedRedrivePolicy[maxReceiveCount].IsString() || validatedRedrivePolicy[maxReceiveCount].IsIntNumber()) && + validatedRedrivePolicy.Has(deadLetterTargetArn) && validatedRedrivePolicy[deadLetterTargetArn].IsString()) + { + size_t maxReceiveCountValue = 0; + if (TryFromString(validatedRedrivePolicy[maxReceiveCount].ForceString(), maxReceiveCountValue)) { + if (maxReceiveCountValue < TLimits::MinMaxReceiveCount || maxReceiveCountValue > TLimits::MaxMaxReceiveCount) { policy.ErrorText = "maxReceiveCount must be greater than 0 and less than 1001."; - return policy; - } - } else { + return policy; + } + } else { policy.ErrorText = "Failed to parse maxReceiveCount as an integer."; - return policy; - } - - const TString arn = validatedRedrivePolicy[deadLetterTargetArn].ForceString(); - if (!IsValidArn(arn, config)) { - policy.ErrorText = "Invalid deadLetterTargetArn."; - return policy; - } - - TStringBuf skipped, queueName; - if (!TStringBuf(arn).TryRSplit(':', skipped, queueName)) { - queueName = arn; - } - - if (!queueName) { + return policy; + } + + const TString arn = validatedRedrivePolicy[deadLetterTargetArn].ForceString(); + if (!IsValidArn(arn, config)) { + policy.ErrorText = "Invalid deadLetterTargetArn."; + return policy; + } + + TStringBuf skipped, queueName; + if (!TStringBuf(arn).TryRSplit(':', skipped, queueName)) { + queueName = arn; + } + + if (!queueName) { policy.ErrorText = "Empty dead letter target queue name."; - return policy; - } - - policy.MaxReceiveCount = maxReceiveCountValue; - policy.TargetQueueName = TString(queueName); - policy.TargetArn = TString(arn); - } else if (!json) { - // Clear redrive policy - policy.MaxReceiveCount = 0; - policy.TargetQueueName = ""; - policy.TargetArn = ""; - } else { + return policy; + } + + policy.MaxReceiveCount = maxReceiveCountValue; + policy.TargetQueueName = TString(queueName); + policy.TargetArn = TString(arn); + } else if (!json) { + // Clear redrive policy + policy.MaxReceiveCount = 0; + policy.TargetQueueName = ""; + policy.TargetArn = ""; + } else { policy.ErrorText = "RedrivePolicy attribute is ill-constructed."; - } - - return policy; -} - -TString TRedrivePolicy::ToJson() const { - if (MaxReceiveCount && TargetArn) { - NSc::TValue policy; - policy.SetDict(); - policy[maxReceiveCount] = *MaxReceiveCount; - policy[deadLetterTargetArn] = *TargetArn; - return policy.ToJson(); - } - return {}; -} - -bool TRedrivePolicy::IsValid() const { - return ErrorText.empty(); -} - -const TString& TRedrivePolicy::GetErrorText() const { - return ErrorText; -} - -} // namespace NKikimr::NSQS + } + + return policy; +} + +TString TRedrivePolicy::ToJson() const { + if (MaxReceiveCount && TargetArn) { + NSc::TValue policy; + policy.SetDict(); + policy[maxReceiveCount] = *MaxReceiveCount; + policy[deadLetterTargetArn] = *TargetArn; + return policy.ToJson(); + } + return {}; +} + +bool TRedrivePolicy::IsValid() const { + return ErrorText.empty(); +} + +const TString& TRedrivePolicy::GetErrorText() const { + return ErrorText; +} + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/dlq_helpers.h b/ydb/core/ymq/base/dlq_helpers.h index 9c2e1d292b..968eb87f73 100644 --- a/ydb/core/ymq/base/dlq_helpers.h +++ b/ydb/core/ymq/base/dlq_helpers.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/config.pb.h> - -#include <util/generic/maybe.h> -#include <util/generic/string.h> - -namespace NKikimr::NSQS { - -struct TRedrivePolicy { - static TRedrivePolicy FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config); - - TString ToJson() const; - bool IsValid() const; - const TString& GetErrorText() const; - - TString ErrorText; - TMaybe<size_t> MaxReceiveCount; - TMaybe<TString> TargetArn; - TMaybe<TString> TargetQueueName; -}; - -} // namespace NKikimr::NSQS + +#include <util/generic/maybe.h> +#include <util/generic/string.h> + +namespace NKikimr::NSQS { + +struct TRedrivePolicy { + static TRedrivePolicy FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config); + + TString ToJson() const; + bool IsValid() const; + const TString& GetErrorText() const; + + TString ErrorText; + TMaybe<size_t> MaxReceiveCount; + TMaybe<TString> TargetArn; + TMaybe<TString> TargetQueueName; +}; + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/limits.h b/ydb/core/ymq/base/limits.h index b251b3ff62..6762fa9cf1 100644 --- a/ydb/core/ymq/base/limits.h +++ b/ydb/core/ymq/base/limits.h @@ -1,6 +1,6 @@ #pragma once -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/system/defaults.h> namespace NKikimr::NSQS { @@ -25,10 +25,10 @@ namespace TLimits { static constexpr size_t VisibilityTimeout = 30; static constexpr TDuration MaxVisibilityTimeout = TDuration::Hours(12); - - static constexpr size_t MinMaxReceiveCount = 1; - - static constexpr size_t MaxMaxReceiveCount = 1000; + + static constexpr size_t MinMaxReceiveCount = 1; + + static constexpr size_t MaxMaxReceiveCount = 1000; }; } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/processed_request_attributes.h b/ydb/core/ymq/base/processed_request_attributes.h index 9a5169b3ab..2892b899ee 100644 --- a/ydb/core/ymq/base/processed_request_attributes.h +++ b/ydb/core/ymq/base/processed_request_attributes.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/core/ymq/base/action.h> - -#include <util/generic/string.h> - -namespace NKikimr::NSQS { - -struct TProcessedRequestAttributes { - TProcessedRequestAttributes() = default; - TProcessedRequestAttributes(TProcessedRequestAttributes&& rhs) = default; - - bool IsFifo = false; - int HttpStatusCode = 0; - - ui64 RequestSizeInBytes = 0; - ui64 ResponseSizeInBytes = 0; - - TString FolderId; - TString ResourceId; - TString SourceAddress; - - EAction Action; -}; - -} // namespace NKikimr::NSQS + +#include <util/generic/string.h> + +namespace NKikimr::NSQS { + +struct TProcessedRequestAttributes { + TProcessedRequestAttributes() = default; + TProcessedRequestAttributes(TProcessedRequestAttributes&& rhs) = default; + + bool IsFifo = false; + int HttpStatusCode = 0; + + ui64 RequestSizeInBytes = 0; + ui64 ResponseSizeInBytes = 0; + + TString FolderId; + TString ResourceId; + TString SourceAddress; + + EAction Action; +}; + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/query_id.h b/ydb/core/ymq/base/query_id.h index ad54d0c70d..d313c432f8 100644 --- a/ydb/core/ymq/base/query_id.h +++ b/ydb/core/ymq/base/query_id.h @@ -25,9 +25,9 @@ enum EQueryId { LOAD_INFLY_ID = 17, ADD_MESSAGES_TO_INFLY_ID = 18, GET_STATE_ID = 19, - LIST_DEAD_LETTER_SOURCE_QUEUES_ID = 20, - LOAD_OR_REDRIVE_MESSAGE_ID = 21, - READ_OR_REDRIVE_MESSAGE_ID = 22, + LIST_DEAD_LETTER_SOURCE_QUEUES_ID = 20, + LOAD_OR_REDRIVE_MESSAGE_ID = 21, + READ_OR_REDRIVE_MESSAGE_ID = 22, GET_USER_SETTINGS_ID = 23, GET_QUEUES_LIST_ID = 24, diff --git a/ydb/core/ymq/base/queue_attributes.cpp b/ydb/core/ymq/base/queue_attributes.cpp index 8c6436aba9..4f71ba1d47 100644 --- a/ydb/core/ymq/base/queue_attributes.cpp +++ b/ydb/core/ymq/base/queue_attributes.cpp @@ -1,115 +1,115 @@ -#include "limits.h" -#include "queue_attributes.h" - -#include <util/generic/utility.h> -#include <util/string/cast.h> -#include <util/string/printf.h> - -namespace NKikimr::NSQS { - -TQueueAttributes TQueueAttributes::FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp) { - TQueueAttributes result; - -#define INVALID_PARAM_MSG "Invalid value for the parameter %s." - -#define TRY_SET_ATTRIBUTE(NAME, MIN_VAL, MAX_VAL) \ - if (name == Y_STRINGIZE(NAME)) { \ - if (!result.TryParseLimitedValue(Y_STRINGIZE(NAME), \ - value, \ - MIN_VAL, \ - MAX_VAL, \ - result.NAME, \ - clamp)) \ - { \ - break; \ - } \ - } - - for (const auto& [name, value] : attributes) { - TRY_SET_ATTRIBUTE(VisibilityTimeout, - 0, - TLimits::MaxVisibilityTimeout.Seconds()) - else TRY_SET_ATTRIBUTE(MessageRetentionPeriod, - TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), - TLimits::MaxMessageRetentionPeriod.Seconds()) - else TRY_SET_ATTRIBUTE(ReceiveMessageWaitTimeSeconds, - 0, - TDuration::MilliSeconds(config.GetMaxWaitTimeoutMs()).Seconds()) - else TRY_SET_ATTRIBUTE(DelaySeconds, - 0, - TLimits::MaxDelaySeconds) - else TRY_SET_ATTRIBUTE(MaximumMessageSize, - 1024, - TLimits::MaxMessageSize) - else if (name == "ContentBasedDeduplication" && isFifoQueue) { - if (value == "true") { - result.ContentBasedDeduplication = true; - } else if (value == "false") { - result.ContentBasedDeduplication = false; - } else { - result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE; +#include "limits.h" +#include "queue_attributes.h" + +#include <util/generic/utility.h> +#include <util/string/cast.h> +#include <util/string/printf.h> + +namespace NKikimr::NSQS { + +TQueueAttributes TQueueAttributes::FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp) { + TQueueAttributes result; + +#define INVALID_PARAM_MSG "Invalid value for the parameter %s." + +#define TRY_SET_ATTRIBUTE(NAME, MIN_VAL, MAX_VAL) \ + if (name == Y_STRINGIZE(NAME)) { \ + if (!result.TryParseLimitedValue(Y_STRINGIZE(NAME), \ + value, \ + MIN_VAL, \ + MAX_VAL, \ + result.NAME, \ + clamp)) \ + { \ + break; \ + } \ + } + + for (const auto& [name, value] : attributes) { + TRY_SET_ATTRIBUTE(VisibilityTimeout, + 0, + TLimits::MaxVisibilityTimeout.Seconds()) + else TRY_SET_ATTRIBUTE(MessageRetentionPeriod, + TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), + TLimits::MaxMessageRetentionPeriod.Seconds()) + else TRY_SET_ATTRIBUTE(ReceiveMessageWaitTimeSeconds, + 0, + TDuration::MilliSeconds(config.GetMaxWaitTimeoutMs()).Seconds()) + else TRY_SET_ATTRIBUTE(DelaySeconds, + 0, + TLimits::MaxDelaySeconds) + else TRY_SET_ATTRIBUTE(MaximumMessageSize, + 1024, + TLimits::MaxMessageSize) + else if (name == "ContentBasedDeduplication" && isFifoQueue) { + if (value == "true") { + result.ContentBasedDeduplication = true; + } else if (value == "false") { + result.ContentBasedDeduplication = false; + } else { + result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE; result.ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values: true, false.", name.c_str()); - break; - } - } else if (name == "FifoQueue" && isFifoQueue) { - if (value == "true") { - continue; // just ignore the value - } else { - result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE; - if (value == "false") { + break; + } + } else if (name == "FifoQueue" && isFifoQueue) { + if (value == "true") { + continue; // just ignore the value + } else { + result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE; + if (value == "false") { result.ErrorText = Sprintf(INVALID_PARAM_MSG " Reason: Modifying queue type is not supported.", name.c_str()); - } else { + } else { result.ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values: true, false.", name.c_str()); - } - break; - } - } else if (config.GetEnableDeadLetterQueues() && name == "RedrivePolicy") { - result.RedrivePolicy = TRedrivePolicy::FromJson(value, config); - if (result.RedrivePolicy.IsValid()) { - if (*result.RedrivePolicy.TargetQueueName && isFifoQueue != result.RedrivePolicy.TargetQueueName->EndsWith(".fifo")) { + } + break; + } + } else if (config.GetEnableDeadLetterQueues() && name == "RedrivePolicy") { + result.RedrivePolicy = TRedrivePolicy::FromJson(value, config); + if (result.RedrivePolicy.IsValid()) { + if (*result.RedrivePolicy.TargetQueueName && isFifoQueue != result.RedrivePolicy.TargetQueueName->EndsWith(".fifo")) { result.ErrorText = "Target dead letter queue should have the same type as source queue."; - } else { - continue; - } - } else { - result.ErrorText = result.RedrivePolicy.GetErrorText(); - } - result.Error = &NErrors::INVALID_PARAMETER_VALUE; - break; - } else { - result.Error = &NErrors::INVALID_ATTRIBUTE_NAME; + } else { + continue; + } + } else { + result.ErrorText = result.RedrivePolicy.GetErrorText(); + } + result.Error = &NErrors::INVALID_PARAMETER_VALUE; + break; + } else { + result.Error = &NErrors::INVALID_ATTRIBUTE_NAME; result.ErrorText = Sprintf("Unknown Attribute %s.", name.c_str()); - break; - } - } - - return result; -} - -bool TQueueAttributes::Validate() const { - return !Error; -} - -bool TQueueAttributes::HasClampedAttributes() const { - return Clamped; -} - -bool TQueueAttributes::TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp) { - ui64 value; - if (TryFromString<ui64>(valueStr, value)) { - const bool valid = (value >= allowedMinValue && value <= allowedMaxValue); - if (valid || clamp) { - result = valid ? value : ClampVal(value, allowedMinValue, allowedMaxValue); - Clamped |= !valid; - return true; - } - } - - result = Nothing(); - Error = &NErrors::INVALID_ATTRIBUTE_VALUE; + break; + } + } + + return result; +} + +bool TQueueAttributes::Validate() const { + return !Error; +} + +bool TQueueAttributes::HasClampedAttributes() const { + return Clamped; +} + +bool TQueueAttributes::TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp) { + ui64 value; + if (TryFromString<ui64>(valueStr, value)) { + const bool valid = (value >= allowedMinValue && value <= allowedMaxValue); + if (valid || clamp) { + result = valid ? value : ClampVal(value, allowedMinValue, allowedMaxValue); + Clamped |= !valid; + return true; + } + } + + result = Nothing(); + Error = &NErrors::INVALID_ATTRIBUTE_VALUE; ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values are from %" PRIu64 " to %" PRIu64 " both inclusive.", attrName.c_str(), allowedMinValue, allowedMaxValue); - - return false; -} - -} // namespace NKikimr::NQS + + return false; +} + +} // namespace NKikimr::NQS diff --git a/ydb/core/ymq/base/queue_attributes.h b/ydb/core/ymq/base/queue_attributes.h index 3e0dfd19e0..645b9bff9c 100644 --- a/ydb/core/ymq/base/queue_attributes.h +++ b/ydb/core/ymq/base/queue_attributes.h @@ -1,34 +1,34 @@ -#pragma once - -#include "dlq_helpers.h" +#pragma once + +#include "dlq_helpers.h" #include <ydb/library/http_proxy/error/error.h> - + #include <ydb/core/protos/config.pb.h> - -#include <util/generic/hash.h> -#include <util/generic/maybe.h> -#include <util/generic/string.h> - -namespace NKikimr::NSQS { - -struct TQueueAttributes { - static TQueueAttributes FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp); - bool Validate() const; - bool HasClampedAttributes() const; -public: - bool Clamped = false; - const TErrorClass* Error = nullptr; - TString ErrorText; - TMaybe<ui64> VisibilityTimeout; // seconds - TMaybe<ui64> MessageRetentionPeriod; // seconds - TMaybe<ui64> ReceiveMessageWaitTimeSeconds; // seconds - TMaybe<ui64> DelaySeconds; // seconds - TMaybe<ui64> MaximumMessageSize; // bytes - TMaybe<bool> ContentBasedDeduplication; // bool - TRedrivePolicy RedrivePolicy; - -private: - bool TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp); -}; - -} // namespace NKikimr::NSQS + +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/string.h> + +namespace NKikimr::NSQS { + +struct TQueueAttributes { + static TQueueAttributes FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp); + bool Validate() const; + bool HasClampedAttributes() const; +public: + bool Clamped = false; + const TErrorClass* Error = nullptr; + TString ErrorText; + TMaybe<ui64> VisibilityTimeout; // seconds + TMaybe<ui64> MessageRetentionPeriod; // seconds + TMaybe<ui64> ReceiveMessageWaitTimeSeconds; // seconds + TMaybe<ui64> DelaySeconds; // seconds + TMaybe<ui64> MaximumMessageSize; // bytes + TMaybe<bool> ContentBasedDeduplication; // bool + TRedrivePolicy RedrivePolicy; + +private: + bool TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp); +}; + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/queue_id.cpp b/ydb/core/ymq/base/queue_id.cpp index 0b93440a39..2f0324efe2 100644 --- a/ydb/core/ymq/base/queue_id.cpp +++ b/ydb/core/ymq/base/queue_id.cpp @@ -1,74 +1,74 @@ -#include <util/generic/hash.h> -#include <util/generic/string.h> -#include <util/string/cast.h> - +#include <util/generic/hash.h> +#include <util/generic/string.h> +#include <util/string/cast.h> + namespace NKikimr::NSQS { - -// Queue id breakdown in bits [15][1][64][7][13]: -// [15] - service id -// [1] - reserved == 0 -// [64] - globally unique number -// [7] - reserved == 0 -// [13] - partial hash result for account name (it's not really important to have stable hash impl) -// total: 15 + 1 + 64 + 7 + 13 == 100 - -class TQueueId { -public: - TQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) { - char* bufPtr = Buf; - - // first 15 bits - Y_VERIFY(serviceId <= 0x7FFF); - ConvertAndAppendLastNBits(serviceId, 15, bufPtr); - - // next 65 bits (1 bit is reserved and 64 are the place for the unique num) - *bufPtr++ = SmallIntToBase32(uniqueNum >> 60); // first 5 bits - ConvertAndAppendLastNBits(uniqueNum, 60, bufPtr); // 60 other bits - - // last 20 bits - const ui32 appendix = THash<TString>()(accountName) & 0x1FFF; // use last 13 bits only - ConvertAndAppendLastNBits(appendix, 20, bufPtr); - - Y_VERIFY(bufPtr == &Buf[20]); - - *bufPtr = '\0'; - } - - TString AsString() const { - return TString(Buf); - } - -private: - static const ui64 BasicMask = 0x1F; // 11111 as binary - static const ui8 Radix = 5; - - char Buf[21] = {}; - -private: - char SmallIntToBase32(const ui8 v) const { - Y_VERIFY(v < 32); - - if (v < 10) { - return '0' + v; - } else { - return 'a' + v - 10; - } - } - - void ConvertAndAppendLastNBits(const ui64 num, const ui8 n, char*& output) const { - Y_VERIFY(n % Radix == 0); - - for (int i = n / Radix - 1; i >= 0; --i) { - const ui8 shift = i * Radix; - const ui64 mask = BasicMask << shift; - - *output++ = SmallIntToBase32((num & mask) >> shift); - } - } -}; - -TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) { - return TQueueId(serviceId, uniqueNum, accountName).AsString(); -} - + +// Queue id breakdown in bits [15][1][64][7][13]: +// [15] - service id +// [1] - reserved == 0 +// [64] - globally unique number +// [7] - reserved == 0 +// [13] - partial hash result for account name (it's not really important to have stable hash impl) +// total: 15 + 1 + 64 + 7 + 13 == 100 + +class TQueueId { +public: + TQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) { + char* bufPtr = Buf; + + // first 15 bits + Y_VERIFY(serviceId <= 0x7FFF); + ConvertAndAppendLastNBits(serviceId, 15, bufPtr); + + // next 65 bits (1 bit is reserved and 64 are the place for the unique num) + *bufPtr++ = SmallIntToBase32(uniqueNum >> 60); // first 5 bits + ConvertAndAppendLastNBits(uniqueNum, 60, bufPtr); // 60 other bits + + // last 20 bits + const ui32 appendix = THash<TString>()(accountName) & 0x1FFF; // use last 13 bits only + ConvertAndAppendLastNBits(appendix, 20, bufPtr); + + Y_VERIFY(bufPtr == &Buf[20]); + + *bufPtr = '\0'; + } + + TString AsString() const { + return TString(Buf); + } + +private: + static const ui64 BasicMask = 0x1F; // 11111 as binary + static const ui8 Radix = 5; + + char Buf[21] = {}; + +private: + char SmallIntToBase32(const ui8 v) const { + Y_VERIFY(v < 32); + + if (v < 10) { + return '0' + v; + } else { + return 'a' + v - 10; + } + } + + void ConvertAndAppendLastNBits(const ui64 num, const ui8 n, char*& output) const { + Y_VERIFY(n % Radix == 0); + + for (int i = n / Radix - 1; i >= 0; --i) { + const ui8 shift = i * Radix; + const ui64 mask = BasicMask << shift; + + *output++ = SmallIntToBase32((num & mask) >> shift); + } + } +}; + +TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) { + return TQueueId(serviceId, uniqueNum, accountName).AsString(); +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/queue_id.h b/ydb/core/ymq/base/queue_id.h index a2cac100e0..11ce258b4c 100644 --- a/ydb/core/ymq/base/queue_id.h +++ b/ydb/core/ymq/base/queue_id.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/string.h> - +#pragma once + +#include <util/generic/string.h> + namespace NKikimr::NSQS { - -TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName); - + +TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName); + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/queue_path.h b/ydb/core/ymq/base/queue_path.h index baaa781b00..dcf08c3c8b 100644 --- a/ydb/core/ymq/base/queue_path.h +++ b/ydb/core/ymq/base/queue_path.h @@ -8,7 +8,7 @@ struct TQueuePath { TString Root; TString UserName; TString QueueName; - TString VersionSuffix; + TString VersionSuffix; TQueuePath() { } @@ -20,11 +20,11 @@ struct TQueuePath { : Root(root) , UserName(userName) , QueueName(queueName) - { - if (version) { - VersionSuffix = TString::Join("v", ToString(version)); - } - } + { + if (version) { + VersionSuffix = TString::Join("v", ToString(version)); + } + } operator TString() const { return GetQueuePath(); @@ -35,14 +35,14 @@ struct TQueuePath { } TString GetVersionedQueuePath() const { - auto tmp = GetQueuePath(); - if (!VersionSuffix) { - return tmp; - } else { - return Join("/", tmp, VersionSuffix); - } - } - + auto tmp = GetQueuePath(); + if (!VersionSuffix) { + return tmp; + } else { + return Join("/", tmp, VersionSuffix); + } + } + TString GetUserPath() const { return Join("/", Root, UserName); } diff --git a/ydb/core/ymq/base/security.h b/ydb/core/ymq/base/security.h index 764be96265..58a1f6afc3 100644 --- a/ydb/core/ymq/base/security.h +++ b/ydb/core/ymq/base/security.h @@ -1,30 +1,30 @@ -#pragma once - -#include <util/generic/string.h> - - +#pragma once + +#include <util/generic/string.h> + + namespace NKikimr::NSQS { - -template <typename TReq, typename TCreds> -TString ExtractSecurityToken(const TReq& request) { - if (request.HasCredentials()) { - switch (request.GetCredentials().AccessKey_case()) { - case TCreds::kOAuthToken: { - return request.GetCredentials().GetOAuthToken(); - } - case TCreds::kTvmTicket: { - return request.GetCredentials().GetTvmTicket(); - } - case TCreds::kStaticCreds: { - return request.GetCredentials().GetStaticCreds(); - } - default: { - // leave the token empty - } - } - } - - return {}; -} - + +template <typename TReq, typename TCreds> +TString ExtractSecurityToken(const TReq& request) { + if (request.HasCredentials()) { + switch (request.GetCredentials().AccessKey_case()) { + case TCreds::kOAuthToken: { + return request.GetCredentials().GetOAuthToken(); + } + case TCreds::kTvmTicket: { + return request.GetCredentials().GetTvmTicket(); + } + case TCreds::kStaticCreds: { + return request.GetCredentials().GetStaticCreds(); + } + default: { + // leave the token empty + } + } + } + + return {}; +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp b/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp index 3254de751c..c45ec89184 100644 --- a/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp +++ b/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp @@ -1,76 +1,76 @@ #include <ydb/core/ymq/base/dlq_helpers.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr::NSQS { - -Y_UNIT_TEST_SUITE(RedrivePolicy) { - Y_UNIT_TEST(RedrivePolicyValidationTest) { - NKikimrConfig::TSqsConfig config; - { - // basic sanity + +namespace NKikimr::NSQS { + +Y_UNIT_TEST_SUITE(RedrivePolicy) { + Y_UNIT_TEST(RedrivePolicyValidationTest) { + NKikimrConfig::TSqsConfig config; + { + // basic sanity TRedrivePolicy basicPolicy = TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config); - UNIT_ASSERT(basicPolicy.IsValid()); + UNIT_ASSERT(basicPolicy.IsValid()); UNIT_ASSERT(basicPolicy.TargetArn && *basicPolicy.TargetArn == "yrn:ya:sqs:ru-central1:radix:mymegadlq"); - UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "mymegadlq"); - UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 3); - } - { - // count as str + UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "mymegadlq"); + UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 3); + } + { + // count as str TRedrivePolicy basicPolicy = TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:ultradlq","maxReceiveCount":"42"})__", config); - UNIT_ASSERT(basicPolicy.IsValid()); + UNIT_ASSERT(basicPolicy.IsValid()); UNIT_ASSERT(basicPolicy.TargetArn && *basicPolicy.TargetArn == "yrn:ya:sqs:ru-central1:radix:ultradlq"); - UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "ultradlq"); - UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 42); - } - + UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "ultradlq"); + UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 42); + } + UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - // maxReceiveCount should be from 1 to 1000 + // maxReceiveCount should be from 1 to 1000 UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":0})__", config).IsValid()); UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":1001})__", config).IsValid()); - // maxReceiveCount should be integer + // maxReceiveCount should be integer UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":"omglol"})__", config).IsValid()); - // target arn shouldn't be empty - UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"","maxReceiveCount":3})__", config).IsValid()); - // both fields are expected - UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":""})__", config).IsValid()); - UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"maxReceiveCount":3})__", config).IsValid()); - // no cheats! - UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":":","maxReceiveCount":"8"})__", config).IsValid()); - { - // empty value is okay - TRedrivePolicy clearPolicy = TRedrivePolicy::FromJson("", config); - UNIT_ASSERT(clearPolicy.IsValid()); - UNIT_ASSERT(clearPolicy.TargetArn && clearPolicy.TargetArn->empty()); - UNIT_ASSERT(clearPolicy.TargetQueueName && clearPolicy.TargetQueueName->empty()); - UNIT_ASSERT(clearPolicy.MaxReceiveCount && *clearPolicy.MaxReceiveCount == 0); - } - } - - Y_UNIT_TEST(RedrivePolicyToJsonTest) { - NKikimrConfig::TSqsConfig config; + // target arn shouldn't be empty + UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"","maxReceiveCount":3})__", config).IsValid()); + // both fields are expected + UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":""})__", config).IsValid()); + UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"maxReceiveCount":3})__", config).IsValid()); + // no cheats! + UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":":","maxReceiveCount":"8"})__", config).IsValid()); + { + // empty value is okay + TRedrivePolicy clearPolicy = TRedrivePolicy::FromJson("", config); + UNIT_ASSERT(clearPolicy.IsValid()); + UNIT_ASSERT(clearPolicy.TargetArn && clearPolicy.TargetArn->empty()); + UNIT_ASSERT(clearPolicy.TargetQueueName && clearPolicy.TargetQueueName->empty()); + UNIT_ASSERT(clearPolicy.MaxReceiveCount && *clearPolicy.MaxReceiveCount == 0); + } + } + + Y_UNIT_TEST(RedrivePolicyToJsonTest) { + NKikimrConfig::TSqsConfig config; const TString json = R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__"; - UNIT_ASSERT_EQUAL(TRedrivePolicy::FromJson(json, config).ToJson(), json); - } - - Y_UNIT_TEST(RedrivePolicyArnValidationTest) { - NKikimrConfig::TSqsConfig config; + UNIT_ASSERT_EQUAL(TRedrivePolicy::FromJson(json, config).ToJson(), json); + } + + Y_UNIT_TEST(RedrivePolicyArnValidationTest) { + NKikimrConfig::TSqsConfig config; UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - // empty queue name + // empty queue name UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:","maxReceiveCount":3})__", config).IsValid()); - // empty account + // empty account UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1::mymegadlq","maxReceiveCount":3})__", config).IsValid()); - // bad region + // bad region UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central42:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - // bad prefix + // bad prefix UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrrrr:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - // broken prefix + // broken prefix UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrr:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - - config.SetYandexCloudMode(true); // check cloud prefix + + config.SetYandexCloudMode(true); // check cloud prefix UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yarrr:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); - } -} - -} // namespace NKikimr::NSQS + UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yarrr:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid()); + } +} + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/ut/queue_attributes_ut.cpp b/ydb/core/ymq/base/ut/queue_attributes_ut.cpp index e33ab8ff20..aa2594131e 100644 --- a/ydb/core/ymq/base/ut/queue_attributes_ut.cpp +++ b/ydb/core/ymq/base/ut/queue_attributes_ut.cpp @@ -1,48 +1,48 @@ #include <ydb/core/ymq/base/queue_attributes.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr::NSQS { - -Y_UNIT_TEST_SUITE(QueueAttributes) { - Y_UNIT_TEST(BasicStdTest) { - const bool isFifo = false; - const bool clamp = false; - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "2.5"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "100500"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1023"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1024"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262144"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262145"}}, {}, isFifo, clamp).Validate()); - } - - Y_UNIT_TEST(BasicFifoTest) { - const bool isFifo = true; - const bool clamp = false; - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "false"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "omg"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "0"}, {"VisibilityTimeout", "1.5"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "1"}, {"VisibilityTimeout", "0"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "2"}, {"VisibilityTimeout", "31"}}, {}, isFifo, clamp).Validate()); - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "3"}, {"VisibilityTimeout", "100500"}}, {}, isFifo, clamp).Validate()); - } - - Y_UNIT_TEST(BasicClampTest) { - const bool isFifo = false; - const bool clamp = true; - UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "100000"}}, {}, isFifo, clamp).HasClampedAttributes()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).HasClampedAttributes()); - UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).HasClampedAttributes()); - UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).MaximumMessageSize != 1); - UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).MaximumMessageSize != 1000000); - } -} - -} // namespace NKikimr::NSQS + +namespace NKikimr::NSQS { + +Y_UNIT_TEST_SUITE(QueueAttributes) { + Y_UNIT_TEST(BasicStdTest) { + const bool isFifo = false; + const bool clamp = false; + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "2.5"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "100500"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1023"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1024"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262144"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262145"}}, {}, isFifo, clamp).Validate()); + } + + Y_UNIT_TEST(BasicFifoTest) { + const bool isFifo = true; + const bool clamp = false; + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "false"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "omg"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "0"}, {"VisibilityTimeout", "1.5"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "1"}, {"VisibilityTimeout", "0"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "2"}, {"VisibilityTimeout", "31"}}, {}, isFifo, clamp).Validate()); + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "3"}, {"VisibilityTimeout", "100500"}}, {}, isFifo, clamp).Validate()); + } + + Y_UNIT_TEST(BasicClampTest) { + const bool isFifo = false; + const bool clamp = true; + UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "100000"}}, {}, isFifo, clamp).HasClampedAttributes()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).HasClampedAttributes()); + UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).HasClampedAttributes()); + UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).MaximumMessageSize != 1); + UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).MaximumMessageSize != 1000000); + } +} + +} // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/ut/ya.make b/ydb/core/ymq/base/ut/ya.make index f927641a90..7a9218946a 100644 --- a/ydb/core/ymq/base/ut/ya.make +++ b/ydb/core/ymq/base/ut/ya.make @@ -14,10 +14,10 @@ PEERDIR( SRCS( action_ut.cpp counters_ut.cpp - dlq_helpers_ut.cpp + dlq_helpers_ut.cpp helpers_ut.cpp secure_protobuf_printer_ut.cpp - queue_attributes_ut.cpp + queue_attributes_ut.cpp ) END() diff --git a/ydb/core/ymq/base/ya.make b/ydb/core/ymq/base/ya.make index 14cd4e72a1..3d5ea20516 100644 --- a/ydb/core/ymq/base/ya.make +++ b/ydb/core/ymq/base/ya.make @@ -6,25 +6,25 @@ OWNER( LIBRARY() SRCS( - acl.cpp + acl.cpp action.cpp counters.cpp debug_info.cpp - dlq_helpers.cpp + dlq_helpers.cpp helpers.cpp probes.cpp - queue_attributes.cpp - queue_id.cpp + queue_attributes.cpp + queue_id.cpp secure_protobuf_printer.cpp events_writer_iface.h ) GENERATE_ENUM_SERIALIZATION(query_id.h) -GENERATE_ENUM_SERIALIZATION(cloud_enums.h) +GENERATE_ENUM_SERIALIZATION(cloud_enums.h) PEERDIR( - contrib/libs/openssl + contrib/libs/openssl library/cpp/cgiparam library/cpp/ipmath library/cpp/lwtrace diff --git a/ydb/core/ymq/client/bin/main.cpp b/ydb/core/ymq/client/bin/main.cpp index cb7dbd2a2c..ac081949d7 100644 --- a/ydb/core/ymq/client/bin/main.cpp +++ b/ydb/core/ymq/client/bin/main.cpp @@ -6,7 +6,7 @@ #include <util/stream/file.h> #include <util/stream/output.h> -#include <util/system/env.h> +#include <util/system/env.h> #include <util/system/user.h> #include <google/protobuf/text_format.h> @@ -114,7 +114,7 @@ static int HandleCreate(int argc, const char* argv[]) { TString queue; ui64 shards = 0; ui64 partitions = 0; - ui64 retentionSeconds = 0; + ui64 retentionSeconds = 0; bool enableOutOfOrderExecution = false; bool disableOutOfOrderExecution = false; bool contentBasedDeduplication = false; @@ -186,13 +186,13 @@ static int HandleCreate(int argc, const char* argv[]) { req.SetEnableOutOfOrderTransactionsExecution(false); } opts.SetCredentials(req); - - if (retentionSeconds) { - auto* newAttribute = req.mutable_attributes()->Add(); - newAttribute->SetName("MessageRetentionPeriod"); - newAttribute->SetValue(ToString(retentionSeconds)); - } - + + if (retentionSeconds) { + auto* newAttribute = req.mutable_attributes()->Add(); + newAttribute->SetName("MessageRetentionPeriod"); + newAttribute->SetValue(ToString(retentionSeconds)); + } + if (enableAutosplit) { req.SetEnableAutosplit(true); // explicitly } else if (disableAutosplit) { @@ -235,7 +235,7 @@ static int HandleDelete(int argc, const char* argv[]) { req.MutableAuth()->SetUserName(opts.User); req.SetQueueName(queue); opts.SetCredentials(req); - + auto resp = TQueueClient(TClientOptions().SetHost(opts.Host).SetPort(opts.Port)).DeleteQueue(req); if (resp.HasError()) { @@ -511,13 +511,13 @@ static int HandleUser(int argc, const char* argv[]) { return 0; } -static int HandlePermissions(int argc, const char* argv[]) { - TString resource; - TString grantRequest; - TString revokeRequest; - TString setRequest; - bool clearACL; - +static int HandlePermissions(int argc, const char* argv[]) { + TString resource; + TString grantRequest; + TString revokeRequest; + TString setRequest; + bool clearACL; + TSqsOptions opts(false); opts.AddLongOption('q', "resource", "resource path") .Required() @@ -535,95 +535,95 @@ static int HandlePermissions(int argc, const char* argv[]) { .Optional() .NoArgument() .SetFlag(&clearACL); - + TOptsParseResult res(&opts, argc, argv); - + TQueueClient client(TClientOptions().SetHost(opts.Host).SetPort(opts.Port)); - - TModifyPermissionsRequest req; - - auto parseActionsAndSubject = [](auto& action, TStringBuf request) { - auto subject = request.NextTok(':'); - if (!subject || !request) { - return false; - } - while (auto actionName = request.NextTok(',')) { - *action.MutablePermissionNames()->Add() = TString(actionName); - } - action.SetSubject(TString(subject)); - return true; - }; - - bool valid = true; - if (grantRequest) { - valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableGrant(), grantRequest); - } - if (revokeRequest) { - valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableRevoke(), revokeRequest); - } - if (setRequest) { - valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableSet(), setRequest); - } - - req.SetResource(resource); - - if (!valid || !(grantRequest || revokeRequest || setRequest || clearACL)) { - Cerr << "Modify permissions command is invalid. Usage example: --grant username@staff:CreateQueue,SendMessage" << Endl; - return 1; - } - + + TModifyPermissionsRequest req; + + auto parseActionsAndSubject = [](auto& action, TStringBuf request) { + auto subject = request.NextTok(':'); + if (!subject || !request) { + return false; + } + while (auto actionName = request.NextTok(',')) { + *action.MutablePermissionNames()->Add() = TString(actionName); + } + action.SetSubject(TString(subject)); + return true; + }; + + bool valid = true; + if (grantRequest) { + valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableGrant(), grantRequest); + } + if (revokeRequest) { + valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableRevoke(), revokeRequest); + } + if (setRequest) { + valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableSet(), setRequest); + } + + req.SetResource(resource); + + if (!valid || !(grantRequest || revokeRequest || setRequest || clearACL)) { + Cerr << "Modify permissions command is invalid. Usage example: --grant username@staff:CreateQueue,SendMessage" << Endl; + return 1; + } + opts.SetCredentials(req); - - if (clearACL) { - req.SetClearACL(true); - } - - auto resp = client.ModifyPermissions(req); - if (resp.HasError()) { - Cerr << "Got error for resource : " - << resource << " : " - << resp.GetError().GetMessage() << Endl; - return 1; - } else { - Cerr << "OK" - << Endl; - } - - return 0; -} - -static int HandleListPermissions(int argc, const char* argv[]) { - TString path; - + + if (clearACL) { + req.SetClearACL(true); + } + + auto resp = client.ModifyPermissions(req); + if (resp.HasError()) { + Cerr << "Got error for resource : " + << resource << " : " + << resp.GetError().GetMessage() << Endl; + return 1; + } else { + Cerr << "OK" + << Endl; + } + + return 0; +} + +static int HandleListPermissions(int argc, const char* argv[]) { + TString path; + TSqsOptions opts(false); opts.AddLongOption('P', "path", "path to node") .Required() .StoreResult(&path); - + TOptsParseResult res(&opts, argc, argv); - + TQueueClient client(TClientOptions().SetHost(opts.Host).SetPort(opts.Port)); - - TListPermissionsRequest req; - - req.SetPath(path); + + TListPermissionsRequest req; + + req.SetPath(path); opts.SetCredentials(req); - - auto resp = client.ListPermissions(req); - if (resp.HasError()) { - Cerr << "Got error for path : " - << path << " : " - << resp.GetError().GetMessage() << Endl; - return 1; - } else { - Cout << resp.Utf8DebugString() << Endl; - } - - return 0; -} - - - + + auto resp = client.ListPermissions(req); + if (resp.HasError()) { + Cerr << "Got error for path : " + << path << " : " + << resp.GetError().GetMessage() << Endl; + return 1; + } else { + Cout << resp.Utf8DebugString() << Endl; + } + + return 0; +} + + + static int HandleGetQueueAttributes(int argc, const char* argv[]) { TString queueName; diff --git a/ydb/core/ymq/client/cpp/client.cpp b/ydb/core/ymq/client/cpp/client.cpp index c96f2dcaab..1814aaa9e3 100644 --- a/ydb/core/ymq/client/cpp/client.cpp +++ b/ydb/core/ymq/client/cpp/client.cpp @@ -56,10 +56,10 @@ public: METHOD_IMPL(PurgeQueue, "can't purge queue"); METHOD_IMPL(ReceiveMessage, "can't receive a message"); METHOD_IMPL(SendMessage, "can't enqueue a message"); - METHOD_IMPL(ModifyPermissions, "can't modify permissions"); + METHOD_IMPL(ModifyPermissions, "can't modify permissions"); METHOD_IMPL(GetQueueAttributes, "can't get queue attributes"); METHOD_IMPL(SetQueueAttributes, "can't set queue attributes"); - METHOD_IMPL(ListPermissions, "can't list permissions"); + METHOD_IMPL(ListPermissions, "can't list permissions"); #undef METHOD_IMPL @@ -130,10 +130,10 @@ TSendMessageResponse TQueueClient::SendMessage(const TSendMessageRequest& req) { return Impl_->SendMessage(req); } -TModifyPermissionsResponse TQueueClient::ModifyPermissions(const TModifyPermissionsRequest& req) { - return Impl_->ModifyPermissions(req); -} - +TModifyPermissionsResponse TQueueClient::ModifyPermissions(const TModifyPermissionsRequest& req) { + return Impl_->ModifyPermissions(req); +} + TGetQueueAttributesResponse TQueueClient::GetQueueAttributes(const TGetQueueAttributesRequest& req) { return Impl_->GetQueueAttributes(req); } @@ -142,8 +142,8 @@ TSetQueueAttributesResponse TQueueClient::SetQueueAttributes(const TSetQueueAttr return Impl_->SetQueueAttributes(req); } -TListPermissionsResponse TQueueClient::ListPermissions(const TListPermissionsRequest& req) { - return Impl_->ListPermissions(req); -} - +TListPermissionsResponse TQueueClient::ListPermissions(const TListPermissionsRequest& req) { + return Impl_->ListPermissions(req); +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/client/cpp/client.h b/ydb/core/ymq/client/cpp/client.h index d42ed84326..055431f109 100644 --- a/ydb/core/ymq/client/cpp/client.h +++ b/ydb/core/ymq/client/cpp/client.h @@ -90,14 +90,14 @@ public: TSendMessageResponse SendMessage(const TSendMessageRequest& req); - TModifyPermissionsResponse ModifyPermissions(const TModifyPermissionsRequest& req); - + TModifyPermissionsResponse ModifyPermissions(const TModifyPermissionsRequest& req); + TGetQueueAttributesResponse GetQueueAttributes(const TGetQueueAttributesRequest& req); TSetQueueAttributesResponse SetQueueAttributes(const TSetQueueAttributesRequest& req); - TListPermissionsResponse ListPermissions(const TListPermissionsRequest& req); - + TListPermissionsResponse ListPermissions(const TListPermissionsRequest& req); + private: class TImpl; THolder<TImpl> Impl_; diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp index 5d0963bfb2..3aa9151646 100644 --- a/ydb/core/ymq/http/http.cpp +++ b/ydb/core/ymq/http/http.cpp @@ -20,7 +20,7 @@ #include <util/generic/guid.h> #include <util/generic/hash.h> -#include <util/generic/set.h> +#include <util/generic/set.h> #include <util/generic/hash_set.h> #include <util/network/init.h> #include <util/string/ascii.h> @@ -52,7 +52,7 @@ const TString CREDENTIAL_PARAM = "credential"; constexpr TStringBuf PRIVATE_REQUEST_PATH_PREFIX = "/private"; const TSet<TString> ModifyPermissionsActions = {"GrantPermissions", "RevokePermissions", "SetPermissions"}; - + bool IsPrivateTokenHeader(TStringBuf headerName) { for (const TStringBuf h : PRIVATE_TOKENS_HEADERS) { if (AsciiEqualsIgnoreCase(h, headerName)) { @@ -71,14 +71,14 @@ public: } void DoSendReply(const TSqsResponse& resp) override { - auto response = ResponseToAmazonXmlFormat(resp); + auto response = ResponseToAmazonXmlFormat(resp); LogRequest(resp, response); - - response.FolderId = resp.GetFolderId(); - response.IsFifo = resp.GetIsFifo(); - response.ResourceId = resp.GetResourceId(); - - Request_->SendResponse(response); + + response.FolderId = resp.GetFolderId(); + response.IsFifo = resp.GetIsFifo(); + response.ResourceId = resp.GetResourceId(); + + Request_->SendResponse(response); } @@ -154,23 +154,23 @@ void THttpRequest::WriteResponse(const TReplyParams& replyParams, const TSqsHttp } if (Parent_->Config.GetYandexCloudMode() && !IsPrivateRequest_) { - // Send request attributes to the metering actor - auto reportRequestAttributes = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>(); - - auto& requestAttributes = reportRequestAttributes->Data; - - requestAttributes.HttpStatusCode = response.StatusCode; - requestAttributes.IsFifo = response.IsFifo; - requestAttributes.FolderId = response.FolderId; - requestAttributes.RequestSizeInBytes = RequestSizeInBytes_; - requestAttributes.ResponseSizeInBytes = response.Body.size(); - requestAttributes.SourceAddress = SourceAddress_; - requestAttributes.ResourceId = response.ResourceId; - requestAttributes.Action = Action_; - - Parent_->ActorSystem_->Send(MakeSqsMeteringServiceID(), reportRequestAttributes.Release()); - } - + // Send request attributes to the metering actor + auto reportRequestAttributes = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>(); + + auto& requestAttributes = reportRequestAttributes->Data; + + requestAttributes.HttpStatusCode = response.StatusCode; + requestAttributes.IsFifo = response.IsFifo; + requestAttributes.FolderId = response.FolderId; + requestAttributes.RequestSizeInBytes = RequestSizeInBytes_; + requestAttributes.ResponseSizeInBytes = response.Body.size(); + requestAttributes.SourceAddress = SourceAddress_; + requestAttributes.ResourceId = response.ResourceId; + requestAttributes.Action = Action_; + + Parent_->ActorSystem_->Send(MakeSqsMeteringServiceID(), reportRequestAttributes.Release()); + } + httpResponse.OutTo(replyParams.Output); } @@ -222,12 +222,12 @@ bool THttpRequest::DoReply(const TReplyParams& p) { return true; } - try { - ParseHeaders(p.Input); + try { + ParseHeaders(p.Input); - if (SetupPing(p)) { - return false; - } + if (SetupPing(p)) { + return false; + } ParseRequest(p.Input); @@ -240,7 +240,7 @@ bool THttpRequest::DoReply(const TReplyParams& p) { << "] IP [" << SourceAddress_ << "]" ); - if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) { + if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) { WriteResponse(p, MakeErrorXmlResponse(NErrors::MISSING_PARAMETER, Parent_->AggregatedUserCounters_.Get(), "No user name was provided.")); return true; } @@ -277,24 +277,24 @@ TString THttpRequest::GetRequestPathPart(TStringBuf path, size_t partIdx) const TVector<TStringBuf> items; StringSplitter(path).Split('/').AddTo(&items); - if (items.size() > partIdx) { - return TString(items[partIdx]); + if (items.size() > partIdx) { + return TString(items[partIdx]); } return TString(); } TString THttpRequest::ExtractQueueNameFromPath(const TStringBuf path) { - return GetRequestPathPart(path, 2); -} - + return GetRequestPathPart(path, 2); +} + TString THttpRequest::ExtractAccountNameFromPath(const TStringBuf path) { - return GetRequestPathPart(path, 1); -} - + return GetRequestPathPart(path, 1); +} + void THttpRequest::ExtractQueueAndAccountNames(const TStringBuf path) { - if (Action_ == EAction::ModifyPermissions) - return; - + if (Action_ == EAction::ModifyPermissions) + return; + if (Action_ == EAction::GetQueueUrl || Action_ == EAction::CreateQueue) { if (!QueryParams_.QueueName) { throw TSQSException(NErrors::MISSING_PARAMETER) << "No queue name was provided."; @@ -302,9 +302,9 @@ void THttpRequest::ExtractQueueAndAccountNames(const TStringBuf path) { QueueName_ = *QueryParams_.QueueName; } else { - const auto pathAndQuery = QueryParams_.QueueUrl ? GetPathAndQuery(*QueryParams_.QueueUrl) : GetPathAndQuery(path); - QueueName_ = ExtractQueueNameFromPath(pathAndQuery); - AccountName_ = ExtractAccountNameFromPath(pathAndQuery); + const auto pathAndQuery = QueryParams_.QueueUrl ? GetPathAndQuery(*QueryParams_.QueueUrl) : GetPathAndQuery(path); + QueueName_ = ExtractQueueNameFromPath(pathAndQuery); + AccountName_ = ExtractAccountNameFromPath(pathAndQuery); if (IsProxyAction(Action_)) { if (QueryParams_.QueueUrl && *QueryParams_.QueueUrl) { @@ -346,10 +346,10 @@ void THttpRequest::ParseHeaders(const THttpInput& input) { for (const auto& header : input.Headers()) { if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) { ParseAuthorization(header.Value()); - } else if (AsciiEqualsIgnoreCase(header.Name(), SECURITY_TOKEN_HEADER)) { - SecurityToken_ = header.Value(); - } else if (AsciiEqualsIgnoreCase(header.Name(), IAM_TOKEN_HEADER)) { - IamToken_ = header.Value(); + } else if (AsciiEqualsIgnoreCase(header.Name(), SECURITY_TOKEN_HEADER)) { + SecurityToken_ = header.Value(); + } else if (AsciiEqualsIgnoreCase(header.Name(), IAM_TOKEN_HEADER)) { + IamToken_ = header.Value(); } else if (AsciiEqualsIgnoreCase(header.Name(), FORWARDED_IP_HEADER)) { SourceAddress_ = header.Value(); } else if (AsciiEqualsIgnoreCase(header.Name(), REQUEST_ID_HEADER)) { @@ -392,20 +392,20 @@ void THttpRequest::ParsePrivateRequestPathPrefix(const TStringBuf& path) { } } -ui64 THttpRequest::CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const { - ui64 requestSize = input.FirstLine().size(); - for (const auto& header : input.Headers()) { - requestSize += header.Name().size() + header.Value().size(); - } - if (input.Trailers()) { - for (const auto& header : *input.Trailers()) { - requestSize += header.Name().size() + header.Value().size(); - } - } - requestSize += contentLength; - return requestSize; -} - +ui64 THttpRequest::CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const { + ui64 requestSize = input.FirstLine().size(); + for (const auto& header : input.Headers()) { + requestSize += header.Name().size() + header.Value().size(); + } + if (input.Trailers()) { + for (const auto& header : *input.Trailers()) { + requestSize += header.Name().size() + header.Value().size(); + } + } + requestSize += contentLength; + return requestSize; +} + void THttpRequest::ParseRequest(THttpInput& input) { if (Parent_->HttpCounters_ && UserName_) { UserCounters_ = Parent_->HttpCounters_->GetUserCounters(UserName_); @@ -433,9 +433,9 @@ void THttpRequest::ParseRequest(THttpInput& input) { RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Incoming http request: " << input.FirstLine()); ParsePrivateRequestPathPrefix(parsed.Path); - - RequestSizeInBytes_ = CalculateRequestSizeInBytes(input, contentLength); - + + RequestSizeInBytes_ = CalculateRequestSizeInBytes(input, contentLength); + if (HttpMethod == "POST") { ParseCgiParameters(TCgiParameters(TStringBuf(InputData->Data(), contentLength))); } else if (HttpMethod == "GET") { @@ -445,52 +445,52 @@ void THttpRequest::ParseRequest(THttpInput& input) { } if (QueryParams_.Action) { - if (IsIn(ModifyPermissionsActions, *QueryParams_.Action)) { - Action_ = EAction::ModifyPermissions; - } else { - Action_ = ActionFromString(*QueryParams_.Action); - } - + if (IsIn(ModifyPermissionsActions, *QueryParams_.Action)) { + Action_ = EAction::ModifyPermissions; + } else { + Action_ = ActionFromString(*QueryParams_.Action); + } + THttpActionCounters* counters = GetActionCounters(); INC_COUNTER(counters, Requests); } else { throw TSQSException(NErrors::MISSING_ACTION) << "Action param was not found."; } - if (QueryParams_.FolderId) { - FolderId_ = *QueryParams_.FolderId; - } - - ExtractQueueAndAccountNames(parsed.Path); - - if (Parent_->Config.GetYandexCloudMode() && !IamToken_ && !FolderId_) { - AwsSignature_.Reset(new TAwsRequestSignV4(input, parsed, InputData)); - } + if (QueryParams_.FolderId) { + FolderId_ = *QueryParams_.FolderId; + } + + ExtractQueueAndAccountNames(parsed.Path); + + if (Parent_->Config.GetYandexCloudMode() && !IamToken_ && !FolderId_) { + AwsSignature_.Reset(new TAwsRequestSignV4(input, parsed, InputData)); + } } -#define HANDLE_SETUP_ACTION_CASE(NAME) \ - case EAction::NAME: { \ - Y_CAT(Setup, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \ - CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \ - break; \ +#define HANDLE_SETUP_ACTION_CASE(NAME) \ + case EAction::NAME: { \ + Y_CAT(Setup, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \ + CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \ + break; \ } - -#define HANDLE_SETUP_PRIVATE_ACTION_CASE(NAME) \ - case EAction::NAME: { \ - if (!IsPrivateRequest_) { \ + +#define HANDLE_SETUP_PRIVATE_ACTION_CASE(NAME) \ + case EAction::NAME: { \ + if (!IsPrivateRequest_) { \ RLOG_SQS_BASE_ERROR(*Parent_->ActorSystem_, \ - "Attempt to call " \ - Y_STRINGIZE(NAME) \ - " action without private url path"); \ - throw TSQSException(NErrors::INVALID_ACTION); \ - } \ - Y_CAT(SetupPrivate, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \ - CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \ - break; \ + "Attempt to call " \ + Y_STRINGIZE(NAME) \ + " action without private url path"); \ + throw TSQSException(NErrors::INVALID_ACTION); \ + } \ + Y_CAT(SetupPrivate, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \ + CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \ + break; \ } bool THttpRequest::SetupRequest() { - auto requestHolder = MakeHolder<TSqsRequest>(); + auto requestHolder = MakeHolder<TSqsRequest>(); requestHolder->SetRequestId(RequestId_); // Validate batches @@ -514,29 +514,29 @@ bool THttpRequest::SetupRequest() { } switch (Action_) { - HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibility); - HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibilityBatch); - HANDLE_SETUP_ACTION_CASE(CreateQueue); - HANDLE_SETUP_ACTION_CASE(CreateUser); - HANDLE_SETUP_ACTION_CASE(GetQueueAttributes); - HANDLE_SETUP_ACTION_CASE(GetQueueUrl); - HANDLE_SETUP_ACTION_CASE(DeleteMessage); - HANDLE_SETUP_ACTION_CASE(DeleteMessageBatch); - HANDLE_SETUP_ACTION_CASE(DeleteQueue); - HANDLE_SETUP_ACTION_CASE(DeleteUser); - HANDLE_SETUP_ACTION_CASE(ListPermissions); - HANDLE_SETUP_ACTION_CASE(ListDeadLetterSourceQueues); - HANDLE_SETUP_ACTION_CASE(ListQueues); - HANDLE_SETUP_ACTION_CASE(ListUsers); - HANDLE_SETUP_ACTION_CASE(ModifyPermissions); - HANDLE_SETUP_ACTION_CASE(PurgeQueue); - HANDLE_SETUP_ACTION_CASE(ReceiveMessage); - HANDLE_SETUP_ACTION_CASE(SendMessage); - HANDLE_SETUP_ACTION_CASE(SendMessageBatch); - HANDLE_SETUP_ACTION_CASE(SetQueueAttributes); + HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibility); + HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibilityBatch); + HANDLE_SETUP_ACTION_CASE(CreateQueue); + HANDLE_SETUP_ACTION_CASE(CreateUser); + HANDLE_SETUP_ACTION_CASE(GetQueueAttributes); + HANDLE_SETUP_ACTION_CASE(GetQueueUrl); + HANDLE_SETUP_ACTION_CASE(DeleteMessage); + HANDLE_SETUP_ACTION_CASE(DeleteMessageBatch); + HANDLE_SETUP_ACTION_CASE(DeleteQueue); + HANDLE_SETUP_ACTION_CASE(DeleteUser); + HANDLE_SETUP_ACTION_CASE(ListPermissions); + HANDLE_SETUP_ACTION_CASE(ListDeadLetterSourceQueues); + HANDLE_SETUP_ACTION_CASE(ListQueues); + HANDLE_SETUP_ACTION_CASE(ListUsers); + HANDLE_SETUP_ACTION_CASE(ModifyPermissions); + HANDLE_SETUP_ACTION_CASE(PurgeQueue); + HANDLE_SETUP_ACTION_CASE(ReceiveMessage); + HANDLE_SETUP_ACTION_CASE(SendMessage); + HANDLE_SETUP_ACTION_CASE(SendMessageBatch); + HANDLE_SETUP_ACTION_CASE(SetQueueAttributes); HANDLE_SETUP_PRIVATE_ACTION_CASE(DeleteQueueBatch); - HANDLE_SETUP_PRIVATE_ACTION_CASE(CountQueues); + HANDLE_SETUP_PRIVATE_ACTION_CASE(CountQueues); HANDLE_SETUP_PRIVATE_ACTION_CASE(PurgeQueueBatch); HANDLE_SETUP_PRIVATE_ACTION_CASE(GetQueueAttributesBatch); @@ -547,7 +547,7 @@ bool THttpRequest::SetupRequest() { } RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Create proxy action actor for request " << SecureShortUtf8DebugString(*requestHolder)); - + const bool enableQueueLeader = Parent_->Config.HasEnableQueueMaster() ? Parent_->Config.GetEnableQueueMaster() : Parent_->Config.GetEnableQueueLeader(); @@ -656,16 +656,16 @@ void THttpRequest::SetupDeleteQueue(TDeleteQueueRequest* const req) { } void THttpRequest::SetupListPermissions(TListPermissionsRequest* const req) { - if (QueryParams_.Path) { - req->SetPath(*QueryParams_.Path); - } -} - -void THttpRequest::SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req) { - req->SetQueueName(QueueName_); - req->MutableAuth()->SetUserName(UserName_); -} - + if (QueryParams_.Path) { + req->SetPath(*QueryParams_.Path); + } +} + +void THttpRequest::SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req) { + req->SetQueueName(QueueName_); + req->MutableAuth()->SetUserName(UserName_); +} + void THttpRequest::SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const req) { for (const auto& entry : QueryParams_.BatchEntries) { auto* protoEntry = req->AddEntries(); @@ -680,10 +680,10 @@ void THttpRequest::SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const req->MutableAuth()->SetUserName(UserName_); } -void THttpRequest::SetupPrivateCountQueues(TCountQueuesRequest* const req) { - req->MutableAuth()->SetUserName(UserName_); -} - +void THttpRequest::SetupPrivateCountQueues(TCountQueuesRequest* const req) { + req->MutableAuth()->SetUserName(UserName_); +} + void THttpRequest::SetupPrivatePurgeQueueBatch(TPurgeQueueBatchRequest* const req) { for (const auto& entry : QueryParams_.BatchEntries) { auto* protoEntry = req->AddEntries(); @@ -753,43 +753,43 @@ void THttpRequest::SetupListUsers(TListUsersRequest* const req) { } } -template<typename TModifyPermissionsAction> -static void SetupModifyPermissionsAction(const TParameters& queryParams, TModifyPermissionsAction& action) { - if (queryParams.Subject) { - action.SetSubject(*queryParams.Subject); - } - - for (const auto& [index, entry] : queryParams.BatchEntries) { - if (entry.Action) { - *action.MutablePermissionNames()->Add() = *entry.Action; - } - } -} - +template<typename TModifyPermissionsAction> +static void SetupModifyPermissionsAction(const TParameters& queryParams, TModifyPermissionsAction& action) { + if (queryParams.Subject) { + action.SetSubject(*queryParams.Subject); + } + + for (const auto& [index, entry] : queryParams.BatchEntries) { + if (entry.Action) { + *action.MutablePermissionNames()->Add() = *entry.Action; + } + } +} + void THttpRequest::SetupModifyPermissions(TModifyPermissionsRequest* const req) { - auto it = ModifyPermissionsActions.begin(); - if (*QueryParams_.Action == *it++) { - SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableGrant()); - } else if (*QueryParams_.Action == *it++) { - SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableRevoke()); - } else if (*QueryParams_.Action == *it) { - SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableSet()); - } - - Y_VERIFY(it != ModifyPermissionsActions.end()); - - if (QueryParams_.Path) { - req->SetResource(*QueryParams_.Path); - } - - if (QueryParams_.Clear) { - bool clear = false; - if (TryFromString(*QueryParams_.Clear, clear)) { - req->SetClearACL(clear); - } - } -} - + auto it = ModifyPermissionsActions.begin(); + if (*QueryParams_.Action == *it++) { + SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableGrant()); + } else if (*QueryParams_.Action == *it++) { + SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableRevoke()); + } else if (*QueryParams_.Action == *it) { + SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableSet()); + } + + Y_VERIFY(it != ModifyPermissionsActions.end()); + + if (QueryParams_.Path) { + req->SetResource(*QueryParams_.Path); + } + + if (QueryParams_.Clear) { + bool clear = false; + if (TryFromString(*QueryParams_.Clear, clear)) { + req->SetClearACL(clear); + } + } +} + void THttpRequest::SetupPurgeQueue(TPurgeQueueRequest* const req) { req->SetQueueName(QueueName_); req->MutableAuth()->SetUserName(UserName_); diff --git a/ydb/core/ymq/http/http.h b/ydb/core/ymq/http/http.h index 01a2b0edc4..d0f6e15cfd 100644 --- a/ydb/core/ymq/http/http.h +++ b/ydb/core/ymq/http/http.h @@ -51,14 +51,14 @@ private: if (SecurityToken_ && !config.GetYandexCloudMode()) { // it's also TVM-compatible due to universal TicketParser request->MutableCredentials()->SetOAuthToken(SecurityToken_); - } + } } - + TString GetRequestPathPart(TStringBuf path, size_t partIdx) const; TString ExtractQueueNameFromPath(const TStringBuf path); TString ExtractAccountNameFromPath(const TStringBuf path); - ui64 CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const; + ui64 CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const; void ExtractQueueAndAccountNames(const TStringBuf path); TString HttpHeadersLogString(const THttpInput& input); @@ -80,13 +80,13 @@ private: void SetupDeleteMessageBatch(TDeleteMessageBatchRequest* const req); void SetupDeleteQueue(TDeleteQueueRequest* const req); void SetupListPermissions(TListPermissionsRequest* const req); - void SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req); + void SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req); void SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const req); void SetupPrivatePurgeQueueBatch(TPurgeQueueBatchRequest* const req); void SetupPrivateGetQueueAttributesBatch(TGetQueueAttributesBatchRequest* const req); void SetupDeleteUser(TDeleteUserRequest* const req); void SetupListQueues(TListQueuesRequest* const req); - void SetupPrivateCountQueues(TCountQueuesRequest* const req); + void SetupPrivateCountQueues(TCountQueuesRequest* const req); void SetupListUsers(TListUsersRequest* const req); void SetupModifyPermissions(TModifyPermissionsRequest* const req); void SetupReceiveMessage(TReceiveMessageRequest* const req); @@ -120,7 +120,7 @@ private: TString ApiMethod_; THolder<TAwsRequestSignV4> AwsSignature_; - + TMaybe<TBuffer> InputData; TString HttpMethod; TMaybe<TSqsHttpResponse> Response_; @@ -129,8 +129,8 @@ private: // Source values parsed from headers TString SourceAddress_; - ui64 RequestSizeInBytes_ = 0; - + ui64 RequestSizeInBytes_ = 0; + bool IsPrivateRequest_ = false; // Has "/private" path prefix TInstant StartTime_ = TInstant::Now(); }; @@ -169,7 +169,7 @@ private: const NKikimrConfig::TSqsConfig Config; NActors::TActorSystem* ActorSystem_ = nullptr; TIntrusivePtr<THttpCounters> HttpCounters_; // http subsystem counters - THolder<TCloudAuthCounters> CloudAuthCounters_; // cloud_auth subsystem counters + THolder<TCloudAuthCounters> CloudAuthCounters_; // cloud_auth subsystem counters TIntrusivePtr<TUserCounters> AggregatedUserCounters_; // aggregated counters for user in core subsystem ui32 PoolId_ = 0; }; diff --git a/ydb/core/ymq/http/params.h b/ydb/core/ymq/http/params.h index 7834fc4f0f..601e0e7f94 100644 --- a/ydb/core/ymq/http/params.h +++ b/ydb/core/ymq/http/params.h @@ -10,21 +10,21 @@ namespace NKikimr::NSQS { struct TParameters { TMaybe<TString> Action; - TMaybe<TString> Clear; + TMaybe<TString> Clear; TMaybe<ui64> DelaySeconds; - TMaybe<TString> FolderId; + TMaybe<TString> FolderId; TMaybe<TString> Id; TMaybe<ui32> MaxNumberOfMessages; TMaybe<TString> MessageBody; TMaybe<TString> MessageDeduplicationId; TMaybe<TString> MessageGroupId; - TMaybe<TString> Path; + TMaybe<TString> Path; TMaybe<TString> QueueName; TMaybe<TString> QueueNamePrefix; TMaybe<TString> QueueUrl; TMaybe<TString> ReceiptHandle; TMaybe<TString> ReceiveRequestAttemptId; - TMaybe<TString> Subject; + TMaybe<TString> Subject; TMaybe<TString> UserName; TMaybe<TString> UserNamePrefix; TMaybe<TString> Version; diff --git a/ydb/core/ymq/http/parser.rl6 b/ydb/core/ymq/http/parser.rl6 index b617c334a3..6db93b4a1f 100644 --- a/ydb/core/ymq/http/parser.rl6 +++ b/ydb/core/ymq/http/parser.rl6 @@ -112,28 +112,28 @@ get_queue_attributes_entry = (("Id" % { CurrentParams_->Id = value; }) | ("QueueUrl" % { CurrentParams_->QueueUrl = value; })); -permissions_entry = - "Permission" > { Id_ = 1; } ('.' int %{ Id_ = I; CurrentParams_ = &Params_->BatchEntries[Id_]; CurrentParams_->Action = value; }); - +permissions_entry = + "Permission" > { Id_ = 1; } ('.' int %{ Id_ = I; CurrentParams_ = &Params_->BatchEntries[Id_]; CurrentParams_->Action = value; }); + main := |* ('Action') { CurrentParams_->Action = value; }; - ('Clear') { CurrentParams_->Clear = value; }; + ('Clear') { CurrentParams_->Clear = value; }; ('DelaySeconds') { CurrentParams_->DelaySeconds = ParseAndValidate(value, TStringBuf("DelaySeconds")); }; - ('folderId') { CurrentParams_->FolderId = value; }; + ('folderId') { CurrentParams_->FolderId = value; }; ('MaxNumberOfMessages') { CurrentParams_->MaxNumberOfMessages = ParseAndValidate(value, TStringBuf("MaxNumberOfMessages")); }; ('MessageBody') { CurrentParams_->MessageBody = value; }; ('MessageDeduplicationId') { CurrentParams_->MessageDeduplicationId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("MessageDeduplicationId")); }; ('MessageGroupId') { CurrentParams_->MessageGroupId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("MessageGroupId")); }; - ('Path') { CurrentParams_->Path = value; }; + ('Path') { CurrentParams_->Path = value; }; ('QueueName') { CurrentParams_->QueueName = value; }; ('QueueNamePrefix') { CurrentParams_->QueueNamePrefix = value; }; ('QueueUrl') { CurrentParams_->QueueUrl = value; }; ('ReceiptHandle') { CurrentParams_->ReceiptHandle = value; }; ('ReceiveRequestAttemptId') { CurrentParams_->ReceiveRequestAttemptId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("ReceiveRequestAttemptId")); }; - ('Subject') { CurrentParams_->Subject = value; }; + ('Subject') { CurrentParams_->Subject = value; }; ('UserName') { CurrentParams_->UserName = value; }; ('UserNamePrefix') { CurrentParams_->UserNamePrefix = value; }; - ('Version') { CurrentParams_->Version = value; }; + ('Version') { CurrentParams_->Version = value; }; ('VisibilityTimeout') { CurrentParams_->VisibilityTimeout = ParseAndValidate(value, TStringBuf("VisibilityTimeout")); }; ('WaitTimeSeconds') { CurrentParams_->WaitTimeSeconds = ParseAndValidate(value, TStringBuf("WaitTimeSeconds")); }; @@ -142,7 +142,7 @@ main := |* change_visibility_entry; delete_message_entry; message_attribute; - permissions_entry; + permissions_entry; send_message_entry; delete_queue_entry; purge_queue_entry; diff --git a/ydb/core/ymq/http/types.h b/ydb/core/ymq/http/types.h index c33574572e..fdf98e4b02 100644 --- a/ydb/core/ymq/http/types.h +++ b/ydb/core/ymq/http/types.h @@ -14,10 +14,10 @@ struct TSqsHttpResponse { TString ContentType; int StatusCode = 0; - TString FolderId; - TString ResourceId; + TString FolderId; + TString ResourceId; bool IsFifo = false; - + TSqsHttpResponse() = default; TSqsHttpResponse(const TString& body, int status, const TString& contentType = XML_CONTENT_TYPE); }; diff --git a/ydb/core/ymq/http/xml.cpp b/ydb/core/ymq/http/xml.cpp index 0cb6f85274..d589cc9a32 100644 --- a/ydb/core/ymq/http/xml.cpp +++ b/ydb/core/ymq/http/xml.cpp @@ -69,10 +69,10 @@ static bool MaybeErrorResponse(const T& resp, TStringBuilder& builder) { return false; } -static TString BoolToString(const bool b) { - return TString(b ? "true" : "false"); -} - +static TString BoolToString(const bool b) { + return TString(b ? "true" : "false"); +} + void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStringBuilder& xmlBuilder) { if (rec.HasApproximateNumberOfMessages()) { XML_ELEM("Attribute") { @@ -95,7 +95,7 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin if (rec.HasContentBasedDeduplication()) { XML_ELEM("Attribute") { XML_ELEM_CONT("Name", "ContentBasedDeduplication"); - XML_ELEM_CONT("Value", BoolToString(rec.GetContentBasedDeduplication())); + XML_ELEM_CONT("Value", BoolToString(rec.GetContentBasedDeduplication())); } } if (rec.HasCreatedTimestamp()) { @@ -113,7 +113,7 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin if (rec.HasFifoQueue()) { XML_ELEM("Attribute") { XML_ELEM_CONT("Name", "FifoQueue"); - XML_ELEM_CONT("Value", BoolToString(rec.GetFifoQueue())); + XML_ELEM_CONT("Value", BoolToString(rec.GetFifoQueue())); } } if (rec.HasMaximumMessageSize()) { @@ -140,18 +140,18 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin XML_ELEM_CONT("Value", ToString(rec.GetVisibilityTimeout())); } } - if (rec.HasRedrivePolicy()) { - XML_ELEM("Attribute") { - XML_ELEM_CONT("Name", "RedrivePolicy"); - XML_ELEM_CONT("Value", ToString(rec.GetRedrivePolicy())); - } - } - if (rec.HasQueueArn()) { - XML_ELEM("Attribute") { - XML_ELEM_CONT("Name", "QueueArn"); - XML_ELEM_CONT("Value", ToString(rec.GetQueueArn())); - } - } + if (rec.HasRedrivePolicy()) { + XML_ELEM("Attribute") { + XML_ELEM_CONT("Name", "RedrivePolicy"); + XML_ELEM_CONT("Value", ToString(rec.GetRedrivePolicy())); + } + } + if (rec.HasQueueArn()) { + XML_ELEM("Attribute") { + XML_ELEM_CONT("Name", "QueueArn"); + XML_ELEM_CONT("Value", ToString(rec.GetQueueArn())); + } + } } TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) { @@ -409,24 +409,24 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) { break; } - case TSqsResponse::kCountQueues: { + case TSqsResponse::kCountQueues: { HANDLE_ERROR(CountQueues); - XML_BUILDER() { - XML_DOC() { - XML_ELEM("CountQueuesResponse") { - XML_ELEM("CountQueuesResult") { - XML_ELEM_CONT("Count", ::ToString(resp.GetCountQueues().GetCount())); - } - XML_ELEM("ResponseMetadata") { - XML_ELEM_CONT("RequestId", resp.GetCountQueues().GetRequestId()); - } - } - } - } - result << XML_RESULT(); - break; - } - + XML_BUILDER() { + XML_DOC() { + XML_ELEM("CountQueuesResponse") { + XML_ELEM("CountQueuesResult") { + XML_ELEM_CONT("Count", ::ToString(resp.GetCountQueues().GetCount())); + } + XML_ELEM("ResponseMetadata") { + XML_ELEM_CONT("RequestId", resp.GetCountQueues().GetRequestId()); + } + } + } + } + result << XML_RESULT(); + break; + } + case TSqsResponse::kListUsers: { HANDLE_ERROR(ListUsers); XML_BUILDER() { @@ -535,9 +535,9 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) { if (message.HasSentTimestamp()) { ATTRIBUTE("SentTimestamp", message.GetSentTimestamp()); } - if (message.HasSenderId()) { - ATTRIBUTE("SenderId", message.GetSenderId()); - } + if (message.HasSenderId()) { + ATTRIBUTE("SenderId", message.GetSenderId()); + } // message attributes for (const auto& attr : message.messageattributes()) { @@ -679,83 +679,83 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) { break; } - case TSqsResponse::kModifyPermissions: { + case TSqsResponse::kModifyPermissions: { HANDLE_ERROR(ModifyPermissions); - XML_BUILDER() { - XML_DOC() { - XML_ELEM("ModifyPermissionsResponse") { - XML_ELEM("ResponseMetadata") { - XML_ELEM_CONT("RequestId", resp.GetModifyPermissions().GetRequestId()); - } - } - } - } - result << XML_RESULT(); - break; - } - - case TSqsResponse::kListPermissions: { + XML_BUILDER() { + XML_DOC() { + XML_ELEM("ModifyPermissionsResponse") { + XML_ELEM("ResponseMetadata") { + XML_ELEM_CONT("RequestId", resp.GetModifyPermissions().GetRequestId()); + } + } + } + } + result << XML_RESULT(); + break; + } + + case TSqsResponse::kListPermissions: { HANDLE_ERROR(ListPermissions); - const auto& listPermissions = resp.GetListPermissions(); - + const auto& listPermissions = resp.GetListPermissions(); + #define SERIALIZE_PERMISSIONS(resource, permission) \ - for (size_t i = 0; i < listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Y_CAT(permission, s), Size)(); ++i) { \ - XML_ELEM_IMPL("Ya" Y_STRINGIZE(permission), Y_CAT(__LINE__, a)) { \ - const auto& permissions = listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Get, Y_CAT(permission, s))(i); \ - XML_ELEM_CONT_IMPL("Subject", permissions.GetSubject(), Y_CAT(__LINE__, b)); \ - for (size_t j = 0; j < permissions.PermissionNamesSize(); ++j) { \ - XML_ELEM_CONT_IMPL("Permission", permissions.GetPermissionNames(j), Y_CAT(__LINE__, c)); \ - } \ - } \ - } - -#define SERIALIZE_PERMISSIONS_FOR_RESOURCE(resource) \ - SERIALIZE_PERMISSIONS(resource, EffectivePermission); \ - SERIALIZE_PERMISSIONS(resource, Permission); \ - XML_ELEM_CONT("ResourceType", Y_STRINGIZE(resource)); - - XML_BUILDER() { - XML_DOC() { - XML_ELEM("ListPermissionsResponse") { - XML_ELEM("YaListPermissionsResult") { - if (listPermissions.HasQueuePermissions()) { - SERIALIZE_PERMISSIONS_FOR_RESOURCE(Queue); - } else if (listPermissions.HasAccountPermissions()) { - SERIALIZE_PERMISSIONS_FOR_RESOURCE(Account); - } - XML_ELEM("ResponseMetadata") { - XML_ELEM_CONT("RequestId", listPermissions.GetRequestId()); - } - } - } - } - } - result << XML_RESULT(); - break; - } -#undef SERIALIZE_PERMISSIONS_FOR_RESOURCE -#undef SERIALIZE_PERMISSIONS - case TSqsResponse::kListDeadLetterSourceQueues: { + for (size_t i = 0; i < listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Y_CAT(permission, s), Size)(); ++i) { \ + XML_ELEM_IMPL("Ya" Y_STRINGIZE(permission), Y_CAT(__LINE__, a)) { \ + const auto& permissions = listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Get, Y_CAT(permission, s))(i); \ + XML_ELEM_CONT_IMPL("Subject", permissions.GetSubject(), Y_CAT(__LINE__, b)); \ + for (size_t j = 0; j < permissions.PermissionNamesSize(); ++j) { \ + XML_ELEM_CONT_IMPL("Permission", permissions.GetPermissionNames(j), Y_CAT(__LINE__, c)); \ + } \ + } \ + } + +#define SERIALIZE_PERMISSIONS_FOR_RESOURCE(resource) \ + SERIALIZE_PERMISSIONS(resource, EffectivePermission); \ + SERIALIZE_PERMISSIONS(resource, Permission); \ + XML_ELEM_CONT("ResourceType", Y_STRINGIZE(resource)); + + XML_BUILDER() { + XML_DOC() { + XML_ELEM("ListPermissionsResponse") { + XML_ELEM("YaListPermissionsResult") { + if (listPermissions.HasQueuePermissions()) { + SERIALIZE_PERMISSIONS_FOR_RESOURCE(Queue); + } else if (listPermissions.HasAccountPermissions()) { + SERIALIZE_PERMISSIONS_FOR_RESOURCE(Account); + } + XML_ELEM("ResponseMetadata") { + XML_ELEM_CONT("RequestId", listPermissions.GetRequestId()); + } + } + } + } + } + result << XML_RESULT(); + break; + } +#undef SERIALIZE_PERMISSIONS_FOR_RESOURCE +#undef SERIALIZE_PERMISSIONS + case TSqsResponse::kListDeadLetterSourceQueues: { HANDLE_ERROR(ListDeadLetterSourceQueues); - XML_BUILDER() { - XML_DOC() { - XML_ELEM("ListDeadLetterSourceQueuesResponse") { - XML_ELEM("ListDeadLetterSourceQueuesResult") { - for (const auto& item : resp.GetListDeadLetterSourceQueues().queues()) { - XML_ELEM_CONT("QueueUrl", item.GetQueueUrl()); - } - } - XML_ELEM("ResponseMetadata") { - XML_ELEM_CONT("RequestId", resp.GetListDeadLetterSourceQueues().GetRequestId()); - } - } - } - } - result << XML_RESULT(); - break; - } - + XML_BUILDER() { + XML_DOC() { + XML_ELEM("ListDeadLetterSourceQueuesResponse") { + XML_ELEM("ListDeadLetterSourceQueuesResult") { + for (const auto& item : resp.GetListDeadLetterSourceQueues().queues()) { + XML_ELEM_CONT("QueueUrl", item.GetQueueUrl()); + } + } + XML_ELEM("ResponseMetadata") { + XML_ELEM_CONT("RequestId", resp.GetListDeadLetterSourceQueues().GetRequestId()); + } + } + } + } + result << XML_RESULT(); + break; + } + case TSqsResponse::RESPONSE_NOT_SET: { return MakeErrorXmlResponse(NErrors::INTERNAL_FAILURE, nullptr, "Not implemented."); } diff --git a/ydb/core/ymq/queues/fifo/queries.cpp b/ydb/core/ymq/queues/fifo/queries.cpp index 5876be81d6..e43ac55f2e 100644 --- a/ydb/core/ymq/queues/fifo/queries.cpp +++ b/ydb/core/ymq/queues/fifo/queries.cpp @@ -420,15 +420,15 @@ const char* const DeleteMessageQuery = R"__( const char* const SetQueueAttributesQuery = R"__( ( - (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64)))) - (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64)))) - (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64)))) - (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64)))) - (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64)))) + (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64)))) + (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64)))) + (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64)))) + (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64)))) + (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64)))) (let contentBasedDeduplication (Parameter 'CONTENT_BASED_DEDUPLICATION (OptionalType (DataType 'Bool)))) - (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64)))) - (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String)))) - (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String)))) + (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64)))) + (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String)))) + (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String)))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) (let attrsTable '%1$s/Attributes) @@ -441,9 +441,9 @@ const char* const SetQueueAttributesQuery = R"__( 'ReceiveMessageWaitTime 'VisibilityTimeout 'MaximumMessageSize - 'DlqName - 'DlqArn - 'MaxReceiveCount + 'DlqName + 'DlqArn + 'MaxReceiveCount 'ContentBasedDeduplication)) (let attrsRead (SelectRow attrsTable attrsRow attrsSelect)) @@ -453,9 +453,9 @@ const char* const SetQueueAttributesQuery = R"__( '('ReceiveMessageWaitTime (Coalesce wait (Member attrsRead 'ReceiveMessageWaitTime))) '('VisibilityTimeout (Coalesce visibility (Member attrsRead 'VisibilityTimeout))) '('MaximumMessageSize (Coalesce maxMessageSize (Member attrsRead 'MaximumMessageSize))) - '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount))) - '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName))) - '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn))) + '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount))) + '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName))) + '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn))) '('ContentBasedDeduplication (Coalesce contentBasedDeduplication (Member attrsRead 'ContentBasedDeduplication))))) (let queuesTable '%5$s/.Queues) @@ -472,7 +472,7 @@ const char* const SetQueueAttributesQuery = R"__( '('DlqName (Coalesce dlqName (Member queuesRowRead 'DlqName))))) (return (AsList - (UpdateRow attrsTable attrsRow attrsUpdate) + (UpdateRow attrsTable attrsRow attrsUpdate) (UpdateRow queuesTable queuesRow queuesRowUpdate))) ) )__"; @@ -490,9 +490,9 @@ const char* const InternalGetQueueAttributesQuery = R"__( 'MaximumMessageSize 'MessageRetentionPeriod 'ReceiveMessageWaitTime - 'MaxReceiveCount - 'DlqName - 'DlqArn + 'MaxReceiveCount + 'DlqName + 'DlqArn 'VisibilityTimeout 'ShowDetailedCountersDeadline)) @@ -508,24 +508,24 @@ const char* const ListQueuesQuery = R"__( (let queuesTable '%5$s/.Queues) - (let skipFolderIdFilter (Equal folderId (Utf8String '""))) - + (let skipFolderIdFilter (Equal folderId (Utf8String '""))) + (let queuesRange '( '('Account userName userName) '('QueueName (Utf8String '"") (Void)))) - (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) + (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) (let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List)) (let filtered (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (Or - (Equal (Member item 'QueueState) (Uint64 '1)) - (Equal (Member item 'QueueState) (Uint64 '3))) - (Or - (Equal (Member item 'FolderId) folderId) - skipFolderIdFilter)) - (Bool 'false))) + (return (Coalesce + (And + (Or + (Equal (Member item 'QueueState) (Uint64 '1)) + (Equal (Member item 'QueueState) (Uint64 '3))) + (Or + (Equal (Member item 'FolderId) folderId) + skipFolderIdFilter)) + (Bool 'false))) ))))) (return (AsList @@ -613,7 +613,7 @@ const char* const ReadMessageQuery = R"__( (let dataTable '%1$s/Data) (let messageTable '%1$s/Messages) - (let unfilteredResult + (let unfilteredResult (MapParameter keys (lambda '(item) (block '( (let dataRow '( '('RandomId (Member item 'RandomId)) @@ -623,8 +623,8 @@ const char* const ReadMessageQuery = R"__( 'DedupId 'Attributes 'Data - 'MessageId - 'SenderId)) + 'MessageId + 'SenderId)) (let messageRow '( '('Offset (Member item 'Offset)))) @@ -635,24 +635,24 @@ const char* const ReadMessageQuery = R"__( 'FirstReceiveTimestamp 'SentTimestamp)) - (let sourceDataFieldsRead (SelectRow dataTable dataRow dataFields)) - - (return (AsStruct - '('Valid (Exists sourceDataFieldsRead)) - '('SourceDataFieldsRead sourceDataFieldsRead) - '('SourceMessageFieldsRead (SelectRow messageTable messageRow messageFields))))))))) - - (let result - (Filter unfilteredResult (lambda '(item) (block '( - (return (Coalesce (Member item 'Valid) (Bool 'false))) - ))))) - + (let sourceDataFieldsRead (SelectRow dataTable dataRow dataFields)) + + (return (AsStruct + '('Valid (Exists sourceDataFieldsRead)) + '('SourceDataFieldsRead sourceDataFieldsRead) + '('SourceMessageFieldsRead (SelectRow messageTable messageRow messageFields))))))))) + + (let result + (Filter unfilteredResult (lambda '(item) (block '( + (return (Coalesce (Member item 'Valid) (Bool 'false))) + ))))) + (return (Extend (AsList (SetResult 'result result)) - (AsList (SetResult 'movedMessagesCount (Uint64 '0))) + (AsList (SetResult 'movedMessagesCount (Uint64 '0))) (Map result (lambda '(item) (block '( - (let message (Member item 'SourceMessageFieldsRead)) + (let message (Member item 'SourceMessageFieldsRead)) (let row '( '('Offset (Member message 'Offset)))) (let receiveTimestamp @@ -664,271 +664,271 @@ const char* const ReadMessageQuery = R"__( ) )__"; -const char* const ReadOrRedriveMessageQuery = R"__( - ( - (let keys (Parameter 'KEYS - (ListType (StructType - '('RandomId (DataType 'Uint64)) - '('GroupId (DataType 'String)) - '('Index (DataType 'Uint64)) - '('Offset (DataType 'Uint64)))))) - (let now (Parameter 'NOW (DataType 'Uint64))) - (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint32))) - (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64))) - - (let sourceDataTable '%1$s/Data) - (let sourceStateTable '%1$s/State) - (let sourceMsgTable '%1$s/Messages) - (let sourceGroupTable '%1$s/Groups) - (let sourceSentTsIdx '%1$s/SentTimestampIdx) - - (let dlqDataTable '%3$s/Data) - (let dlqDedupTable '%3$s/Deduplication) - (let dlqStateTable '%3$s/State) - (let dlqMsgTable '%3$s/Messages) - (let dlqGroupTable '%3$s/Groups) - (let dlqSentTsIdx '%3$s/SentTimestampIdx) - - (let unfilteredAllMessages - (MapParameter keys (lambda '(item) (block '( - (let dataRow '( - '('RandomId (Member item 'RandomId)) - '('Offset (Member item 'Offset)))) - (let dataFields '( - 'RandomId - 'Offset - 'DedupId - 'Attributes - 'Data - 'MessageId - 'SenderId)) - - (let messageRow '( - '('Offset (Member item 'Offset)))) - (let messageFields '( - 'GroupId - 'Offset - 'ReceiveCount - 'FirstReceiveTimestamp - 'NextOffset - 'NextRandomId - 'SentTimestamp)) - - (let dlqGroupRow '( - '('GroupId (Member item 'GroupId)))) - (let dlqGroupSelect '( - 'Head - 'ReceiveAttemptId - 'Tail)) - (let dlqGroupRead (SelectRow dlqGroupTable dlqGroupRow dlqGroupSelect)) - (let dlqTail (IfPresent dlqGroupRead (lambda '(x) (Coalesce (Member x 'Tail) (Uint64 '0))) (Uint64 '0))) - - (let sourceDataFieldsRead (SelectRow sourceDataTable dataRow dataFields)) - (let sourceMessageFieldsRead (SelectRow sourceMsgTable messageRow messageFields)) - - (return (AsStruct - '('Valid (Exists sourceDataFieldsRead)) - '('SourceDataFieldsRead sourceDataFieldsRead) - '('SourceMessageFieldsRead sourceMessageFieldsRead) - '('DlqGroupRead dlqGroupRead) - '('DlqTail dlqTail) - '('Attributes (Member sourceDataFieldsRead 'Attributes)) - '('Data (Member sourceDataFieldsRead 'Data)) - '('MessageId (Member sourceDataFieldsRead 'MessageId)) - '('SenderId (Member sourceDataFieldsRead 'SenderId)) - '('DeduplicationId (Member sourceDataFieldsRead 'DedupId)) - '('SourceRandomId (Member sourceDataFieldsRead 'RandomId)) - '('SourceOffset (Member sourceMessageFieldsRead 'Offset)) - '('SourceNextOffset (Member sourceMessageFieldsRead 'NextOffset)) - '('SourceNextRandomId (Member sourceMessageFieldsRead 'NextRandomId)) - '('SourceSentTimestamp (Member sourceMessageFieldsRead 'SentTimestamp)) - '('GroupId (Member item 'GroupId)) - '('Delay (Uint64 '0)) - '('Index (Member item 'Index))))))))) - - (let allMessages - (Filter unfilteredAllMessages (lambda '(item) (block '( - (return (Coalesce (Member item 'Valid) (Bool 'false))) - ))))) - - (let messagesToMoveAsStruct - (Filter allMessages (lambda '(item) (block '( - (let message (Member item 'SourceMessageFieldsRead)) - (return (Coalesce (GreaterOrEqual (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false))) - ))))) - - (let messagesToReturnAsStruct - (Filter allMessages (lambda '(item) (block '( - (let message (Member item 'SourceMessageFieldsRead)) - (return (Coalesce (Less (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false))) - ))))) - - (let dlqStateRow '( - '('State (Uint64 '0)))) - (let dlqStateSelect '( - 'MessageCount - 'WriteOffset - 'LastModifiedTimestamp)) - (let dlqStateRead (SelectRow dlqStateTable dlqStateRow dlqStateSelect)) - - (let dlqSentTimestamp (Max now (Member dlqStateRead 'LastModifiedTimestamp))) - (let dlqStartOffset (Add (Member dlqStateRead 'WriteOffset) (Uint64 '1))) - (let dlqNewWriteOffset (Add (Member dlqStateRead 'WriteOffset) (Length messagesToMoveAsStruct))) - - (let dlqStateUpdate '( - '('LastModifiedTimestamp dlqSentTimestamp) - '('MessageCount (Add (Member dlqStateRead 'MessageCount) (Length messagesToMoveAsStruct))) - '('WriteOffset dlqNewWriteOffset))) - - (let dlqMessagesInfoWithProperIndexes - (Enumerate messagesToMoveAsStruct (Coalesce dlqStartOffset (Uint64 '1)))) - - (let dlqMessagesInfoWithProperIndexesSorted - (Sort dlqMessagesInfoWithProperIndexes (Bool 'true) (lambda '(item) (block '( - (return (Member (Nth item '1) 'Index)) - )))) - ) - - (let sourceStateRow '( - '('State (Uint64 '0)))) - (let sourceStateSelect '( - 'MessageCount - 'LastModifiedTimestamp)) - (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect)) - (let newSourceMsgCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMoveAsStruct))) - (let sourceLastModifiedTimestamp (Max now (Member sourceStateRead 'LastModifiedTimestamp))) - - (let sourceStateUpdate '( - '('LastModifiedTimestamp sourceLastModifiedTimestamp) - '('MessageCount newSourceMsgCount))) - - (return (Extend - (AsList (SetResult 'result messagesToReturnAsStruct)) - (AsList (SetResult 'movedMessagesCount (Length messagesToMoveAsStruct))) +const char* const ReadOrRedriveMessageQuery = R"__( + ( + (let keys (Parameter 'KEYS + (ListType (StructType + '('RandomId (DataType 'Uint64)) + '('GroupId (DataType 'String)) + '('Index (DataType 'Uint64)) + '('Offset (DataType 'Uint64)))))) + (let now (Parameter 'NOW (DataType 'Uint64))) + (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint32))) + (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64))) + + (let sourceDataTable '%1$s/Data) + (let sourceStateTable '%1$s/State) + (let sourceMsgTable '%1$s/Messages) + (let sourceGroupTable '%1$s/Groups) + (let sourceSentTsIdx '%1$s/SentTimestampIdx) + + (let dlqDataTable '%3$s/Data) + (let dlqDedupTable '%3$s/Deduplication) + (let dlqStateTable '%3$s/State) + (let dlqMsgTable '%3$s/Messages) + (let dlqGroupTable '%3$s/Groups) + (let dlqSentTsIdx '%3$s/SentTimestampIdx) + + (let unfilteredAllMessages + (MapParameter keys (lambda '(item) (block '( + (let dataRow '( + '('RandomId (Member item 'RandomId)) + '('Offset (Member item 'Offset)))) + (let dataFields '( + 'RandomId + 'Offset + 'DedupId + 'Attributes + 'Data + 'MessageId + 'SenderId)) + + (let messageRow '( + '('Offset (Member item 'Offset)))) + (let messageFields '( + 'GroupId + 'Offset + 'ReceiveCount + 'FirstReceiveTimestamp + 'NextOffset + 'NextRandomId + 'SentTimestamp)) + + (let dlqGroupRow '( + '('GroupId (Member item 'GroupId)))) + (let dlqGroupSelect '( + 'Head + 'ReceiveAttemptId + 'Tail)) + (let dlqGroupRead (SelectRow dlqGroupTable dlqGroupRow dlqGroupSelect)) + (let dlqTail (IfPresent dlqGroupRead (lambda '(x) (Coalesce (Member x 'Tail) (Uint64 '0))) (Uint64 '0))) + + (let sourceDataFieldsRead (SelectRow sourceDataTable dataRow dataFields)) + (let sourceMessageFieldsRead (SelectRow sourceMsgTable messageRow messageFields)) + + (return (AsStruct + '('Valid (Exists sourceDataFieldsRead)) + '('SourceDataFieldsRead sourceDataFieldsRead) + '('SourceMessageFieldsRead sourceMessageFieldsRead) + '('DlqGroupRead dlqGroupRead) + '('DlqTail dlqTail) + '('Attributes (Member sourceDataFieldsRead 'Attributes)) + '('Data (Member sourceDataFieldsRead 'Data)) + '('MessageId (Member sourceDataFieldsRead 'MessageId)) + '('SenderId (Member sourceDataFieldsRead 'SenderId)) + '('DeduplicationId (Member sourceDataFieldsRead 'DedupId)) + '('SourceRandomId (Member sourceDataFieldsRead 'RandomId)) + '('SourceOffset (Member sourceMessageFieldsRead 'Offset)) + '('SourceNextOffset (Member sourceMessageFieldsRead 'NextOffset)) + '('SourceNextRandomId (Member sourceMessageFieldsRead 'NextRandomId)) + '('SourceSentTimestamp (Member sourceMessageFieldsRead 'SentTimestamp)) + '('GroupId (Member item 'GroupId)) + '('Delay (Uint64 '0)) + '('Index (Member item 'Index))))))))) + + (let allMessages + (Filter unfilteredAllMessages (lambda '(item) (block '( + (return (Coalesce (Member item 'Valid) (Bool 'false))) + ))))) + + (let messagesToMoveAsStruct + (Filter allMessages (lambda '(item) (block '( + (let message (Member item 'SourceMessageFieldsRead)) + (return (Coalesce (GreaterOrEqual (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false))) + ))))) + + (let messagesToReturnAsStruct + (Filter allMessages (lambda '(item) (block '( + (let message (Member item 'SourceMessageFieldsRead)) + (return (Coalesce (Less (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false))) + ))))) + + (let dlqStateRow '( + '('State (Uint64 '0)))) + (let dlqStateSelect '( + 'MessageCount + 'WriteOffset + 'LastModifiedTimestamp)) + (let dlqStateRead (SelectRow dlqStateTable dlqStateRow dlqStateSelect)) + + (let dlqSentTimestamp (Max now (Member dlqStateRead 'LastModifiedTimestamp))) + (let dlqStartOffset (Add (Member dlqStateRead 'WriteOffset) (Uint64 '1))) + (let dlqNewWriteOffset (Add (Member dlqStateRead 'WriteOffset) (Length messagesToMoveAsStruct))) + + (let dlqStateUpdate '( + '('LastModifiedTimestamp dlqSentTimestamp) + '('MessageCount (Add (Member dlqStateRead 'MessageCount) (Length messagesToMoveAsStruct))) + '('WriteOffset dlqNewWriteOffset))) + + (let dlqMessagesInfoWithProperIndexes + (Enumerate messagesToMoveAsStruct (Coalesce dlqStartOffset (Uint64 '1)))) + + (let dlqMessagesInfoWithProperIndexesSorted + (Sort dlqMessagesInfoWithProperIndexes (Bool 'true) (lambda '(item) (block '( + (return (Member (Nth item '1) 'Index)) + )))) + ) + + (let sourceStateRow '( + '('State (Uint64 '0)))) + (let sourceStateSelect '( + 'MessageCount + 'LastModifiedTimestamp)) + (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect)) + (let newSourceMsgCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMoveAsStruct))) + (let sourceLastModifiedTimestamp (Max now (Member sourceStateRead 'LastModifiedTimestamp))) + + (let sourceStateUpdate '( + '('LastModifiedTimestamp sourceLastModifiedTimestamp) + '('MessageCount newSourceMsgCount))) + + (return (Extend + (AsList (SetResult 'result messagesToReturnAsStruct)) + (AsList (SetResult 'movedMessagesCount (Length messagesToMoveAsStruct))) (AsList (SetResult 'newMessagesCount newSourceMsgCount)) - (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow dlqStateTable dlqStateRow dlqStateUpdate)) - (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate)) - - # copy messages to dlq - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let dataRow '( - '('RandomId randomId) - '('Offset (Nth item '0)))) - (let dataUpdate '( - '('Data (Member (Nth item '1) 'Data)) - '('DedupId (Member (Nth item '1) 'DeduplicationId)) - '('Attributes (Member (Nth item '1) 'Attributes)) - '('SenderId (Member (Nth item '1) 'SenderId)) - '('MessageId (Member (Nth item '1) 'MessageId)))) - (return (UpdateRow dlqDataTable dataRow dataUpdate)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let msgRow '( - '('Offset (Nth item '0)))) - (let messageUpdate '( - '('RandomId randomId) - '('GroupId (Member (Nth item '1) 'GroupId)) - '('NextOffset (Uint64 '0)) - '('NextRandomId (Uint64 '0)) - '('ReceiveCount (Uint32 '0)) - '('FirstReceiveTimestamp (Uint64 '0)) - '('SentTimestamp dlqSentTimestamp))) - (return (UpdateRow dlqMsgTable msgRow messageUpdate)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let sentTsRow '( - '('SentTimestamp dlqSentTimestamp) - '('Offset (Nth item '0)))) - (let delay (Member (Nth item '1) 'Delay)) - (let delayDeadline (Uint64 '0)) - (let sentTsUpdate '( - '('RandomId randomId) - '('DelayDeadline delayDeadline) - '('GroupId (Member (Nth item '1) 'GroupId)))) - (return (UpdateRow dlqSentTsIdx sentTsRow sentTsUpdate)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let groupRow '( - '('GroupId (Member (Nth item '1) 'GroupId)))) - (let delay (Member (Nth item '1) 'Delay)) - (let delayDeadline (Uint64 '0)) - (let groupInsert '( - '('RandomId randomId) - '('Head (Nth item '0)) - '('Tail (Nth item '0)) - '('LockTimestamp (Uint64 '0)) - '('VisibilityDeadline (Uint64 '0)))) - (let dlqGroupRead (Member (Nth item '1) 'DlqGroupRead)) - (let groupUpdate '( - '('Head (Member dlqGroupRead 'Head)) - '('Tail (Nth item '0)))) - (let dlqTail (Member (Nth item '1) 'DlqTail)) - (return - (If (Equal dlqTail (Uint64 '0)) - (UpdateRow dlqGroupTable groupRow groupInsert) - (UpdateRow dlqGroupTable groupRow groupUpdate) - ) - ))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let dlqTail (Member (Nth item '1) 'DlqTail)) - (let prevMessageRow '( - '('Offset dlqTail))) - (let prevMessageUpdate '( - '('NextOffset (Nth item '0)) - '('NextRandomId randomId))) - (return - (If (NotEqual dlqTail (Uint64 '0)) - (UpdateRow dlqMsgTable prevMessageRow prevMessageUpdate) - (Void)) - ))))) - - # remove dead letters' content from source queue - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let row '( - '('RandomId (Member (Nth item '1) 'SourceRandomId)) - '('Offset (Member (Nth item '1) 'SourceOffset)))) - (return (EraseRow sourceDataTable row)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let row '( - '('Offset (Member (Nth item '1) 'SourceOffset)))) - (return (EraseRow sourceMsgTable row)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let row '( - '('SentTimestamp (Member (Nth item '1) 'SourceSentTimestamp)) - '('Offset (Member (Nth item '1) 'SourceOffset)))) - (return (EraseRow sourceSentTsIdx row)))))) - - (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( - (let row '( - '('GroupId (Member (Nth item '1) 'GroupId)))) - (let update '( - '('RandomId (Member (Nth item '1) 'SourceNextRandomId)) - '('Head (Member (Nth item '1) 'SourceNextOffset)) - '('LockTimestamp (Uint64 '0)) - '('VisibilityDeadline (Uint64 '0)))) - - (return - (If (Coalesce (Equal (Member (Nth item '1) 'SourceNextOffset) (Uint64 '0)) (Bool 'false)) - (EraseRow sourceGroupTable row) - (UpdateRow sourceGroupTable row update))))))) - - # just return ordinary messages - (Map messagesToReturnAsStruct (lambda '(item) (block '( - (let message (Member item 'SourceMessageFieldsRead)) - (let row '( - '('Offset (Member message 'Offset)))) - (let receiveTimestamp - (If (Coalesce (Equal (Member message 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member message 'FirstReceiveTimestamp))) - (let update '( - '('FirstReceiveTimestamp receiveTimestamp) - '('ReceiveCount (Add (Member message 'ReceiveCount) (Uint32 '1))))) - (return (UpdateRow sourceMsgTable row update)))))))) - ) -)__"; - + (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow dlqStateTable dlqStateRow dlqStateUpdate)) + (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate)) + + # copy messages to dlq + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let dataRow '( + '('RandomId randomId) + '('Offset (Nth item '0)))) + (let dataUpdate '( + '('Data (Member (Nth item '1) 'Data)) + '('DedupId (Member (Nth item '1) 'DeduplicationId)) + '('Attributes (Member (Nth item '1) 'Attributes)) + '('SenderId (Member (Nth item '1) 'SenderId)) + '('MessageId (Member (Nth item '1) 'MessageId)))) + (return (UpdateRow dlqDataTable dataRow dataUpdate)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let msgRow '( + '('Offset (Nth item '0)))) + (let messageUpdate '( + '('RandomId randomId) + '('GroupId (Member (Nth item '1) 'GroupId)) + '('NextOffset (Uint64 '0)) + '('NextRandomId (Uint64 '0)) + '('ReceiveCount (Uint32 '0)) + '('FirstReceiveTimestamp (Uint64 '0)) + '('SentTimestamp dlqSentTimestamp))) + (return (UpdateRow dlqMsgTable msgRow messageUpdate)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let sentTsRow '( + '('SentTimestamp dlqSentTimestamp) + '('Offset (Nth item '0)))) + (let delay (Member (Nth item '1) 'Delay)) + (let delayDeadline (Uint64 '0)) + (let sentTsUpdate '( + '('RandomId randomId) + '('DelayDeadline delayDeadline) + '('GroupId (Member (Nth item '1) 'GroupId)))) + (return (UpdateRow dlqSentTsIdx sentTsRow sentTsUpdate)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let groupRow '( + '('GroupId (Member (Nth item '1) 'GroupId)))) + (let delay (Member (Nth item '1) 'Delay)) + (let delayDeadline (Uint64 '0)) + (let groupInsert '( + '('RandomId randomId) + '('Head (Nth item '0)) + '('Tail (Nth item '0)) + '('LockTimestamp (Uint64 '0)) + '('VisibilityDeadline (Uint64 '0)))) + (let dlqGroupRead (Member (Nth item '1) 'DlqGroupRead)) + (let groupUpdate '( + '('Head (Member dlqGroupRead 'Head)) + '('Tail (Nth item '0)))) + (let dlqTail (Member (Nth item '1) 'DlqTail)) + (return + (If (Equal dlqTail (Uint64 '0)) + (UpdateRow dlqGroupTable groupRow groupInsert) + (UpdateRow dlqGroupTable groupRow groupUpdate) + ) + ))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let dlqTail (Member (Nth item '1) 'DlqTail)) + (let prevMessageRow '( + '('Offset dlqTail))) + (let prevMessageUpdate '( + '('NextOffset (Nth item '0)) + '('NextRandomId randomId))) + (return + (If (NotEqual dlqTail (Uint64 '0)) + (UpdateRow dlqMsgTable prevMessageRow prevMessageUpdate) + (Void)) + ))))) + + # remove dead letters' content from source queue + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let row '( + '('RandomId (Member (Nth item '1) 'SourceRandomId)) + '('Offset (Member (Nth item '1) 'SourceOffset)))) + (return (EraseRow sourceDataTable row)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let row '( + '('Offset (Member (Nth item '1) 'SourceOffset)))) + (return (EraseRow sourceMsgTable row)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let row '( + '('SentTimestamp (Member (Nth item '1) 'SourceSentTimestamp)) + '('Offset (Member (Nth item '1) 'SourceOffset)))) + (return (EraseRow sourceSentTsIdx row)))))) + + (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '( + (let row '( + '('GroupId (Member (Nth item '1) 'GroupId)))) + (let update '( + '('RandomId (Member (Nth item '1) 'SourceNextRandomId)) + '('Head (Member (Nth item '1) 'SourceNextOffset)) + '('LockTimestamp (Uint64 '0)) + '('VisibilityDeadline (Uint64 '0)))) + + (return + (If (Coalesce (Equal (Member (Nth item '1) 'SourceNextOffset) (Uint64 '0)) (Bool 'false)) + (EraseRow sourceGroupTable row) + (UpdateRow sourceGroupTable row update))))))) + + # just return ordinary messages + (Map messagesToReturnAsStruct (lambda '(item) (block '( + (let message (Member item 'SourceMessageFieldsRead)) + (let row '( + '('Offset (Member message 'Offset)))) + (let receiveTimestamp + (If (Coalesce (Equal (Member message 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member message 'FirstReceiveTimestamp))) + (let update '( + '('FirstReceiveTimestamp receiveTimestamp) + '('ReceiveCount (Add (Member message 'ReceiveCount) (Uint32 '1))))) + (return (UpdateRow sourceMsgTable row update)))))))) + ) +)__"; + const char* const WriteMessageQuery = R"__( ( (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64))) @@ -1333,53 +1333,53 @@ const char* const GetRetentionOffsetQuery = R"__( ) )__"; -const char* const ListDeadLetterSourceQueuesQuery = R"__( - ( +const char* const ListDeadLetterSourceQueuesQuery = R"__( + ( (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - + (let queuesTable '%5$s/.Queues) (let queuesRow '( '('Account userName) - '('QueueName (Utf8String '"%4$s")))) - + '('QueueName (Utf8String '"%4$s")))) + (let queuesRowSelect '( - 'QueueName - 'CustomQueueName)) - + 'QueueName + 'CustomQueueName)) + (let queuesRowRead (SelectRow queuesTable queuesRow queuesRowSelect)) - - (let skipFolderIdFilter (Equal folderId (Utf8String '""))) - - (let dlqName + + (let skipFolderIdFilter (Equal folderId (Utf8String '""))) + + (let dlqName (If skipFolderIdFilter (Member queuesRowRead 'QueueName) (Coalesce (Member queuesRowRead 'CustomQueueName) (Utf8String '"")))) - + (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) + '('QueueName (Utf8String '"") (Void)))) (let queuesSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName)) (let queues (Member (SelectRange queuesTable queuesRange queuesSelect '()) 'List)) - - (let filtered (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (And - (Or - (Equal (Member item 'QueueState) (Uint64 '1)) - (Equal (Member item 'QueueState) (Uint64 '3))) - (Or - (Equal (Member item 'FolderId) folderId) - skipFolderIdFilter)) - (Equal (Member item 'DlqName) dlqName)) - (Bool 'false))) - ))))) - - (return (AsList - (SetResult 'queues filtered))) - ) -)__"; - + + (let filtered (Filter queues (lambda '(item) (block '( + (return (Coalesce + (And + (And + (Or + (Equal (Member item 'QueueState) (Uint64 '1)) + (Equal (Member item 'QueueState) (Uint64 '3))) + (Or + (Equal (Member item 'FolderId) folderId) + skipFolderIdFilter)) + (Equal (Member item 'DlqName) dlqName)) + (Bool 'false))) + ))))) + + (return (AsList + (SetResult 'queues filtered))) + ) +)__"; + } // namespace const char* GetFifoQueryById(size_t id) { @@ -1416,10 +1416,10 @@ const char* GetFifoQueryById(size_t id) { return GetOldestMessageTimestampMetricsQuery; case GET_RETENTION_OFFSET_ID: // 17 return GetRetentionOffsetQuery; - case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 18 - return ListDeadLetterSourceQueuesQuery; - case READ_OR_REDRIVE_MESSAGE_ID: // 22 - return ReadOrRedriveMessageQuery; + case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 18 + return ListDeadLetterSourceQueuesQuery; + case READ_OR_REDRIVE_MESSAGE_ID: // 22 + return ReadOrRedriveMessageQuery; } return nullptr; diff --git a/ydb/core/ymq/queues/fifo/schema.cpp b/ydb/core/ymq/queues/fifo/schema.cpp index 49e04d773b..da394c7599 100644 --- a/ydb/core/ymq/queues/fifo/schema.cpp +++ b/ydb/core/ymq/queues/fifo/schema.cpp @@ -10,9 +10,9 @@ static const TVector<TColumn> AttributesColumns = { TColumn("MaximumMessageSize", NScheme::NTypeIds::Uint64), TColumn("MessageRetentionPeriod", NScheme::NTypeIds::Uint64), TColumn("ReceiveMessageWaitTime", NScheme::NTypeIds::Uint64), - TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64), - TColumn("DlqName", NScheme::NTypeIds::Utf8), - TColumn("DlqArn", NScheme::NTypeIds::Utf8), + TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64), + TColumn("DlqName", NScheme::NTypeIds::Utf8), + TColumn("DlqArn", NScheme::NTypeIds::Utf8), TColumn("MaxReceiveCount", NScheme::NTypeIds::Uint64), TColumn("ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64)}; @@ -36,7 +36,7 @@ static const TVector<TColumn> DataColumns = { TColumn("Attributes", NScheme::NTypeIds::String), TColumn("Data", NScheme::NTypeIds::String), TColumn("MessageId", NScheme::NTypeIds::String), - TColumn("SenderId", NScheme::NTypeIds::String), + TColumn("SenderId", NScheme::NTypeIds::String), }; static const TVector<TColumn> MessagesColumns = { diff --git a/ydb/core/ymq/queues/std/queries.cpp b/ydb/core/ymq/queues/std/queries.cpp index 202d9c1489..442ae6389f 100644 --- a/ydb/core/ymq/queues/std/queries.cpp +++ b/ydb/core/ymq/queues/std/queries.cpp @@ -415,14 +415,14 @@ const char* const DeleteMessageQuery = R"__( const char* const SetQueueAttributesQuery = R"__( ( - (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64)))) - (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64)))) - (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64)))) - (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64)))) - (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64)))) - (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64)))) - (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String)))) - (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String)))) + (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64)))) + (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64)))) + (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64)))) + (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64)))) + (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64)))) + (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64)))) + (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String)))) + (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String)))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) (let attrsTable '%1$s/Attributes) @@ -434,10 +434,10 @@ const char* const SetQueueAttributesQuery = R"__( 'MessageRetentionPeriod 'ReceiveMessageWaitTime 'VisibilityTimeout - 'MaximumMessageSize - 'DlqName - 'DlqArn - 'MaxReceiveCount)) + 'MaximumMessageSize + 'DlqName + 'DlqArn + 'MaxReceiveCount)) (let attrsRead (SelectRow attrsTable attrsRow attrsSelect)) (let attrsUpdate '( @@ -445,9 +445,9 @@ const char* const SetQueueAttributesQuery = R"__( '('MessageRetentionPeriod (Coalesce retention (Member attrsRead 'MessageRetentionPeriod))) '('ReceiveMessageWaitTime (Coalesce wait (Member attrsRead 'ReceiveMessageWaitTime))) '('VisibilityTimeout (Coalesce visibility (Member attrsRead 'VisibilityTimeout))) - '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount))) - '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName))) - '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn))) + '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount))) + '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName))) + '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn))) '('MaximumMessageSize (Coalesce maxMessageSize (Member attrsRead 'MaximumMessageSize))))) (let queuesTable '%5$s/.Queues) @@ -464,7 +464,7 @@ const char* const SetQueueAttributesQuery = R"__( '('DlqName (Coalesce dlqName (Member queuesRowRead 'DlqName))))) (return (AsList - (UpdateRow attrsTable attrsRow attrsUpdate) + (UpdateRow attrsTable attrsRow attrsUpdate) (UpdateRow queuesTable queuesRow queuesRowUpdate))) ) )__"; @@ -482,9 +482,9 @@ const char* const InternalGetQueueAttributesQuery = R"__( 'MaximumMessageSize 'MessageRetentionPeriod 'ReceiveMessageWaitTime - 'MaxReceiveCount - 'DlqName - 'DlqArn + 'MaxReceiveCount + 'DlqName + 'DlqArn 'VisibilityTimeout 'ShowDetailedCountersDeadline)) @@ -500,24 +500,24 @@ const char* const ListQueuesQuery = R"__( (let queuesTable '%5$s/.Queues) - (let skipFolderIdFilter (Equal folderId (Utf8String '""))) - + (let skipFolderIdFilter (Equal folderId (Utf8String '""))) + (let queuesRange '( '('Account userName userName) '('QueueName (Utf8String '"") (Void)))) - (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) + (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) (let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List)) (let filtered (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (Or - (Equal (Member item 'QueueState) (Uint64 '1)) - (Equal (Member item 'QueueState) (Uint64 '3))) - (Or - (Equal (Member item 'FolderId) folderId) - skipFolderIdFilter)) - (Bool 'false))) + (return (Coalesce + (And + (Or + (Equal (Member item 'QueueState) (Uint64 '1)) + (Equal (Member item 'QueueState) (Uint64 '3))) + (Or + (Equal (Member item 'FolderId) folderId) + skipFolderIdFilter)) + (Bool 'false))) ))))) (return (AsList @@ -532,8 +532,8 @@ const char* const LoadMessageQuery = R"__( '('RandomId (DataType 'Uint64)) '('Offset (DataType 'Uint64)) '('CurrentVisibilityDeadline (DataType 'Uint64)) - '('DlqIndex (DataType 'Uint64)) - '('IsDeadLetter (DataType 'Bool)) + '('DlqIndex (DataType 'Uint64)) + '('IsDeadLetter (DataType 'Bool)) '('VisibilityDeadline (DataType 'Uint64)))))) (let now (Parameter 'NOW (DataType 'Uint64))) (let readId (Parameter 'READ_ID (DataType 'Uint64))) @@ -563,7 +563,7 @@ const char* const LoadMessageQuery = R"__( (let fields '( 'Attributes 'Data - 'SenderId + 'SenderId 'MessageId)) (return (SelectRow dataTable row fields))))) @@ -596,12 +596,12 @@ const char* const LoadMessageQuery = R"__( '('LoadId readId) '('Attributes (Member data 'Attributes)) '('Data (Member data 'Data)) - '('SenderId (Member data 'SenderId)) + '('SenderId (Member data 'SenderId)) '('MessageId (Member data 'MessageId)) '('FirstReceiveTimestamp receiveTimestamp) '('LockTimestamp now) - '('IsDeadLetter (Member item 'IsDeadLetter)) - '('DlqIndex (Member item 'DlqIndex)) + '('IsDeadLetter (Member item 'IsDeadLetter)) + '('DlqIndex (Member item 'DlqIndex)) '('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1))) '('SentTimestamp (Member read 'SentTimestamp)) '('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline))) @@ -615,7 +615,7 @@ const char* const LoadMessageQuery = R"__( (return (Extend (AsList (SetResult 'result records)) - (AsList (SetResult 'movedMessagesCount (Uint64 '0))) + (AsList (SetResult 'movedMessagesCount (Uint64 '0))) (Map result (lambda '(item) (block '( (let row '( @@ -630,204 +630,204 @@ const char* const LoadMessageQuery = R"__( ) )__"; -const char* const LoadOrRedriveMessageQuery = R"__( - ( - (let keys (Parameter 'KEYS - (ListType (StructType - '('RandomId (DataType 'Uint64)) - '('Offset (DataType 'Uint64)) - '('CurrentVisibilityDeadline (DataType 'Uint64)) - '('DlqIndex (DataType 'Uint64)) - '('IsDeadLetter (DataType 'Bool)) - '('VisibilityDeadline (DataType 'Uint64)))))) - - (let now (Parameter 'NOW (DataType 'Uint64))) - (let readId (Parameter 'READ_ID (DataType 'Uint64))) - (let shard (Parameter 'SHARD (DataType 'Uint64))) - (let deadLettersCount (Parameter 'DEAD_LETTERS_COUNT (DataType 'Uint64))) - - (let sourceMessageDataTable '%1$s/%2$lu/MessageData) - (let sourceSentTsIdxTable '%1$s/%2$lu/SentTimestampIdx) - (let sourceInflyTable '%1$s/%2$lu/Infly) - (let sourceStateTable '%1$s/State) - - (let deadLetterMessageDataTable '%3$s/%4$lu/MessageData) - (let deadLetterMessagesTable '%3$s/%4$lu/Messages) - (let deadLetterSentTsIdxTable '%3$s/%4$lu/SentTimestampIdx) - (let deadLetterStateTable '%3$s/State) - - (let sourceStateRow '( - '('State (Uint64 '%2$lu)))) - (let sourceStateSelect '( - 'MessageCount - 'LastModifiedTimestamp - 'InflyCount)) - (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect)) - - (let records - (MapParameter keys (lambda '(item) (block '( - (let read (block '( - (let row '( - '('Offset (Member item 'Offset)))) - (let fields '( - 'LoadId - 'FirstReceiveTimestamp - 'ReceiveCount - 'SentTimestamp - 'VisibilityDeadline)) - (return (SelectRow sourceInflyTable row fields))))) - - (let data (block '( - (let row '( - '('RandomId (Member item 'RandomId)) - '('Offset (Member item 'Offset)))) - (let fields '( - 'Attributes - 'Data - 'SenderId - 'MessageId)) - (return (SelectRow sourceMessageDataTable row fields))))) - - (let receiveTimestamp - (If (Coalesce (Equal (Member read 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member read 'FirstReceiveTimestamp))) - - (let valid - (Coalesce - (Or - (Equal (Member read 'VisibilityDeadline) (Member item 'CurrentVisibilityDeadline)) - (And - (Equal (Member read 'LoadId) readId) - (Equal (Member read 'VisibilityDeadline) (Member item 'VisibilityDeadline)))) - (Bool 'false))) - - (return - (AsStruct - '('Offset (Member item 'Offset)) - '('RandomId (Member item 'RandomId)) - '('IsDeadLetter (Member item 'IsDeadLetter)) - '('DlqIndex (Member item 'DlqIndex)) - '('LoadId readId) - '('Attributes (Member data 'Attributes)) - '('Data (Member data 'Data)) - '('SenderId (Member data 'SenderId)) - '('MessageId (Member data 'MessageId)) - '('FirstReceiveTimestamp receiveTimestamp) - '('LockTimestamp now) - '('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1))) - '('SentTimestamp (Member read 'SentTimestamp)) - '('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline))) - '('Valid valid) - '('Exists (Exists read)))) - ))))) - - (let validMessages - (Filter records (lambda '(item) (block '( - (return (Coalesce (Member item 'Valid) (Bool 'false)))))))) - - (let messagesToMove - (Filter validMessages (lambda '(item) (block '( - (return (Coalesce (Member item 'IsDeadLetter) (Bool 'false)))))))) - - (let messagesToUpdate - (Filter validMessages (lambda '(item) (block '( - (return (Coalesce (Not (Member item 'IsDeadLetter)) (Bool 'false)))))))) - - (let deadLetterStateRow '( - '('State (Uint64 '%4$lu)))) - (let deadLetterStateSelect '( - 'MessageCount - 'WriteOffset - 'LastModifiedTimestamp)) - (let deadLetterStateRead (SelectRow deadLetterStateTable deadLetterStateRow deadLetterStateSelect)) - - (let newDlqMessagesCount (Add (Member deadLetterStateRead 'MessageCount) (Length messagesToMove))) - (let newDlqWriteOffset (Add (Member deadLetterStateRead 'WriteOffset) deadLettersCount)) - (let dlqStartOffset (Add (Member deadLetterStateRead 'WriteOffset) (Uint64 '1))) - - (let dlqMostRecentTimestamp (Max (Member deadLetterStateRead 'LastModifiedTimestamp) now)) - - (let deadLetterStateUpdate '( - '('LastModifiedTimestamp dlqMostRecentTimestamp) - '('MessageCount newDlqMessagesCount) - '('WriteOffset newDlqWriteOffset))) - +const char* const LoadOrRedriveMessageQuery = R"__( + ( + (let keys (Parameter 'KEYS + (ListType (StructType + '('RandomId (DataType 'Uint64)) + '('Offset (DataType 'Uint64)) + '('CurrentVisibilityDeadline (DataType 'Uint64)) + '('DlqIndex (DataType 'Uint64)) + '('IsDeadLetter (DataType 'Bool)) + '('VisibilityDeadline (DataType 'Uint64)))))) + + (let now (Parameter 'NOW (DataType 'Uint64))) + (let readId (Parameter 'READ_ID (DataType 'Uint64))) + (let shard (Parameter 'SHARD (DataType 'Uint64))) + (let deadLettersCount (Parameter 'DEAD_LETTERS_COUNT (DataType 'Uint64))) + + (let sourceMessageDataTable '%1$s/%2$lu/MessageData) + (let sourceSentTsIdxTable '%1$s/%2$lu/SentTimestampIdx) + (let sourceInflyTable '%1$s/%2$lu/Infly) + (let sourceStateTable '%1$s/State) + + (let deadLetterMessageDataTable '%3$s/%4$lu/MessageData) + (let deadLetterMessagesTable '%3$s/%4$lu/Messages) + (let deadLetterSentTsIdxTable '%3$s/%4$lu/SentTimestampIdx) + (let deadLetterStateTable '%3$s/State) + + (let sourceStateRow '( + '('State (Uint64 '%2$lu)))) + (let sourceStateSelect '( + 'MessageCount + 'LastModifiedTimestamp + 'InflyCount)) + (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect)) + + (let records + (MapParameter keys (lambda '(item) (block '( + (let read (block '( + (let row '( + '('Offset (Member item 'Offset)))) + (let fields '( + 'LoadId + 'FirstReceiveTimestamp + 'ReceiveCount + 'SentTimestamp + 'VisibilityDeadline)) + (return (SelectRow sourceInflyTable row fields))))) + + (let data (block '( + (let row '( + '('RandomId (Member item 'RandomId)) + '('Offset (Member item 'Offset)))) + (let fields '( + 'Attributes + 'Data + 'SenderId + 'MessageId)) + (return (SelectRow sourceMessageDataTable row fields))))) + + (let receiveTimestamp + (If (Coalesce (Equal (Member read 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member read 'FirstReceiveTimestamp))) + + (let valid + (Coalesce + (Or + (Equal (Member read 'VisibilityDeadline) (Member item 'CurrentVisibilityDeadline)) + (And + (Equal (Member read 'LoadId) readId) + (Equal (Member read 'VisibilityDeadline) (Member item 'VisibilityDeadline)))) + (Bool 'false))) + + (return + (AsStruct + '('Offset (Member item 'Offset)) + '('RandomId (Member item 'RandomId)) + '('IsDeadLetter (Member item 'IsDeadLetter)) + '('DlqIndex (Member item 'DlqIndex)) + '('LoadId readId) + '('Attributes (Member data 'Attributes)) + '('Data (Member data 'Data)) + '('SenderId (Member data 'SenderId)) + '('MessageId (Member data 'MessageId)) + '('FirstReceiveTimestamp receiveTimestamp) + '('LockTimestamp now) + '('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1))) + '('SentTimestamp (Member read 'SentTimestamp)) + '('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline))) + '('Valid valid) + '('Exists (Exists read)))) + ))))) + + (let validMessages + (Filter records (lambda '(item) (block '( + (return (Coalesce (Member item 'Valid) (Bool 'false)))))))) + + (let messagesToMove + (Filter validMessages (lambda '(item) (block '( + (return (Coalesce (Member item 'IsDeadLetter) (Bool 'false)))))))) + + (let messagesToUpdate + (Filter validMessages (lambda '(item) (block '( + (return (Coalesce (Not (Member item 'IsDeadLetter)) (Bool 'false)))))))) + + (let deadLetterStateRow '( + '('State (Uint64 '%4$lu)))) + (let deadLetterStateSelect '( + 'MessageCount + 'WriteOffset + 'LastModifiedTimestamp)) + (let deadLetterStateRead (SelectRow deadLetterStateTable deadLetterStateRow deadLetterStateSelect)) + + (let newDlqMessagesCount (Add (Member deadLetterStateRead 'MessageCount) (Length messagesToMove))) + (let newDlqWriteOffset (Add (Member deadLetterStateRead 'WriteOffset) deadLettersCount)) + (let dlqStartOffset (Add (Member deadLetterStateRead 'WriteOffset) (Uint64 '1))) + + (let dlqMostRecentTimestamp (Max (Member deadLetterStateRead 'LastModifiedTimestamp) now)) + + (let deadLetterStateUpdate '( + '('LastModifiedTimestamp dlqMostRecentTimestamp) + '('MessageCount newDlqMessagesCount) + '('WriteOffset newDlqWriteOffset))) + (let newSourceMessagesCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMove))) - (let sourceStateUpdate '( - '('LastModifiedTimestamp (Max (Member sourceStateRead 'LastModifiedTimestamp) now)) + (let sourceStateUpdate '( + '('LastModifiedTimestamp (Max (Member sourceStateRead 'LastModifiedTimestamp) now)) '('MessageCount newSourceMessagesCount) - '('InflyCount (Sub (Member sourceStateRead 'InflyCount) (Length messagesToMove))))) - - (return (Extend + '('InflyCount (Sub (Member sourceStateRead 'InflyCount) (Length messagesToMove))))) + + (return (Extend (AsList (SetResult 'result records)) - (AsList (SetResult 'movedMessagesCount (Length messagesToMove))) + (AsList (SetResult 'movedMessagesCount (Length messagesToMove))) (AsList (SetResult 'newMessagesCount newSourceMessagesCount)) - (AsList (UpdateRow deadLetterStateTable deadLetterStateRow deadLetterStateUpdate)) - (AsList (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate)) - - (Map messagesToUpdate (lambda '(item) (block '( - (let row '( - '('Offset (Member item 'Offset)))) - (let update '( - '('LoadId readId) - '('FirstReceiveTimestamp (Member item 'FirstReceiveTimestamp)) - '('LockTimestamp (Member item 'LockTimestamp)) - '('ReceiveCount (Member item 'ReceiveCount)) - '('VisibilityDeadline (Member item 'VisibilityDeadline)))) - (return (UpdateRow sourceInflyTable row update)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let msgRow '( - '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) - (let delayDeadline (Uint64 '0)) - (let messageUpdate '( - '('RandomId readId) - '('SentTimestamp dlqMostRecentTimestamp) - '('DelayDeadline delayDeadline))) - (return (UpdateRow deadLetterMessagesTable msgRow messageUpdate)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let sentTsRow '( - '('SentTimestamp dlqMostRecentTimestamp) - '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) - (let delayDeadline (Uint64 '0)) - (let sentTsUpdate '( - '('RandomId readId) - '('DelayDeadline delayDeadline))) - (return (UpdateRow deadLetterSentTsIdxTable sentTsRow sentTsUpdate)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let dataRow '( - '('RandomId readId) - '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) - - (let dataUpdate '( - '('Data (Member item 'Data)) - '('Attributes (Member item 'Attributes)) - '('SenderId (Member item 'SenderId)) - '('MessageId (Member item 'MessageId)))) - (return (UpdateRow deadLetterMessageDataTable dataRow dataUpdate)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let inflyRow '( - '('Offset (Member item 'Offset)))) - (return (EraseRow sourceInflyTable inflyRow)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let dataRow '( - '('RandomId (Member item 'RandomId)) - '('Offset (Member item 'Offset)))) - - (return (EraseRow sourceMessageDataTable dataRow)))))) - - (Map messagesToMove (lambda '(item) (block '( - (let sentTsRow '( - '('SentTimestamp (Member item 'SentTimestamp)) - '('Offset (Member item 'Offset)))) - (return (EraseRow sourceSentTsIdxTable sentTsRow)))))) - )) - ) -)__"; - + (AsList (UpdateRow deadLetterStateTable deadLetterStateRow deadLetterStateUpdate)) + (AsList (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate)) + + (Map messagesToUpdate (lambda '(item) (block '( + (let row '( + '('Offset (Member item 'Offset)))) + (let update '( + '('LoadId readId) + '('FirstReceiveTimestamp (Member item 'FirstReceiveTimestamp)) + '('LockTimestamp (Member item 'LockTimestamp)) + '('ReceiveCount (Member item 'ReceiveCount)) + '('VisibilityDeadline (Member item 'VisibilityDeadline)))) + (return (UpdateRow sourceInflyTable row update)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let msgRow '( + '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) + (let delayDeadline (Uint64 '0)) + (let messageUpdate '( + '('RandomId readId) + '('SentTimestamp dlqMostRecentTimestamp) + '('DelayDeadline delayDeadline))) + (return (UpdateRow deadLetterMessagesTable msgRow messageUpdate)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let sentTsRow '( + '('SentTimestamp dlqMostRecentTimestamp) + '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) + (let delayDeadline (Uint64 '0)) + (let sentTsUpdate '( + '('RandomId readId) + '('DelayDeadline delayDeadline))) + (return (UpdateRow deadLetterSentTsIdxTable sentTsRow sentTsUpdate)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let dataRow '( + '('RandomId readId) + '('Offset (Add dlqStartOffset (Member item 'DlqIndex))))) + + (let dataUpdate '( + '('Data (Member item 'Data)) + '('Attributes (Member item 'Attributes)) + '('SenderId (Member item 'SenderId)) + '('MessageId (Member item 'MessageId)))) + (return (UpdateRow deadLetterMessageDataTable dataRow dataUpdate)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let inflyRow '( + '('Offset (Member item 'Offset)))) + (return (EraseRow sourceInflyTable inflyRow)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let dataRow '( + '('RandomId (Member item 'RandomId)) + '('Offset (Member item 'Offset)))) + + (return (EraseRow sourceMessageDataTable dataRow)))))) + + (Map messagesToMove (lambda '(item) (block '( + (let sentTsRow '( + '('SentTimestamp (Member item 'SentTimestamp)) + '('Offset (Member item 'Offset)))) + (return (EraseRow sourceSentTsIdxTable sentTsRow)))))) + )) + ) +)__"; + const char* const WriteMessageQuery = R"__( ( (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64))) @@ -1031,7 +1031,7 @@ const char* const LoadInflyQuery = R"__( 'Offset 'RandomId 'DelayDeadline - 'ReceiveCount + 'ReceiveCount 'VisibilityDeadline)) (let infly (Member (SelectRange inflyTable range fields '()) 'List)) @@ -1069,53 +1069,53 @@ const char* const GetStateQuery = R"__( ) )__"; -const char* const ListDeadLetterSourceQueuesQuery = R"__( - ( +const char* const ListDeadLetterSourceQueuesQuery = R"__( + ( (let folderId (Parameter 'FOLDERID (DataType 'Utf8String))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - + (let queuesTable '%5$s/.Queues) (let queuesRow '( '('Account userName) - '('QueueName (Utf8String '"%4$s")))) - + '('QueueName (Utf8String '"%4$s")))) + (let queuesRowSelect '( - 'QueueName - 'CustomQueueName)) - + 'QueueName + 'CustomQueueName)) + (let queuesRowRead (SelectRow queuesTable queuesRow queuesRowSelect)) - - (let skipFolderIdFilter (Equal folderId (Utf8String '""))) - - (let dlqName + + (let skipFolderIdFilter (Equal folderId (Utf8String '""))) + + (let dlqName (If skipFolderIdFilter (Member queuesRowRead 'QueueName) (Coalesce (Member queuesRowRead 'CustomQueueName) (Utf8String '"")))) - + (let queuesRange '( '('Account userName userName) - '('QueueName (Utf8String '"") (Void)))) - (let queueSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName)) + '('QueueName (Utf8String '"") (Void)))) + (let queueSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName)) (let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List)) - - (let filtered (Filter queues (lambda '(item) (block '( - (return (Coalesce - (And - (And - (Or - (Equal (Member item 'QueueState) (Uint64 '1)) - (Equal (Member item 'QueueState) (Uint64 '3))) - (Or - (Equal (Member item 'FolderId) folderId) - skipFolderIdFilter)) - (Equal (Member item 'DlqName) dlqName)) - (Bool 'false))) - ))))) - - (return (AsList - (SetResult 'queues filtered))) - ) -)__"; - + + (let filtered (Filter queues (lambda '(item) (block '( + (return (Coalesce + (And + (And + (Or + (Equal (Member item 'QueueState) (Uint64 '1)) + (Equal (Member item 'QueueState) (Uint64 '3))) + (Or + (Equal (Member item 'FolderId) folderId) + skipFolderIdFilter)) + (Equal (Member item 'DlqName) dlqName)) + (Bool 'false))) + ))))) + + (return (AsList + (SetResult 'queues filtered))) + ) +)__"; + const char* const GetUserSettingsQuery = R"__( ( (let fromUser (Parameter 'FROM_USER (DataType 'Utf8String))) @@ -1142,29 +1142,29 @@ const char* const GetUserSettingsQuery = R"__( ) )__"; -const char* const GetMessageCountMetricsQuery = R"__( - ( - (let shard (Parameter 'SHARD (DataType 'Uint64))) - - (let stateTable '%1$s/State) - - (let stateRow '( - '('State shard))) - (let stateSelect '( - 'MessageCount - 'InflyCount - 'CreatedTimestamp)) - - (let stateRead - (SelectRow stateTable stateRow stateSelect)) - - (return (AsList - (SetResult 'messagesCount (Member stateRead 'MessageCount)) - (SetResult 'inflyMessagesCount (Member stateRead 'InflyCount)) - (SetResult 'createdTimestamp (Member stateRead 'CreatedTimestamp)))) - ) -)__"; - +const char* const GetMessageCountMetricsQuery = R"__( + ( + (let shard (Parameter 'SHARD (DataType 'Uint64))) + + (let stateTable '%1$s/State) + + (let stateRow '( + '('State shard))) + (let stateSelect '( + 'MessageCount + 'InflyCount + 'CreatedTimestamp)) + + (let stateRead + (SelectRow stateTable stateRow stateSelect)) + + (return (AsList + (SetResult 'messagesCount (Member stateRead 'MessageCount)) + (SetResult 'inflyMessagesCount (Member stateRead 'InflyCount)) + (SetResult 'createdTimestamp (Member stateRead 'CreatedTimestamp)))) + ) +)__"; + const char* const GetQueuesListQuery = R"__( ( (let fromUser (Parameter 'FROM_USER (DataType 'Utf8String))) @@ -1182,7 +1182,7 @@ const char* const GetQueuesListQuery = R"__( 'QueueState 'CreatedTimestamp 'CustomQueueName - 'DlqName + 'DlqName 'FolderId 'MasterTabletId 'Version @@ -1232,16 +1232,16 @@ const char* GetStdQueryById(size_t id) { return AddMessagesToInflyQuery; case GET_STATE_ID: // 19 return GetStateQuery; - case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 20 - return ListDeadLetterSourceQueuesQuery; - case LOAD_OR_REDRIVE_MESSAGE_ID: // 21 - return LoadOrRedriveMessageQuery; + case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 20 + return ListDeadLetterSourceQueuesQuery; + case LOAD_OR_REDRIVE_MESSAGE_ID: // 21 + return LoadOrRedriveMessageQuery; case GET_USER_SETTINGS_ID: // 23 return GetUserSettingsQuery; case GET_QUEUES_LIST_ID: // 24 return GetQueuesListQuery; - case GET_MESSAGE_COUNT_METRIC_ID: // 14 - return GetMessageCountMetricsQuery; + case GET_MESSAGE_COUNT_METRIC_ID: // 14 + return GetMessageCountMetricsQuery; } return nullptr; } diff --git a/ydb/core/ymq/queues/std/schema.cpp b/ydb/core/ymq/queues/std/schema.cpp index 4ba75d17a2..806a2f1d7f 100644 --- a/ydb/core/ymq/queues/std/schema.cpp +++ b/ydb/core/ymq/queues/std/schema.cpp @@ -11,9 +11,9 @@ TVector<TTable> GetStandardTables(ui64 shards, ui64 partitions, bool enableAutos TColumn("MaximumMessageSize", NScheme::NTypeIds::Uint64), TColumn("MessageRetentionPeriod", NScheme::NTypeIds::Uint64), TColumn("ReceiveMessageWaitTime", NScheme::NTypeIds::Uint64), - TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64), - TColumn("DlqName", NScheme::NTypeIds::Utf8), - TColumn("DlqArn", NScheme::NTypeIds::Utf8), + TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64), + TColumn("DlqName", NScheme::NTypeIds::Utf8), + TColumn("DlqArn", NScheme::NTypeIds::Utf8), TColumn("MaxReceiveCount", NScheme::NTypeIds::Uint64), TColumn("ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64)}; @@ -35,8 +35,8 @@ TVector<TTable> GetStandardTables(ui64 shards, ui64 partitions, bool enableAutos TColumn("Offset", NScheme::NTypeIds::Uint64, true), TColumn("Attributes", NScheme::NTypeIds::String), TColumn("Data", NScheme::NTypeIds::String), - TColumn("MessageId", NScheme::NTypeIds::String), - TColumn("SenderId", NScheme::NTypeIds::String)}; + TColumn("MessageId", NScheme::NTypeIds::String), + TColumn("SenderId", NScheme::NTypeIds::String)}; const TVector<TColumn> MessagesColumns = { TColumn("Offset", NScheme::NTypeIds::Uint64, true), diff --git a/ydb/core/ymq/ut/queue_id_ut.cpp b/ydb/core/ymq/ut/queue_id_ut.cpp index 6e7305fc82..0ac3776df6 100644 --- a/ydb/core/ymq/ut/queue_id_ut.cpp +++ b/ydb/core/ymq/ut/queue_id_ut.cpp @@ -1,28 +1,28 @@ #include <library/cpp/testing/unittest/registar.h> #include <ydb/core/ymq/base/queue_id.h> - -using namespace NKikimr::NSQS; - -Y_UNIT_TEST_SUITE(TGenerateQueueIdTests) { - static TString ValidateAndReturn(const TString& resourceId) { - UNIT_ASSERT_EQUAL(resourceId.size(), 20); - - for (const auto c : resourceId) { - UNIT_ASSERT(c <= 'v' && c >= '0'); - } - - return resourceId; - } - - Y_UNIT_TEST(MakeQueueIdBasic) { - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 0, "radix"), ValidateAndReturn("03r0000000000000024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 1, "radix"), ValidateAndReturn("03r0000000000001024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 1, "radix"), ValidateAndReturn("vvv0000000000001024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("000fvvvvvvvvvvvv024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("vvvfvvvvvvvvvvvv024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0, "Man, it is a very long account name, but does anyone really care?"), ValidateAndReturn("000000000000000006at")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 16, "radix"), ValidateAndReturn("03r000000000000g024u")); - UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0x097778AD59B52C00, "radix"), ValidateAndReturn("0000itrollcrab00024u")); - } - -} + +using namespace NKikimr::NSQS; + +Y_UNIT_TEST_SUITE(TGenerateQueueIdTests) { + static TString ValidateAndReturn(const TString& resourceId) { + UNIT_ASSERT_EQUAL(resourceId.size(), 20); + + for (const auto c : resourceId) { + UNIT_ASSERT(c <= 'v' && c >= '0'); + } + + return resourceId; + } + + Y_UNIT_TEST(MakeQueueIdBasic) { + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 0, "radix"), ValidateAndReturn("03r0000000000000024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 1, "radix"), ValidateAndReturn("03r0000000000001024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 1, "radix"), ValidateAndReturn("vvv0000000000001024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("000fvvvvvvvvvvvv024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("vvvfvvvvvvvvvvvv024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0, "Man, it is a very long account name, but does anyone really care?"), ValidateAndReturn("000000000000000006at")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 16, "radix"), ValidateAndReturn("03r000000000000g024u")); + UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0x097778AD59B52C00, "radix"), ValidateAndReturn("0000itrollcrab00024u")); + } + +} diff --git a/ydb/core/ymq/ut/ya.make b/ydb/core/ymq/ut/ya.make index 27db5cd822..112a1b60bb 100644 --- a/ydb/core/ymq/ut/ya.make +++ b/ydb/core/ymq/ut/ya.make @@ -6,7 +6,7 @@ OWNER( UNITTEST() SRCS( - queue_id_ut.cpp + queue_id_ut.cpp params_ut.cpp ) diff --git a/ydb/library/aclib/aclib.cpp b/ydb/library/aclib/aclib.cpp index 946497ea30..4d8d73370f 100644 --- a/ydb/library/aclib/aclib.cpp +++ b/ydb/library/aclib/aclib.cpp @@ -1,7 +1,7 @@ #include "aclib.h" #include <util/stream/str.h> #include <util/string/vector.h> -#include <algorithm> +#include <algorithm> #include <util/string/split.h> #include <library/cpp/protobuf/util/is_equal.h> @@ -325,7 +325,7 @@ std::pair<ui32, ui32> TACL::ClearAccess() { std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) { std::pair<ui32, ui32> modified = {}; - auto* ACL = MutableACE(); + auto* ACL = MutableACE(); auto matchACE = [&filter](const NACLibProto::TACE& ace) { if (filter.HasAccessType() && ace.GetAccessType() != filter.GetAccessType()) { return false; @@ -340,10 +340,10 @@ std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) { return false; } return true; - }; - + }; + auto newEnd = std::remove_if(ACL->begin(), ACL->end(), std::move(matchACE)); - while (newEnd != ACL->end()) { + while (newEnd != ACL->end()) { switch (static_cast<NACLib::EAccessType>(ACL->rbegin()->GetAccessType())) { case NACLib::EAccessType::Allow: modified |= std::pair<ui32, ui32>(ACL->rbegin()->GetAccessRight(), NACLib::EAccessRights::NoAccess); @@ -352,7 +352,7 @@ std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) { modified |= std::pair<ui32, ui32>(NACLib::EAccessRights::NoAccess, ACL->rbegin()->GetAccessRight()); break; } - ACL->RemoveLast(); + ACL->RemoveLast(); } return modified; } diff --git a/ydb/library/aclib/aclib_ut.cpp b/ydb/library/aclib/aclib_ut.cpp index c5828f8624..b6a00d11d1 100644 --- a/ydb/library/aclib/aclib_ut.cpp +++ b/ydb/library/aclib/aclib_ut.cpp @@ -147,55 +147,55 @@ Y_UNIT_TEST_SUITE(ACLib) { UNIT_ASSERT(effectiveAnimalFilesACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true); UNIT_ASSERT(effectiveAnimalFilesACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == true); } - - Y_UNIT_TEST(TestDiffACL) { - TUserToken jamesToken(James, TVector<TSID>()); - TUserToken catToken(Cat, TVector<TSID>()); + + Y_UNIT_TEST(TestDiffACL) { + TUserToken jamesToken(James, TVector<TSID>()); + TUserToken catToken(Cat, TVector<TSID>()); TUserToken dogToken(Dog, TVector<TSID>()); - - TSecurityObject objACL(James, true /* container */); - TDiffACL addAccessDiff; - addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); - addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer); - addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer); - addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer); + + TSecurityObject objACL(James, true /* container */); + TDiffACL addAccessDiff; + addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); + addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer); + addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer); + addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer); addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Dog, EInheritanceType::InheritContainer); addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Dog, EInheritanceType::InheritContainer); addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Dog, EInheritanceType::InheritContainer); addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Dog, EInheritanceType::InheritContainer); - - objACL.ApplyDiff(addAccessDiff); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); - - TDiffACL removeAccessDiff; - removeAccessDiff.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); - objACL.ApplyDiff(removeAccessDiff); - - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false); - - TDiffACL removeAccessDiff2; - removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); - removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer); - removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer); - removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer); - objACL.ApplyDiff(removeAccessDiff2); - - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); - UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); + + objACL.ApplyDiff(addAccessDiff); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); + + TDiffACL removeAccessDiff; + removeAccessDiff.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); + objACL.ApplyDiff(removeAccessDiff); + + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false); + + TDiffACL removeAccessDiff2; + removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer); + removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer); + removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer); + removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer); + objACL.ApplyDiff(removeAccessDiff2); + + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false); + UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false); UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, dogToken) == true); UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == true); @@ -210,7 +210,7 @@ Y_UNIT_TEST_SUITE(ACLib) { UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == false); UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, dogToken) == false); UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, dogToken) == false); - } + } Y_UNIT_TEST(TestInhertACL) { TSecurityObject objParent(James, true /* container */); diff --git a/ydb/library/http_proxy/authorization/auth_helpers.cpp b/ydb/library/http_proxy/authorization/auth_helpers.cpp index 1d3cf30400..178c658a5c 100644 --- a/ydb/library/http_proxy/authorization/auth_helpers.cpp +++ b/ydb/library/http_proxy/authorization/auth_helpers.cpp @@ -1,4 +1,4 @@ -#include "auth_helpers.h" +#include "auth_helpers.h" #include <ydb/library/http_proxy/error/error.h> diff --git a/ydb/library/http_proxy/authorization/signature.cpp b/ydb/library/http_proxy/authorization/signature.cpp index 60368c0614..5abef495d6 100644 --- a/ydb/library/http_proxy/authorization/signature.cpp +++ b/ydb/library/http_proxy/authorization/signature.cpp @@ -1,262 +1,262 @@ -#include "auth_helpers.h" -#include "signature.h" - +#include "auth_helpers.h" +#include "signature.h" + #include <library/cpp/http/io/stream.h> #include <library/cpp/http/misc/parsed_request.h> - + #include <contrib/libs/openssl/include/openssl/evp.h> #include <contrib/libs/openssl/include/openssl/hmac.h> #include <contrib/libs/openssl/include/openssl/sha.h> - -#include <util/generic/algorithm.h> -#include <util/stream/str.h> -#include <util/string/builder.h> + +#include <util/generic/algorithm.h> +#include <util/stream/str.h> +#include <util/string/builder.h> #include <library/cpp/cgiparam/cgiparam.h> -#include <util/string/hex.h> -#include <util/string/join.h> -#include <util/string/strip.h> - +#include <util/string/hex.h> +#include <util/string/join.h> +#include <util/string/strip.h> + namespace NKikimr::NSQS { - -static TString HmacSHA256(TStringBuf key, TStringBuf data) { - unsigned char hash[SHA256_DIGEST_LENGTH]; - ui32 hl = SHA256_DIGEST_LENGTH; + +static TString HmacSHA256(TStringBuf key, TStringBuf data) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + ui32 hl = SHA256_DIGEST_LENGTH; const auto* res = HMAC(EVP_sha256(), key.data(), key.size(), reinterpret_cast<const unsigned char*>(data.data()), data.size(), hash, &hl); - Y_ENSURE(res); - Y_ENSURE(hl == SHA256_DIGEST_LENGTH); - return TString{reinterpret_cast<const char*>(res), hl}; -} - -static TString HashSHA256(IInputStream& stream) { - SHA256_CTX hasher; - SHA256_Init(&hasher); - char buf[4096]; - size_t read = 0; - while ((read = stream.Read(buf, sizeof(buf))) != 0) { - SHA256_Update(&hasher, buf, read); - } - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256_Final(hash, &hasher); - return to_lower(HexEncode(hash, SHA256_DIGEST_LENGTH)); -} - -static TString HashSHA256(TStringBuf data) { - TMemoryInput in{data}; - return HashSHA256(in); -} - -static const char newline = '\n'; -static const char pathDelim = '/'; - + Y_ENSURE(res); + Y_ENSURE(hl == SHA256_DIGEST_LENGTH); + return TString{reinterpret_cast<const char*>(res), hl}; +} + +static TString HashSHA256(IInputStream& stream) { + SHA256_CTX hasher; + SHA256_Init(&hasher); + char buf[4096]; + size_t read = 0; + while ((read = stream.Read(buf, sizeof(buf))) != 0) { + SHA256_Update(&hasher, buf, read); + } + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, &hasher); + return to_lower(HexEncode(hash, SHA256_DIGEST_LENGTH)); +} + +static TString HashSHA256(TStringBuf data) { + TMemoryInput in{data}; + return HashSHA256(in); +} + +static const char newline = '\n'; +static const char pathDelim = '/'; + constexpr TStringBuf AUTHORIZATION_HEADER = "authorization"; - -static const TString SIGNED_HEADERS_PARAM = "signedheaders"; -static const TString SIGNATURE_PARAM = "signature"; -static const TString CREDENTIAL_PARAM = "credential"; - -TAwsRequestSignV4::TAwsRequestSignV4(const TString& request) { - // special standalone ctor for tests - TStringInput si(request); - THttpInput input(&si); - TParsedHttpFull parsed(input.FirstLine()); - - TMaybe<TBuffer> inputData; - ui64 contentLength = 0; - if (parsed.Method == "POST") { - if (input.GetContentLength(contentLength)) { - inputData.ConstructInPlace(); - inputData->Resize(contentLength); - if (input.Load(inputData->Data(), (size_t)contentLength) != contentLength) { - Y_VERIFY(false); - } - } - } - - Process(input, parsed, inputData); -} - -TAwsRequestSignV4::TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { - Process(input, parsed, inputData); -} - -const TString& TAwsRequestSignV4::GetCanonicalRequest() const { - return CanonicalRequestStr_; -} - -const TString& TAwsRequestSignV4::GetStringToSign() const { - return FinalStringToSignStr_; -} - -const TString& TAwsRequestSignV4::GetParsedSignature() const { - return ParsedSignature_; -} - -const TString& TAwsRequestSignV4::GetSigningTimestamp() const { - return AwsTimestamp_; -} - -const TString& TAwsRequestSignV4::GetRegion() const { - return AwsRegion_; -} - -const TString& TAwsRequestSignV4::GetAccessKeyId() const { - return AccessKeyId_; -} - -void TAwsRequestSignV4::Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { - ParseAuthorization(input); - MakeCanonicalRequest(input, parsed, inputData); - MakeFinalStringToSign(); -} - -TString TAwsRequestSignV4::CalcSignature(const TString& secretKey) const { - const auto dateKey = HmacSHA256(TString::Join("AWS4", secretKey), AwsDate_); - const auto dateRegionKey = HmacSHA256(dateKey, AwsRegion_); - const auto dateRegionServiceKey = HmacSHA256(dateRegionKey, AwsService_); - const auto signingKey = HmacSHA256(dateRegionServiceKey, AwsRequest_); - const auto signatureHmac = HmacSHA256(signingKey, FinalStringToSignStr_); - + +static const TString SIGNED_HEADERS_PARAM = "signedheaders"; +static const TString SIGNATURE_PARAM = "signature"; +static const TString CREDENTIAL_PARAM = "credential"; + +TAwsRequestSignV4::TAwsRequestSignV4(const TString& request) { + // special standalone ctor for tests + TStringInput si(request); + THttpInput input(&si); + TParsedHttpFull parsed(input.FirstLine()); + + TMaybe<TBuffer> inputData; + ui64 contentLength = 0; + if (parsed.Method == "POST") { + if (input.GetContentLength(contentLength)) { + inputData.ConstructInPlace(); + inputData->Resize(contentLength); + if (input.Load(inputData->Data(), (size_t)contentLength) != contentLength) { + Y_VERIFY(false); + } + } + } + + Process(input, parsed, inputData); +} + +TAwsRequestSignV4::TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { + Process(input, parsed, inputData); +} + +const TString& TAwsRequestSignV4::GetCanonicalRequest() const { + return CanonicalRequestStr_; +} + +const TString& TAwsRequestSignV4::GetStringToSign() const { + return FinalStringToSignStr_; +} + +const TString& TAwsRequestSignV4::GetParsedSignature() const { + return ParsedSignature_; +} + +const TString& TAwsRequestSignV4::GetSigningTimestamp() const { + return AwsTimestamp_; +} + +const TString& TAwsRequestSignV4::GetRegion() const { + return AwsRegion_; +} + +const TString& TAwsRequestSignV4::GetAccessKeyId() const { + return AccessKeyId_; +} + +void TAwsRequestSignV4::Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { + ParseAuthorization(input); + MakeCanonicalRequest(input, parsed, inputData); + MakeFinalStringToSign(); +} + +TString TAwsRequestSignV4::CalcSignature(const TString& secretKey) const { + const auto dateKey = HmacSHA256(TString::Join("AWS4", secretKey), AwsDate_); + const auto dateRegionKey = HmacSHA256(dateKey, AwsRegion_); + const auto dateRegionServiceKey = HmacSHA256(dateRegionKey, AwsService_); + const auto signingKey = HmacSHA256(dateRegionServiceKey, AwsRequest_); + const auto signatureHmac = HmacSHA256(signingKey, FinalStringToSignStr_); + return to_lower(HexEncode(signatureHmac.data(), signatureHmac.size())); -} - -void TAwsRequestSignV4::ParseAuthorization(const THttpInput& input) { - for (const auto& header : input.Headers()) { - if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) { - auto params = ParseAuthorizationParams(header.Value()); - - SignedHeadersList_ = TString(params[SIGNED_HEADERS_PARAM]); - ParsedSignature_ = TString(params[SIGNATURE_PARAM]); - - TStringBuf credential = params[CREDENTIAL_PARAM]; - AccessKeyId_ = TString(credential.NextTok(pathDelim)); - AwsDate_ = TString(credential.NextTok(pathDelim)); - AwsRegion_ = TString(credential.NextTok(pathDelim)); - AwsService_ = TString(credential.NextTok(pathDelim)); - break; - } - } -} - -static TString UriEncode(const TStringBuf input, bool encodeSlash = false) { - TStringStream result; - for (const char ch : input) { - if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || - ch == '-' || ch == '~' || ch == '.') { - result << ch; - } else if (ch == '/') { - if (encodeSlash) { - result << "%2F"; - } else { - result << ch; - } - } else { - result << "%" << HexEncode(&ch, 1); - } - } - return result.Str(); -} - -void TAwsRequestSignV4::MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { - TStringStream canonicalRequest; - // METHOD - canonicalRequest << parsed.Method << newline; - // PATH - canonicalRequest << UriEncode(parsed.Path) << newline; - // CGI - TCgiParameters cgi(parsed.Cgi); - TMap<TString, TVector<TString>> sortedCgi; - - for (const auto& [key, value] : cgi) { - sortedCgi[key].push_back(value); - } - - for (auto& pair : sortedCgi) { - ::Sort(pair.second.begin(), pair.second.end()); - } - - if (sortedCgi.size()) { - TStringStream canonicalCgi; - - auto printSingleParam = [&canonicalCgi](const TString& key, const TVector<TString>& values) { - auto it = values.begin(); - canonicalCgi << UriEncode(key) << "=" << UriEncode(*it); - while (++it != values.end()) { - canonicalCgi << "&" << UriEncode(key) << "=" << UriEncode(*it); - } - - }; - - auto it = sortedCgi.begin(); - printSingleParam(it->first, it->second); - while (++it != sortedCgi.end()) { - canonicalCgi << "&"; - printSingleParam(it->first, it->second); - } - - canonicalRequest << canonicalCgi.Str() << newline; - } else { - canonicalRequest << newline; - } - // CANONICAL HEADERS - TMap<TStringBuf, TVector<TString>> canonicalHeaders; - - TStringBuf headersList = SignedHeadersList_; - while (TStringBuf headerName = headersList.NextTok(';')) { - canonicalHeaders[headerName] = {}; - } - - for (const auto& header : input.Headers()) { - const auto lowercaseHeaderName = to_lower(header.Name()); - auto it = canonicalHeaders.find(lowercaseHeaderName); - if (it != canonicalHeaders.end()) { - it->second.push_back(Strip(header.Value())); - } - } - - for (const auto& [key, value] : canonicalHeaders) { +} + +void TAwsRequestSignV4::ParseAuthorization(const THttpInput& input) { + for (const auto& header : input.Headers()) { + if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) { + auto params = ParseAuthorizationParams(header.Value()); + + SignedHeadersList_ = TString(params[SIGNED_HEADERS_PARAM]); + ParsedSignature_ = TString(params[SIGNATURE_PARAM]); + + TStringBuf credential = params[CREDENTIAL_PARAM]; + AccessKeyId_ = TString(credential.NextTok(pathDelim)); + AwsDate_ = TString(credential.NextTok(pathDelim)); + AwsRegion_ = TString(credential.NextTok(pathDelim)); + AwsService_ = TString(credential.NextTok(pathDelim)); + break; + } + } +} + +static TString UriEncode(const TStringBuf input, bool encodeSlash = false) { + TStringStream result; + for (const char ch : input) { + if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || + ch == '-' || ch == '~' || ch == '.') { + result << ch; + } else if (ch == '/') { + if (encodeSlash) { + result << "%2F"; + } else { + result << ch; + } + } else { + result << "%" << HexEncode(&ch, 1); + } + } + return result.Str(); +} + +void TAwsRequestSignV4::MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) { + TStringStream canonicalRequest; + // METHOD + canonicalRequest << parsed.Method << newline; + // PATH + canonicalRequest << UriEncode(parsed.Path) << newline; + // CGI + TCgiParameters cgi(parsed.Cgi); + TMap<TString, TVector<TString>> sortedCgi; + + for (const auto& [key, value] : cgi) { + sortedCgi[key].push_back(value); + } + + for (auto& pair : sortedCgi) { + ::Sort(pair.second.begin(), pair.second.end()); + } + + if (sortedCgi.size()) { + TStringStream canonicalCgi; + + auto printSingleParam = [&canonicalCgi](const TString& key, const TVector<TString>& values) { + auto it = values.begin(); + canonicalCgi << UriEncode(key) << "=" << UriEncode(*it); + while (++it != values.end()) { + canonicalCgi << "&" << UriEncode(key) << "=" << UriEncode(*it); + } + + }; + + auto it = sortedCgi.begin(); + printSingleParam(it->first, it->second); + while (++it != sortedCgi.end()) { + canonicalCgi << "&"; + printSingleParam(it->first, it->second); + } + + canonicalRequest << canonicalCgi.Str() << newline; + } else { + canonicalRequest << newline; + } + // CANONICAL HEADERS + TMap<TStringBuf, TVector<TString>> canonicalHeaders; + + TStringBuf headersList = SignedHeadersList_; + while (TStringBuf headerName = headersList.NextTok(';')) { + canonicalHeaders[headerName] = {}; + } + + for (const auto& header : input.Headers()) { + const auto lowercaseHeaderName = to_lower(header.Name()); + auto it = canonicalHeaders.find(lowercaseHeaderName); + if (it != canonicalHeaders.end()) { + it->second.push_back(Strip(header.Value())); + } + } + + for (const auto& [key, value] : canonicalHeaders) { canonicalRequest << key << ":"sv << JoinRange(",", value.begin(), value.end()) << newline; - } - - canonicalRequest << newline; // skip additional line after headers - // SIGNED HEADERS - canonicalRequest << SignedHeadersList_ << newline; - - static const TStringBuf hashedContentHeader = "x-amz-content-sha256"; - - // PAYLOAD - auto contentIt = canonicalHeaders.find(hashedContentHeader); - static const TStringBuf unsignedPayloadLiteral = "UNSIGNED-PAYLOAD"; - if (contentIt != canonicalHeaders.end() && !contentIt->second.empty() && contentIt->second[0] == unsignedPayloadLiteral) { - canonicalRequest << unsignedPayloadLiteral; - } else { - if (inputData) { - canonicalRequest << HashSHA256(TStringBuf(inputData->Data(), inputData->Size())); - } else { - static const TStringBuf emptyStringSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - canonicalRequest << emptyStringSHA256; - } - } - - auto amzDateIt = canonicalHeaders.find("x-amz-date"); - if (amzDateIt != canonicalHeaders.end() && !amzDateIt->second.empty()) { - AwsTimestamp_ = amzDateIt->second[0]; - } - - CanonicalRequestStr_ = canonicalRequest.Str(); -} - -void TAwsRequestSignV4::MakeFinalStringToSign() { - TStringStream finalStringToSign; + } + + canonicalRequest << newline; // skip additional line after headers + // SIGNED HEADERS + canonicalRequest << SignedHeadersList_ << newline; + + static const TStringBuf hashedContentHeader = "x-amz-content-sha256"; + + // PAYLOAD + auto contentIt = canonicalHeaders.find(hashedContentHeader); + static const TStringBuf unsignedPayloadLiteral = "UNSIGNED-PAYLOAD"; + if (contentIt != canonicalHeaders.end() && !contentIt->second.empty() && contentIt->second[0] == unsignedPayloadLiteral) { + canonicalRequest << unsignedPayloadLiteral; + } else { + if (inputData) { + canonicalRequest << HashSHA256(TStringBuf(inputData->Data(), inputData->Size())); + } else { + static const TStringBuf emptyStringSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + canonicalRequest << emptyStringSHA256; + } + } + + auto amzDateIt = canonicalHeaders.find("x-amz-date"); + if (amzDateIt != canonicalHeaders.end() && !amzDateIt->second.empty()) { + AwsTimestamp_ = amzDateIt->second[0]; + } + + CanonicalRequestStr_ = canonicalRequest.Str(); +} + +void TAwsRequestSignV4::MakeFinalStringToSign() { + TStringStream finalStringToSign; finalStringToSign << "AWS4-HMAC-SHA256"sv << newline; - finalStringToSign << AwsTimestamp_ << newline; - - finalStringToSign << AwsDate_ << pathDelim; - finalStringToSign << AwsRegion_ << pathDelim; - finalStringToSign << AwsService_ << pathDelim; - finalStringToSign << AwsRequest_ << newline; - - finalStringToSign << HashSHA256(CanonicalRequestStr_); - FinalStringToSignStr_ = finalStringToSign.Str(); -} - + finalStringToSign << AwsTimestamp_ << newline; + + finalStringToSign << AwsDate_ << pathDelim; + finalStringToSign << AwsRegion_ << pathDelim; + finalStringToSign << AwsService_ << pathDelim; + finalStringToSign << AwsRequest_ << newline; + + finalStringToSign << HashSHA256(CanonicalRequestStr_); + FinalStringToSignStr_ = finalStringToSign.Str(); +} + } // namespace NKikimr::NSQS diff --git a/ydb/library/http_proxy/authorization/signature.h b/ydb/library/http_proxy/authorization/signature.h index 57468deee6..4e65519d14 100644 --- a/ydb/library/http_proxy/authorization/signature.h +++ b/ydb/library/http_proxy/authorization/signature.h @@ -1,52 +1,52 @@ -#pragma once - -#include <util/generic/maybe.h> -#include <util/generic/buffer.h> -#include <util/generic/string.h> -#include <util/stream/mem.h> - -class THttpInput; -struct TParsedHttpFull; - +#pragma once + +#include <util/generic/maybe.h> +#include <util/generic/buffer.h> +#include <util/generic/string.h> +#include <util/stream/mem.h> + +class THttpInput; +struct TParsedHttpFull; + namespace NKikimr::NSQS { - -class TAwsRequestSignV4 { -public: - TAwsRequestSignV4(const TString& request); - TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); - - const TString& GetCanonicalRequest() const; - - const TString& GetStringToSign() const; - - const TString& GetParsedSignature() const; - - const TString& GetSigningTimestamp() const; - - const TString& GetRegion() const; - - const TString& GetAccessKeyId() const; - - TString CalcSignature(const TString& secretKey) const; - -private: - void Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); - void ParseAuthorization(const THttpInput& input); - void MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); - void MakeFinalStringToSign(); - -private: - TString SignedHeadersList_; - TString ParsedSignature_; - TString CanonicalRequestStr_; - TString FinalStringToSignStr_; - TString StringToSign_; - TString AwsTimestamp_; // e. g. 20130524T000000Z - TString AccessKeyId_; - TString AwsDate_; // YYYYMMDD - TString AwsRegion_; - TString AwsService_; - TString AwsRequest_ = "aws4_request"; -}; - + +class TAwsRequestSignV4 { +public: + TAwsRequestSignV4(const TString& request); + TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); + + const TString& GetCanonicalRequest() const; + + const TString& GetStringToSign() const; + + const TString& GetParsedSignature() const; + + const TString& GetSigningTimestamp() const; + + const TString& GetRegion() const; + + const TString& GetAccessKeyId() const; + + TString CalcSignature(const TString& secretKey) const; + +private: + void Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); + void ParseAuthorization(const THttpInput& input); + void MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData); + void MakeFinalStringToSign(); + +private: + TString SignedHeadersList_; + TString ParsedSignature_; + TString CanonicalRequestStr_; + TString FinalStringToSignStr_; + TString StringToSign_; + TString AwsTimestamp_; // e. g. 20130524T000000Z + TString AccessKeyId_; + TString AwsDate_; // YYYYMMDD + TString AwsRegion_; + TString AwsService_; + TString AwsRequest_ = "aws4_request"; +}; + } // namespace NKikimr::NSQS diff --git a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp index 062ac52146..e193ad3837 100644 --- a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp +++ b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp @@ -2,112 +2,112 @@ #include <library/cpp/testing/unittest/registar.h> #include <ydb/library/http_proxy/error/error.h> - -using namespace NKikimr::NSQS; - -static const TString superSecretAmazonKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; - -Y_UNIT_TEST_SUITE(TestAwsSignatureV4) { - Y_UNIT_TEST(TestGetVanilla) { - const TString request = \ -R"__(GET / HTTP/1.1 -Host:example.amazonaws.com -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31)__"; - - const TString canonicalizedRequest = \ -R"__(GET -/ - -host:example.amazonaws.com -x-amz-date:20150830T123600Z - -host;x-amz-date -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)__"; - - const TString stringToSign = \ -R"__(AWS4-HMAC-SHA256 -20150830T123600Z -20150830/us-east-1/service/aws4_request -bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63)__"; - - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL(canonicalizedRequest, signature.GetCanonicalRequest()); - UNIT_ASSERT_EQUAL(stringToSign, signature.GetStringToSign()); - UNIT_ASSERT_EQUAL("us-east-1", signature.GetRegion()); - UNIT_ASSERT_EQUAL("5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestPostVanilla) { - const TString request = \ -R"__(POST / HTTP/1.1 -Host:example.amazonaws.com -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestPostHeaderCase) { - const TString request = \ -R"__(POST / HTTP/1.1 -Host:example.amazonaws.com -My-Header1:VALUE1 -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestGetParamsOrder) { - const TString request = \ -R"__(GET /?Param2=value2&Param1=value1 HTTP/1.1 -Host:example.amazonaws.com -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestGetUtf8) { - const TString request = \ -R"__(GET /?ሴ=bar HTTP/1.1 -Host:example.amazonaws.com -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestQueryOrderValue) { - const TString request = \ -R"__(GET /?Param1=value2&Param1=value1 HTTP/1.1 -Host:example.amazonaws.com -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestGetHeaderKeyDuplicate) { - const TString request = \ -R"__(GET / HTTP/1.1 -Host:example.amazonaws.com -My-Header1:value4 -My-Header1:value1 -My-Header1:value3 -My-Header1:value2 -X-Amz-Date:20150830T123600Z -Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01)__"; - TAwsRequestSignV4 signature(request); - UNIT_ASSERT_EQUAL("08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01", signature.CalcSignature(superSecretAmazonKey)); - } - - Y_UNIT_TEST(TestEmptyReq) { - const TString request = \ -R"__(GET /?a=b HTTP/1.1)__"; - TAwsRequestSignV4 signature(request); - // this signature is totally incorrect, we only test the fact that the code is stable - UNIT_ASSERT_EQUAL("5b63431f16d95e041cd7507fd76cbc072a1de46c09452621b8f25f1193997c2a", signature.CalcSignature(superSecretAmazonKey)); - } -} + +using namespace NKikimr::NSQS; + +static const TString superSecretAmazonKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + +Y_UNIT_TEST_SUITE(TestAwsSignatureV4) { + Y_UNIT_TEST(TestGetVanilla) { + const TString request = \ +R"__(GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31)__"; + + const TString canonicalizedRequest = \ +R"__(GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)__"; + + const TString stringToSign = \ +R"__(AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63)__"; + + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL(canonicalizedRequest, signature.GetCanonicalRequest()); + UNIT_ASSERT_EQUAL(stringToSign, signature.GetStringToSign()); + UNIT_ASSERT_EQUAL("us-east-1", signature.GetRegion()); + UNIT_ASSERT_EQUAL("5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestPostVanilla) { + const TString request = \ +R"__(POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestPostHeaderCase) { + const TString request = \ +R"__(POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:VALUE1 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestGetParamsOrder) { + const TString request = \ +R"__(GET /?Param2=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestGetUtf8) { + const TString request = \ +R"__(GET /?ሴ=bar HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestQueryOrderValue) { + const TString request = \ +R"__(GET /?Param1=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestGetHeaderKeyDuplicate) { + const TString request = \ +R"__(GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value4 +My-Header1:value1 +My-Header1:value3 +My-Header1:value2 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01)__"; + TAwsRequestSignV4 signature(request); + UNIT_ASSERT_EQUAL("08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01", signature.CalcSignature(superSecretAmazonKey)); + } + + Y_UNIT_TEST(TestEmptyReq) { + const TString request = \ +R"__(GET /?a=b HTTP/1.1)__"; + TAwsRequestSignV4 signature(request); + // this signature is totally incorrect, we only test the fact that the code is stable + UNIT_ASSERT_EQUAL("5b63431f16d95e041cd7507fd76cbc072a1de46c09452621b8f25f1193997c2a", signature.CalcSignature(superSecretAmazonKey)); + } +} diff --git a/ydb/library/http_proxy/error/error.cpp b/ydb/library/http_proxy/error/error.cpp index 0c91aac859..93e3756c9b 100644 --- a/ydb/library/http_proxy/error/error.cpp +++ b/ydb/library/http_proxy/error/error.cpp @@ -191,12 +191,12 @@ extern const TErrorClass INVALID_ATTRIBUTE_NAME = { "The specified attribute doesn't exist." }; -extern const TErrorClass INVALID_ATTRIBUTE_VALUE = { - "InvalidAttributeValue", - 400, - "The specified attribute value is invalid." -}; - +extern const TErrorClass INVALID_ATTRIBUTE_VALUE = { + "InvalidAttributeValue", + 400, + "The specified attribute value is invalid." +}; + extern const TErrorClass LEADER_RESOLVING_ERROR = { "InternalFailure", 500, diff --git a/ydb/library/http_proxy/error/error.h b/ydb/library/http_proxy/error/error.h index 02b9dca3d9..9845778b0e 100644 --- a/ydb/library/http_proxy/error/error.h +++ b/ydb/library/http_proxy/error/error.h @@ -84,7 +84,7 @@ extern const TErrorClass RECEIPT_HANDLE_IS_INVALID; // GetQueueAttributes errors // https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html extern const TErrorClass INVALID_ATTRIBUTE_NAME; -extern const TErrorClass INVALID_ATTRIBUTE_VALUE; +extern const TErrorClass INVALID_ATTRIBUTE_VALUE; // Leader resolving errors extern const TErrorClass LEADER_RESOLVING_ERROR; diff --git a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto index 873ee38b3c..34449dde05 100644 --- a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto +++ b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -option cc_enable_arenas = true; +option cc_enable_arenas = true; package Ydb.PersQueue.V1; @@ -103,9 +103,9 @@ service PersQueueService { * Remove read rule command. */ rpc RemoveReadRule(RemoveReadRuleRequest) returns (RemoveReadRuleResponse); -} +} -service ClusterDiscoveryService { - // Get PQ clusters which are eligible for the specified Write or Read Sessions - rpc DiscoverClusters(Ydb.PersQueue.ClusterDiscovery.DiscoverClustersRequest) returns (Ydb.PersQueue.ClusterDiscovery.DiscoverClustersResponse); -} +service ClusterDiscoveryService { + // Get PQ clusters which are eligible for the specified Write or Read Sessions + rpc DiscoverClusters(Ydb.PersQueue.ClusterDiscovery.DiscoverClustersRequest) returns (Ydb.PersQueue.ClusterDiscovery.DiscoverClustersResponse); +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index 8a156403a3..265061f17b 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -23,7 +23,7 @@ SRCS( persqueue_error_codes_v1.proto ydb_auth.proto ydb_persqueue_v1.proto - ydb_persqueue_cluster_discovery.proto + ydb_persqueue_cluster_discovery.proto ydb_clickhouse_internal.proto ydb_cms.proto ydb_common.proto @@ -52,7 +52,7 @@ CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation) # .pb.h are only available in C++ variant of PROTO_LIBRARY IF (MODULE_TAG == "CPP_PROTO") GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h) - GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h) + GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h) GENERATE_ENUM_SERIALIZATION(draft/datastreams.pb.h) ENDIF() diff --git a/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto b/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto index f39c900943..f3176fb39d 100644 --- a/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto +++ b/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto @@ -1,86 +1,86 @@ -syntax = "proto3"; - -import "google/protobuf/empty.proto"; - +syntax = "proto3"; + +import "google/protobuf/empty.proto"; + import "ydb/public/api/protos/ydb_operation.proto"; - -package Ydb.PersQueue.ClusterDiscovery; - -option java_package = "com.yandex.ydb.persqueue.cluster_discovery"; -option cc_enable_arenas = true; - - -message WriteSessionParams { - // Path to the topic to write to. - string topic = 1; + +package Ydb.PersQueue.ClusterDiscovery; + +option java_package = "com.yandex.ydb.persqueue.cluster_discovery"; +option cc_enable_arenas = true; + + +message WriteSessionParams { + // Path to the topic to write to. + string topic = 1; // Message group identifier. - bytes source_id = 2; - // Partition group to write to. 0 by default. - uint32 partition_group = 3; - // Force the specified cluster via its name. Leave it empty by default. - string preferred_cluster_name = 4; -} - -message ClusterInfo { - // A host discovery endpoint to use at the next step. - string endpoint = 1; - // An official cluster name. - string name = 2; - // Is the cluster available right now? - bool available = 3; -}; - -message ReadSessionParams { - // Path to the topic to read from. - string topic = 1; - // Read mode is set according to the read rule. - oneof read_rule { - string mirror_to_cluster = 2; - google.protobuf.Empty all_original = 3; - } -} - -message WriteSessionClusters { - // Ordered clusters with statuses. - repeated ClusterInfo clusters = 1; - - enum SelectionReason { - SELECTION_REASON_UNSPECIFIED = 0; - CLIENT_PREFERENCE = 1; - CLIENT_LOCATION = 2; - CONSISTENT_DISTRIBUTION = 3; - } - - // The reason why a particular cluster was prioritized. - SelectionReason primary_cluster_selection_reason = 2; -} - -message ReadSessionClusters { - // Ordered clusters with statuses. - repeated ClusterInfo clusters = 1; -} - -message DiscoverClustersRequest { - Ydb.Operations.OperationParams operation_params = 1; - - // Clusters will be discovered separately for each element of the list. - repeated WriteSessionParams write_sessions = 2; - repeated ReadSessionParams read_sessions = 3; - - // Latest clusters status version known to the client application. Use 0 by default. - int64 minimal_version = 4; -} - -message DiscoverClustersResponse { - // Operation contains the result of the request. Check the ydb_operation.proto. - Ydb.Operations.Operation operation = 1; -} - -message DiscoverClustersResult { - // Discovered per-session clusters. - repeated WriteSessionClusters write_sessions_clusters = 1; - repeated ReadSessionClusters read_sessions_clusters = 2; - - // Latest clusters status version known to the cluster discovery service. - int64 version = 3; -} + bytes source_id = 2; + // Partition group to write to. 0 by default. + uint32 partition_group = 3; + // Force the specified cluster via its name. Leave it empty by default. + string preferred_cluster_name = 4; +} + +message ClusterInfo { + // A host discovery endpoint to use at the next step. + string endpoint = 1; + // An official cluster name. + string name = 2; + // Is the cluster available right now? + bool available = 3; +}; + +message ReadSessionParams { + // Path to the topic to read from. + string topic = 1; + // Read mode is set according to the read rule. + oneof read_rule { + string mirror_to_cluster = 2; + google.protobuf.Empty all_original = 3; + } +} + +message WriteSessionClusters { + // Ordered clusters with statuses. + repeated ClusterInfo clusters = 1; + + enum SelectionReason { + SELECTION_REASON_UNSPECIFIED = 0; + CLIENT_PREFERENCE = 1; + CLIENT_LOCATION = 2; + CONSISTENT_DISTRIBUTION = 3; + } + + // The reason why a particular cluster was prioritized. + SelectionReason primary_cluster_selection_reason = 2; +} + +message ReadSessionClusters { + // Ordered clusters with statuses. + repeated ClusterInfo clusters = 1; +} + +message DiscoverClustersRequest { + Ydb.Operations.OperationParams operation_params = 1; + + // Clusters will be discovered separately for each element of the list. + repeated WriteSessionParams write_sessions = 2; + repeated ReadSessionParams read_sessions = 3; + + // Latest clusters status version known to the client application. Use 0 by default. + int64 minimal_version = 4; +} + +message DiscoverClustersResponse { + // Operation contains the result of the request. Check the ydb_operation.proto. + Ydb.Operations.Operation operation = 1; +} + +message DiscoverClustersResult { + // Discovered per-session clusters. + repeated WriteSessionClusters write_sessions_clusters = 1; + repeated ReadSessionClusters read_sessions_clusters = 2; + + // Latest clusters status version known to the cluster discovery service. + int64 version = 3; +} diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp index aabd11d4ca..5167c2c76d 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp @@ -1,81 +1,81 @@ -#include "cluster_discovery_service.h" -#include "cluster_discovery_worker.h" - -#include "counters.h" - +#include "cluster_discovery_service.h" +#include "cluster_discovery_worker.h" + +#include "counters.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/mind/address_classification/net_classifier.h> #include <ydb/core/mon/mon.h> #include <ydb/core/persqueue/cluster_tracker.h> #include <ydb/core/util/address_classifier.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> - -#include <algorithm> - - -namespace NKikimr::NPQ::NClusterDiscovery { - -using namespace NCounters; - -inline auto& Ctx() { - return TActivationContext::AsActorContext(); -} - -class TClusterDiscoveryServiceActor: public TActorBootstrapped<TClusterDiscoveryServiceActor> { -public: - TClusterDiscoveryServiceActor(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) - : RawCounters(counters) - , Counters(BuildCounters()) - { - } - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_DISCOVERY; - } - - const auto& Cfg() const { - return AppData(Ctx())->PQClusterDiscoveryConfig; - } - - void Bootstrap() { - Become(&TThis::Initing); - - SubscribeToNetClassifier(); - SubscribeToClusterTracker(); - - InitCloudNetData(); - - RegisterMonitoringPage(); - - StartPeriodicalMonitoring(); - } - -private: - void InitCloudNetData() { - if (Cfg().HasCloudNetData()) { - CloudNetworksClassifier = NAddressClassifier::BuildLabeledAddressClassifierFromNetData(Cfg().GetCloudNetData()); - - Y_VERIFY(CloudNetworksClassifier); // the config is expected to be correct if specified - } - } - - void SubscribeToNetClassifier() { - Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); - } - - void SubscribeToClusterTracker() { - Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); - } - - static TString FormatTs(const TMaybe<TInstant>& time) { - if (time) { - return time->ToRfc822String(); - } - return "NULL"; - } - + +#include <algorithm> + + +namespace NKikimr::NPQ::NClusterDiscovery { + +using namespace NCounters; + +inline auto& Ctx() { + return TActivationContext::AsActorContext(); +} + +class TClusterDiscoveryServiceActor: public TActorBootstrapped<TClusterDiscoveryServiceActor> { +public: + TClusterDiscoveryServiceActor(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) + : RawCounters(counters) + , Counters(BuildCounters()) + { + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_DISCOVERY; + } + + const auto& Cfg() const { + return AppData(Ctx())->PQClusterDiscoveryConfig; + } + + void Bootstrap() { + Become(&TThis::Initing); + + SubscribeToNetClassifier(); + SubscribeToClusterTracker(); + + InitCloudNetData(); + + RegisterMonitoringPage(); + + StartPeriodicalMonitoring(); + } + +private: + void InitCloudNetData() { + if (Cfg().HasCloudNetData()) { + CloudNetworksClassifier = NAddressClassifier::BuildLabeledAddressClassifierFromNetData(Cfg().GetCloudNetData()); + + Y_VERIFY(CloudNetworksClassifier); // the config is expected to be correct if specified + } + } + + void SubscribeToNetClassifier() { + Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); + } + + void SubscribeToClusterTracker() { + Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); + } + + static TString FormatTs(const TMaybe<TInstant>& time) { + if (time) { + return time->ToRfc822String(); + } + return "NULL"; + } + bool IsHealthy(bool useLocalEnabled = false) const { bool isLocalEnabled = false; if (ClustersList) { @@ -86,216 +86,216 @@ private: } return ClustersList && DatacenterClassifier && (isLocalEnabled || !useLocalEnabled); - } - - TString MakeReport() const { - TStringBuilder message; - message << "PersQueue Cluster Discovery Service info: \n"; - message << "\tOverall status: " << (IsHealthy() ? "GOOD" : "BAD") << "\n\n"; - - message << "\tInitial NetClassifier response ts: " << FormatTs(InitialNetClassifierResponseTs) << "\n"; - message << "\tInitial ClusterTracker response ts: " << FormatTs(InitialClusterTrackerResponseTs) << "\n"; - message << "\tFully initialized at: " << FormatTs(FullyInitializedTs) << "\n\n"; - - message << "\tNetClassifier: " << (DatacenterClassifier ? "OK" : "NULL") << "\n"; - message << "\tNetClassifier NetData update ts: " << FormatTs(NetDataUpdateTs) << "\n\n"; - - message << "\tClusters list: " << (ClustersList ? "OK" : "NULL") << "\n"; - message << "\tClusters list update ts: " << FormatTs(ClustersListUpdateTs) << "\n"; - if (ClustersList && ClustersList->Clusters.size()) { - message << "\tClusters " << "(" << ClustersList->Clusters.size() << "): " << "\n"; - for (const auto& cluster : ClustersList->Clusters) { - message << "\t\t" << cluster.DebugString() << "\n"; - } - } - - message << "\n"; - message << "\tNetClassifier updates count: " << NetClassifierUpdatesCount << "\n"; - message << "\tClusters list updates count: " << ClustersListUpdatesCount << "\n"; - - return TString(message); - } - - void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) { - const TStringBuf path = ev->Get()->Request.GetPath(); - - TStringBuilder responseContent; - if (path.EndsWith("/health")) { + } + + TString MakeReport() const { + TStringBuilder message; + message << "PersQueue Cluster Discovery Service info: \n"; + message << "\tOverall status: " << (IsHealthy() ? "GOOD" : "BAD") << "\n\n"; + + message << "\tInitial NetClassifier response ts: " << FormatTs(InitialNetClassifierResponseTs) << "\n"; + message << "\tInitial ClusterTracker response ts: " << FormatTs(InitialClusterTrackerResponseTs) << "\n"; + message << "\tFully initialized at: " << FormatTs(FullyInitializedTs) << "\n\n"; + + message << "\tNetClassifier: " << (DatacenterClassifier ? "OK" : "NULL") << "\n"; + message << "\tNetClassifier NetData update ts: " << FormatTs(NetDataUpdateTs) << "\n\n"; + + message << "\tClusters list: " << (ClustersList ? "OK" : "NULL") << "\n"; + message << "\tClusters list update ts: " << FormatTs(ClustersListUpdateTs) << "\n"; + if (ClustersList && ClustersList->Clusters.size()) { + message << "\tClusters " << "(" << ClustersList->Clusters.size() << "): " << "\n"; + for (const auto& cluster : ClustersList->Clusters) { + message << "\t\t" << cluster.DebugString() << "\n"; + } + } + + message << "\n"; + message << "\tNetClassifier updates count: " << NetClassifierUpdatesCount << "\n"; + message << "\tClusters list updates count: " << ClustersListUpdatesCount << "\n"; + + return TString(message); + } + + void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) { + const TStringBuf path = ev->Get()->Request.GetPath(); + + TStringBuilder responseContent; + if (path.EndsWith("/health")) { static const char HTTPNOTAVAIL_H[] = "HTTP/1.1 418 I'm a teapot\r\nConnection: Close\r\n\r\nDiscovery service is disabled on the node\r\n"; responseContent << (IsHealthy() ? NMonitoring::HTTPOKTEXT : HTTPNOTAVAIL_H) << "Service statuses: 200 - OK, 418 - DISABLED"; } else if (path.EndsWith("/ping")) { static const char HTTPNOTAVAIL_P[] = "HTTP/1.1 418 I'm a teapot\r\nConnection: Close\r\n\r\nDiscovery service is disabled on the node and local cluster is disabled\r\n"; responseContent << (IsHealthy(true) ? NMonitoring::HTTPOKTEXT : HTTPNOTAVAIL_P) << "Service statuses: 200 - OK, 418 - DISABLED"; } else{ - responseContent << NMonitoring::HTTPOKTEXT << MakeReport(); - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(responseContent), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); - } - - void UpdateNetClassifier(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { - DatacenterClassifier = ev->Get()->Classifier; - NetDataUpdateTs = ev->Get()->NetDataUpdateTimestamp; - - Counters = BuildCounters(); - - ++NetClassifierUpdatesCount; - - UpdateHealthProbe(); - } - - void HandleClassifierUpdateWhileIniting(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { - UpdateNetClassifier(ev); - - InitialNetClassifierResponseTs = Ctx().Now(); - - CompleteInitialization(); - } - - void UpdateClustersList(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { - ClustersListUpdateTs = ev->Get()->ClustersListUpdateTimestamp; - - if (!ClustersList || !(*ClustersList == *ev->Get()->ClustersList)) { - ClustersList = ev->Get()->ClustersList; - Counters = BuildCounters(); - } - - ++ClustersListUpdatesCount; - - UpdateHealthProbe(); - } - - void HandleClustersUpdateWhileIniting(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { - UpdateClustersList(ev); - - InitialClusterTrackerResponseTs = Ctx().Now(); - - CompleteInitialization(); - } - - void CompleteInitialization() { - if (!InitialNetClassifierResponseTs || !InitialClusterTrackerResponseTs) { - return; - } - - FullyInitializedTs = Ctx().Now(); - Become(&TThis::Working); - } - - void RegisterMonitoringPage() const { - NActors::TMon* mon = AppData(Ctx())->Mon; - if (mon) { - NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); - mon->RegisterActorPage(page, "pqcd", "PersQueue Cluster Discovery", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID); - } - } - - STATEFN(Initing) { - switch (ev->GetTypeRewrite()) { - hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileIniting); - hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileIniting); - hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileIniting); - hFunc(NMon::TEvHttpInfo, HandleHttpRequest); - hFunc(TEvents::TEvWakeup, UpdateTimedCounters); - } - } - - void RespondServiceUnavailable(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { - Counters->DroppedRequestsCount->Inc(); - + responseContent << NMonitoring::HTTPOKTEXT << MakeReport(); + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(responseContent), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + } + + void UpdateNetClassifier(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { + DatacenterClassifier = ev->Get()->Classifier; + NetDataUpdateTs = ev->Get()->NetDataUpdateTimestamp; + + Counters = BuildCounters(); + + ++NetClassifierUpdatesCount; + + UpdateHealthProbe(); + } + + void HandleClassifierUpdateWhileIniting(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { + UpdateNetClassifier(ev); + + InitialNetClassifierResponseTs = Ctx().Now(); + + CompleteInitialization(); + } + + void UpdateClustersList(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { + ClustersListUpdateTs = ev->Get()->ClustersListUpdateTimestamp; + + if (!ClustersList || !(*ClustersList == *ev->Get()->ClustersList)) { + ClustersList = ev->Get()->ClustersList; + Counters = BuildCounters(); + } + + ++ClustersListUpdatesCount; + + UpdateHealthProbe(); + } + + void HandleClustersUpdateWhileIniting(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { + UpdateClustersList(ev); + + InitialClusterTrackerResponseTs = Ctx().Now(); + + CompleteInitialization(); + } + + void CompleteInitialization() { + if (!InitialNetClassifierResponseTs || !InitialClusterTrackerResponseTs) { + return; + } + + FullyInitializedTs = Ctx().Now(); + Become(&TThis::Working); + } + + void RegisterMonitoringPage() const { + NActors::TMon* mon = AppData(Ctx())->Mon; + if (mon) { + NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors"); + mon->RegisterActorPage(page, "pqcd", "PersQueue Cluster Discovery", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID); + } + } + + STATEFN(Initing) { + switch (ev->GetTypeRewrite()) { + hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileIniting); + hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileIniting); + hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileIniting); + hFunc(NMon::TEvHttpInfo, HandleHttpRequest); + hFunc(TEvents::TEvWakeup, UpdateTimedCounters); + } + } + + void RespondServiceUnavailable(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { + Counters->DroppedRequestsCount->Inc(); + ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); - } - - void HandleDiscoverPQClustersRequestWhileIniting(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { - Counters->TotalRequestsCount->Inc(); - - RespondServiceUnavailable(ev); - } - - void HandleClassifierUpdateWhileWorking(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { - UpdateNetClassifier(ev); - } - - void HandleClustersUpdateWhileWorking(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { - UpdateClustersList(ev); - } - - void HandleDiscoverPQClustersRequestWhileWorking(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { - Counters->TotalRequestsCount->Inc(); - - if (!IsHealthy()) { - RespondServiceUnavailable(ev); - return; - } - - IActor* actor = NWorker::CreateClusterDiscoveryWorker(ev, DatacenterClassifier, CloudNetworksClassifier, ClustersList, Counters); - Register(actor, TMailboxType::HTSwap, AppData(Ctx())->UserPoolId); - } - - STATEFN(Working) { - switch (ev->GetTypeRewrite()) { - hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileWorking); - hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileWorking); - hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileWorking); - hFunc(NMon::TEvHttpInfo, HandleHttpRequest); - hFunc(TEvents::TEvWakeup, UpdateTimedCounters); - } - } - - // Monitoring features - - void StartPeriodicalMonitoring() { - Y_VERIFY(Cfg().GetTimedCountersUpdateIntervalSeconds()); - - Send(Ctx().SelfID, new TEvents::TEvWakeup); - } - - void UpdateHealthProbe() { - (*Counters->HealthProbe) = IsHealthy(); - } - - void UpdateTimedCounters(TEvents::TEvWakeup::TPtr&) { - if (NetDataUpdateTs) { - (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTs->Seconds(); - } - - if (ClustersListUpdateTs) { - (*Counters->ClustersListUpdateLagSeconds) = Ctx().Now().Seconds() - ClustersListUpdateTs->Seconds(); - } - - UpdateHealthProbe(); - - Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvents::TEvWakeup); - } - - TClusterDiscoveryCounters::TPtr BuildCounters() { - return MakeIntrusive<TClusterDiscoveryCounters>(RawCounters, ClustersList, DatacenterClassifier); - } - -private: + } + + void HandleDiscoverPQClustersRequestWhileIniting(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { + Counters->TotalRequestsCount->Inc(); + + RespondServiceUnavailable(ev); + } + + void HandleClassifierUpdateWhileWorking(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { + UpdateNetClassifier(ev); + } + + void HandleClustersUpdateWhileWorking(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { + UpdateClustersList(ev); + } + + void HandleDiscoverPQClustersRequestWhileWorking(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) { + Counters->TotalRequestsCount->Inc(); + + if (!IsHealthy()) { + RespondServiceUnavailable(ev); + return; + } + + IActor* actor = NWorker::CreateClusterDiscoveryWorker(ev, DatacenterClassifier, CloudNetworksClassifier, ClustersList, Counters); + Register(actor, TMailboxType::HTSwap, AppData(Ctx())->UserPoolId); + } + + STATEFN(Working) { + switch (ev->GetTypeRewrite()) { + hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileWorking); + hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileWorking); + hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileWorking); + hFunc(NMon::TEvHttpInfo, HandleHttpRequest); + hFunc(TEvents::TEvWakeup, UpdateTimedCounters); + } + } + + // Monitoring features + + void StartPeriodicalMonitoring() { + Y_VERIFY(Cfg().GetTimedCountersUpdateIntervalSeconds()); + + Send(Ctx().SelfID, new TEvents::TEvWakeup); + } + + void UpdateHealthProbe() { + (*Counters->HealthProbe) = IsHealthy(); + } + + void UpdateTimedCounters(TEvents::TEvWakeup::TPtr&) { + if (NetDataUpdateTs) { + (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTs->Seconds(); + } + + if (ClustersListUpdateTs) { + (*Counters->ClustersListUpdateLagSeconds) = Ctx().Now().Seconds() - ClustersListUpdateTs->Seconds(); + } + + UpdateHealthProbe(); + + Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvents::TEvWakeup); + } + + TClusterDiscoveryCounters::TPtr BuildCounters() { + return MakeIntrusive<TClusterDiscoveryCounters>(RawCounters, ClustersList, DatacenterClassifier); + } + +private: TActorId NetClassifierActorId; TActorId ClusterTrackerActorId; - NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; - NAddressClassifier::TLabeledAddressClassifier::TConstPtr CloudNetworksClassifier; - TMaybe<TInstant> NetDataUpdateTs; - - NClusterTracker::TClustersList::TConstPtr ClustersList; - TMaybe<TInstant> ClustersListUpdateTs; - - TMaybe<TInstant> InitialNetClassifierResponseTs; - TMaybe<TInstant> InitialClusterTrackerResponseTs; - - TMaybe<TInstant> FullyInitializedTs; - - size_t NetClassifierUpdatesCount = 0; - size_t ClustersListUpdatesCount = 0; - - TIntrusivePtr<NMonitoring::TDynamicCounters> RawCounters; - TClusterDiscoveryCounters::TPtr Counters; -}; - -NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { - return new TClusterDiscoveryServiceActor(counters); -} - -} // namespace NKikimr::NPQ::NClusterDiscovery + NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; + NAddressClassifier::TLabeledAddressClassifier::TConstPtr CloudNetworksClassifier; + TMaybe<TInstant> NetDataUpdateTs; + + NClusterTracker::TClustersList::TConstPtr ClustersList; + TMaybe<TInstant> ClustersListUpdateTs; + + TMaybe<TInstant> InitialNetClassifierResponseTs; + TMaybe<TInstant> InitialClusterTrackerResponseTs; + + TMaybe<TInstant> FullyInitializedTs; + + size_t NetClassifierUpdatesCount = 0; + size_t ClustersListUpdatesCount = 0; + + TIntrusivePtr<NMonitoring::TDynamicCounters> RawCounters; + TClusterDiscoveryCounters::TPtr Counters; +}; + +NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { + return new TClusterDiscoveryServiceActor(counters); +} + +} // namespace NKikimr::NPQ::NClusterDiscovery diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h index fec5708c15..65f91409c4 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <ydb/core/base/events.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actorid.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/event_local.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> - -namespace NKikimr::NPQ::NClusterDiscovery { + +namespace NKikimr::NPQ::NClusterDiscovery { using NActors::TActorId; - + inline TActorId MakeClusterDiscoveryServiceID() { const char x[TActorId::MaxServiceIDLength] = "pq_discosvc"; return TActorId(0, TStringBuf(x, TActorId::MaxServiceIDLength)); - } - - NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters); - -} // namespace NKikimr::NPQ::NClusterDiscovery + } + + NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters); + +} // namespace NKikimr::NPQ::NClusterDiscovery diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp index 34721e262d..2d1612ea1e 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp @@ -1,684 +1,684 @@ #include <ydb/services/persqueue_cluster_discovery/counters.h> - + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> - + #include <ydb/core/mind/address_classification/net_classifier.h> #include <ydb/core/persqueue/cluster_tracker.h> #include <ydb/core/testlib/test_pq_client.h> - + #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> - + #include <library/cpp/actors/http/http_proxy.h> - + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <google/protobuf/text_format.h> - -#include <grpc++/client_context.h> -#include <grpc++/create_channel.h> - -#include <util/stream/file.h> -#include <util/system/tempfile.h> - -namespace NKikimr::NPQCDTests { - -using namespace Tests; -using namespace NKikimrClient; -using namespace Ydb::PersQueue::ClusterDiscovery; - -struct TFancyGRpcWrapper { - TFancyGRpcWrapper(const ui16 grpcPort) { - Channel_ = grpc::CreateChannel("localhost:" + ToString(grpcPort), grpc::InsecureChannelCredentials()); - Stub_ = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel_); - } - - auto PerformRpc(const DiscoverClustersRequest& request, DiscoverClustersResult& result) { - grpc::ClientContext context; - DiscoverClustersResponse response; - - const auto status = Stub_->DiscoverClusters(&context, request, &response); - - UNIT_ASSERT(status.ok()); - UNIT_ASSERT(response.operation().ready()); - - const auto opStatus = response.operation().status(); - - if (opStatus == Ydb::StatusIds::SUCCESS) { - response.operation().result().UnpackTo(&result); - } - - return opStatus; - } - - void Wait() const { - Sleep(TDuration::MilliSeconds(100)); - } - - void EnsureServiceUnavailability() { - DiscoverClustersRequest request; - DiscoverClustersResult result; - - UNIT_ASSERT_VALUES_EQUAL(PerformRpc(request, result), Ydb::StatusIds::UNAVAILABLE); - } - - void WaitForExactClustersDataVersion(const i64 requiredVersion) { - while (true) { - DiscoverClustersRequest request; - DiscoverClustersResult result; - - if (PerformRpc(request, result) == Ydb::StatusIds::SUCCESS && result.version() == requiredVersion) { - break; - } - - UNIT_ASSERT(result.version() < requiredVersion); - - Wait(); - } - } - -private: - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub_; -}; - -static void VerifyClusterInfo(const ClusterInfo& clusterInfo, const TString& name, const TString& endpoint, bool isAvailable) { - UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.name(), name); - UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.endpoint(), endpoint); - UNIT_ASSERT_VALUES_EQUAL(clusterInfo.available(), isAvailable); -} - -static void VerifyEqualClusterInfo(const ClusterInfo& c1, const ClusterInfo& c2) { - VerifyClusterInfo(c1, c2.name(), c2.endpoint(), c2.available()); -} - -static void CallPQCDAndAssert(TFancyGRpcWrapper& wrapper, - const DiscoverClustersRequest& request, - DiscoverClustersResult& result, - const i64 expectedVersion, - const size_t expectedWriteSessionsClusters, - const size_t expectedReadSessionsClusters) -{ - UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.version(), expectedVersion); - - UNIT_ASSERT_VALUES_EQUAL(result.write_sessions_clusters_size(), expectedWriteSessionsClusters); - UNIT_ASSERT_VALUES_EQUAL(result.read_sessions_clusters_size(), expectedReadSessionsClusters); -} - -static void CheckReadDiscoveryHandlers(TFancyGRpcWrapper& wrapper, - const ClusterInfo& c1Info, - const ClusterInfo& c2Info, - i64 expectedVersion) -{ - { - // EMPTY - DiscoverClustersRequest request; - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 0); - } - { - // READ ALL ORIGINAL - DiscoverClustersRequest request; - auto* readSessionParams = request.add_read_sessions(); - readSessionParams->mutable_all_original(); - - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1); - - const auto& readSessionClusters = result.read_sessions_clusters(0); - - UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 2); - - VerifyEqualClusterInfo(readSessionClusters.clusters(0), c1Info); - VerifyEqualClusterInfo(readSessionClusters.clusters(1), c2Info); - } - { - // READ MIRRORED - DiscoverClustersRequest request; - auto* readSessionParams = request.add_read_sessions(); - readSessionParams->set_mirror_to_cluster("dc2"); - readSessionParams->set_topic("megatopic"); - - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1); - - const auto& readSessionClusters = result.read_sessions_clusters(0); - - UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 1); - - VerifyEqualClusterInfo(readSessionClusters.clusters(0), c2Info); - } - { - // READ MIRRORED INEXISTING - DiscoverClustersRequest request; - auto* readSessionParams = request.add_read_sessions(); - readSessionParams->set_mirror_to_cluster("trololo"); - - DiscoverClustersResult result; - - UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST); - } -} - -static void CheckWriteDiscoveryHandlers(TFancyGRpcWrapper& wrapper, - const ClusterInfo& c1Info, - const ClusterInfo& c2Info, - i64 expectedVersion) -{ - { - // WRITE SIMPLE - DiscoverClustersRequest request; - { - // causes the clusters to be arranged in the following order: #1, #2 - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic12"); - writeSessionParams->set_source_id("topic1.log"); - writeSessionParams->set_partition_group(42); - } - { - // causes the clusters to be arranged in the following order: #2, #1 - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic1"); - writeSessionParams->set_source_id("topic1.log"); - writeSessionParams->set_partition_group(42); - } - { - // preferred cluster option causes the clusters to be arranged in the following order: #2, #1 - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic12"); - writeSessionParams->set_source_id("topic1.log"); - writeSessionParams->set_partition_group(42); - writeSessionParams->set_preferred_cluster_name("dc2"); - } - - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, expectedVersion, 3, 0); - - { - const auto& writeSessionClusters = result.write_sessions_clusters(0); - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); - - VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c1Info); - VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c2Info); - - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); - } - { - const auto& writeSessionClusters = result.write_sessions_clusters(1); - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); - - VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info); - VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info); - - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); - } - { - const auto& writeSessionClusters = result.write_sessions_clusters(2); - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); - - VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info); - VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info); - - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_PREFERENCE); - } - } - { - // WRITE INEXISTENT PREFERRED CLUSTER - DiscoverClustersRequest request; - { - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic"); - writeSessionParams->set_source_id("topic1.log"); - writeSessionParams->set_preferred_cluster_name("dc777"); - } - - DiscoverClustersResult result; - - UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST); - } -} - -class TPQCDServer { -public: - TPQCDServer(bool enablePQCD = true) - : PortManager_() - , BusPort_(PortManager_.GetPort(2134)) - , GrpcPort_(PortManager_.GetPort(2135)) - , Settings_(NPersQueueTests::PQSettings(BusPort_, 1)) - { - Settings_.PQConfig.SetClustersUpdateTimeoutSec(1); - - Settings_.PQClusterDiscoveryConfig.SetEnabled(enablePQCD); - Settings_.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1); - } - - ui16 BusPort() const { - return BusPort_; - } - - ui16 MonPort() const { + +#include <grpc++/client_context.h> +#include <grpc++/create_channel.h> + +#include <util/stream/file.h> +#include <util/system/tempfile.h> + +namespace NKikimr::NPQCDTests { + +using namespace Tests; +using namespace NKikimrClient; +using namespace Ydb::PersQueue::ClusterDiscovery; + +struct TFancyGRpcWrapper { + TFancyGRpcWrapper(const ui16 grpcPort) { + Channel_ = grpc::CreateChannel("localhost:" + ToString(grpcPort), grpc::InsecureChannelCredentials()); + Stub_ = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel_); + } + + auto PerformRpc(const DiscoverClustersRequest& request, DiscoverClustersResult& result) { + grpc::ClientContext context; + DiscoverClustersResponse response; + + const auto status = Stub_->DiscoverClusters(&context, request, &response); + + UNIT_ASSERT(status.ok()); + UNIT_ASSERT(response.operation().ready()); + + const auto opStatus = response.operation().status(); + + if (opStatus == Ydb::StatusIds::SUCCESS) { + response.operation().result().UnpackTo(&result); + } + + return opStatus; + } + + void Wait() const { + Sleep(TDuration::MilliSeconds(100)); + } + + void EnsureServiceUnavailability() { + DiscoverClustersRequest request; + DiscoverClustersResult result; + + UNIT_ASSERT_VALUES_EQUAL(PerformRpc(request, result), Ydb::StatusIds::UNAVAILABLE); + } + + void WaitForExactClustersDataVersion(const i64 requiredVersion) { + while (true) { + DiscoverClustersRequest request; + DiscoverClustersResult result; + + if (PerformRpc(request, result) == Ydb::StatusIds::SUCCESS && result.version() == requiredVersion) { + break; + } + + UNIT_ASSERT(result.version() < requiredVersion); + + Wait(); + } + } + +private: + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub_; +}; + +static void VerifyClusterInfo(const ClusterInfo& clusterInfo, const TString& name, const TString& endpoint, bool isAvailable) { + UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.name(), name); + UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.endpoint(), endpoint); + UNIT_ASSERT_VALUES_EQUAL(clusterInfo.available(), isAvailable); +} + +static void VerifyEqualClusterInfo(const ClusterInfo& c1, const ClusterInfo& c2) { + VerifyClusterInfo(c1, c2.name(), c2.endpoint(), c2.available()); +} + +static void CallPQCDAndAssert(TFancyGRpcWrapper& wrapper, + const DiscoverClustersRequest& request, + DiscoverClustersResult& result, + const i64 expectedVersion, + const size_t expectedWriteSessionsClusters, + const size_t expectedReadSessionsClusters) +{ + UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.version(), expectedVersion); + + UNIT_ASSERT_VALUES_EQUAL(result.write_sessions_clusters_size(), expectedWriteSessionsClusters); + UNIT_ASSERT_VALUES_EQUAL(result.read_sessions_clusters_size(), expectedReadSessionsClusters); +} + +static void CheckReadDiscoveryHandlers(TFancyGRpcWrapper& wrapper, + const ClusterInfo& c1Info, + const ClusterInfo& c2Info, + i64 expectedVersion) +{ + { + // EMPTY + DiscoverClustersRequest request; + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 0); + } + { + // READ ALL ORIGINAL + DiscoverClustersRequest request; + auto* readSessionParams = request.add_read_sessions(); + readSessionParams->mutable_all_original(); + + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1); + + const auto& readSessionClusters = result.read_sessions_clusters(0); + + UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 2); + + VerifyEqualClusterInfo(readSessionClusters.clusters(0), c1Info); + VerifyEqualClusterInfo(readSessionClusters.clusters(1), c2Info); + } + { + // READ MIRRORED + DiscoverClustersRequest request; + auto* readSessionParams = request.add_read_sessions(); + readSessionParams->set_mirror_to_cluster("dc2"); + readSessionParams->set_topic("megatopic"); + + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1); + + const auto& readSessionClusters = result.read_sessions_clusters(0); + + UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 1); + + VerifyEqualClusterInfo(readSessionClusters.clusters(0), c2Info); + } + { + // READ MIRRORED INEXISTING + DiscoverClustersRequest request; + auto* readSessionParams = request.add_read_sessions(); + readSessionParams->set_mirror_to_cluster("trololo"); + + DiscoverClustersResult result; + + UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST); + } +} + +static void CheckWriteDiscoveryHandlers(TFancyGRpcWrapper& wrapper, + const ClusterInfo& c1Info, + const ClusterInfo& c2Info, + i64 expectedVersion) +{ + { + // WRITE SIMPLE + DiscoverClustersRequest request; + { + // causes the clusters to be arranged in the following order: #1, #2 + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic12"); + writeSessionParams->set_source_id("topic1.log"); + writeSessionParams->set_partition_group(42); + } + { + // causes the clusters to be arranged in the following order: #2, #1 + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic1"); + writeSessionParams->set_source_id("topic1.log"); + writeSessionParams->set_partition_group(42); + } + { + // preferred cluster option causes the clusters to be arranged in the following order: #2, #1 + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic12"); + writeSessionParams->set_source_id("topic1.log"); + writeSessionParams->set_partition_group(42); + writeSessionParams->set_preferred_cluster_name("dc2"); + } + + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, expectedVersion, 3, 0); + + { + const auto& writeSessionClusters = result.write_sessions_clusters(0); + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); + + VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c1Info); + VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c2Info); + + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); + } + { + const auto& writeSessionClusters = result.write_sessions_clusters(1); + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); + + VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info); + VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info); + + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); + } + { + const auto& writeSessionClusters = result.write_sessions_clusters(2); + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); + + VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info); + VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info); + + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_PREFERENCE); + } + } + { + // WRITE INEXISTENT PREFERRED CLUSTER + DiscoverClustersRequest request; + { + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic"); + writeSessionParams->set_source_id("topic1.log"); + writeSessionParams->set_preferred_cluster_name("dc777"); + } + + DiscoverClustersResult result; + + UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST); + } +} + +class TPQCDServer { +public: + TPQCDServer(bool enablePQCD = true) + : PortManager_() + , BusPort_(PortManager_.GetPort(2134)) + , GrpcPort_(PortManager_.GetPort(2135)) + , Settings_(NPersQueueTests::PQSettings(BusPort_, 1)) + { + Settings_.PQConfig.SetClustersUpdateTimeoutSec(1); + + Settings_.PQClusterDiscoveryConfig.SetEnabled(enablePQCD); + Settings_.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1); + } + + ui16 BusPort() const { + return BusPort_; + } + + ui16 MonPort() const { return Server_->GetRuntime()->GetMonPort(); - } - - ui16 GrpcPort() const { - return GrpcPort_; - } - - TServerSettings& MutableSettings() { - return Settings_; - } - - void SetNetDataViaFile(const TString& netDataTsv) { - UNIT_ASSERT(!Server_); - - NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); - NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); - NetDataFile->FlushData(); - - Settings_.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); - } - - void Run() { - Server_ = MakeHolder<TServer>(Settings_); + } + + ui16 GrpcPort() const { + return GrpcPort_; + } + + TServerSettings& MutableSettings() { + return Settings_; + } + + void SetNetDataViaFile(const TString& netDataTsv) { + UNIT_ASSERT(!Server_); + + NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); + NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); + NetDataFile->FlushData(); + + Settings_.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + } + + void Run() { + Server_ = MakeHolder<TServer>(Settings_); Server_->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcPort_)); - } - - NPersQueueTests::TFlatMsgBusPQClient& PQClient() { - UNIT_ASSERT(Server_); - - if (!PQClient_) { - PQClient_ = MakeHolder<NPersQueueTests::TFlatMsgBusPQClient>(Settings_, GrpcPort_); - } - - return *PQClient_; - } - - auto& ActorSystem() { - UNIT_ASSERT(Server_); - UNIT_ASSERT(Server_->GetRuntime()); - - return *Server_->GetRuntime(); - } - - NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr HttpRequest(const TString& path) { - UNIT_ASSERT(Server_); - - if (!HttpProxyId_) { - IActor* proxy = NHttp::CreateHttpProxy(Sensors_); - HttpProxyId_ = ActorSystem().Register(proxy); - } - + } + + NPersQueueTests::TFlatMsgBusPQClient& PQClient() { + UNIT_ASSERT(Server_); + + if (!PQClient_) { + PQClient_ = MakeHolder<NPersQueueTests::TFlatMsgBusPQClient>(Settings_, GrpcPort_); + } + + return *PQClient_; + } + + auto& ActorSystem() { + UNIT_ASSERT(Server_); + UNIT_ASSERT(Server_->GetRuntime()); + + return *Server_->GetRuntime(); + } + + NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr HttpRequest(const TString& path) { + UNIT_ASSERT(Server_); + + if (!HttpProxyId_) { + IActor* proxy = NHttp::CreateHttpProxy(Sensors_); + HttpProxyId_ = ActorSystem().Register(proxy); + } + TActorId clientId = ActorSystem().AllocateEdgeActor(); NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestGet("http://[::1]:" + ToString(MonPort()) + path); - ActorSystem().Send(new NActors::IEventHandle(HttpProxyId_, clientId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)), 0, true); - - return ActorSystem().GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingResponse>(clientId); - } - - void Wait() { - Sleep(TDuration::MilliSeconds(100)); - } - - auto ServiceCounters() const { - UNIT_ASSERT(Server_); - UNIT_ASSERT(Server_->GetGRpcServerRootCounters()); - - return GetServiceCounters(Server_->GetGRpcServerRootCounters(), "persqueue")->GetSubgroup("subsystem", "cluster_discovery"); - } - - auto GetTimesResolvedByClassifier(const TString& datacenterLabel) const { - return ServiceCounters()->GetSubgroup("resolved_dc", datacenterLabel)->GetCounter("TimesResolvedByClassifier", true)->GetAtomic(); - } - - bool CheckServiceHealth() { - UNIT_ASSERT(Server_); - - while (true) { - auto responsePtr = HttpRequest("/actors/pqcd/health"); - const auto& response = *responsePtr->Get(); - - if (!response.Error && (response.Response->Status == "200") || (response.Response->Status == "418")) { - UNIT_ASSERT(response.Response->Body); - return response.Response->Status == "200"; - } - - Wait(); - } - - return false; - } - - void WaitUntilHealthy() { - while (true) { - if (CheckServiceHealth()) { - return; - } - - Wait(); - } - } - - void EnsureServiceUnavailability() { - // check the response for 3 seconds - for (size_t i = 0; i < 30; ++i) { - UNIT_ASSERT(!CheckServiceHealth()); - Wait(); - } - - TFancyGRpcWrapper(GrpcPort_).EnsureServiceUnavailability(); - } - -private: - TPortManager PortManager_; - ui16 BusPort_; - ui16 GrpcPort_; - TServerSettings Settings_; - + ActorSystem().Send(new NActors::IEventHandle(HttpProxyId_, clientId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)), 0, true); + + return ActorSystem().GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingResponse>(clientId); + } + + void Wait() { + Sleep(TDuration::MilliSeconds(100)); + } + + auto ServiceCounters() const { + UNIT_ASSERT(Server_); + UNIT_ASSERT(Server_->GetGRpcServerRootCounters()); + + return GetServiceCounters(Server_->GetGRpcServerRootCounters(), "persqueue")->GetSubgroup("subsystem", "cluster_discovery"); + } + + auto GetTimesResolvedByClassifier(const TString& datacenterLabel) const { + return ServiceCounters()->GetSubgroup("resolved_dc", datacenterLabel)->GetCounter("TimesResolvedByClassifier", true)->GetAtomic(); + } + + bool CheckServiceHealth() { + UNIT_ASSERT(Server_); + + while (true) { + auto responsePtr = HttpRequest("/actors/pqcd/health"); + const auto& response = *responsePtr->Get(); + + if (!response.Error && (response.Response->Status == "200") || (response.Response->Status == "418")) { + UNIT_ASSERT(response.Response->Body); + return response.Response->Status == "200"; + } + + Wait(); + } + + return false; + } + + void WaitUntilHealthy() { + while (true) { + if (CheckServiceHealth()) { + return; + } + + Wait(); + } + } + + void EnsureServiceUnavailability() { + // check the response for 3 seconds + for (size_t i = 0; i < 30; ++i) { + UNIT_ASSERT(!CheckServiceHealth()); + Wait(); + } + + TFancyGRpcWrapper(GrpcPort_).EnsureServiceUnavailability(); + } + +private: + TPortManager PortManager_; + ui16 BusPort_; + ui16 GrpcPort_; + TServerSettings Settings_; + NMonitoring::TMetricRegistry Sensors_; - THolder<TServer> Server_; - THolder<NPersQueueTests::TFlatMsgBusPQClient> PQClient_; - + THolder<TServer> Server_; + THolder<NPersQueueTests::TFlatMsgBusPQClient> PQClient_; + TActorId HttpProxyId_; - - THolder<TTempFileHandle> NetDataFile; -}; - -Y_UNIT_TEST_SUITE(TPQCDTest) { - Y_UNIT_TEST(TestRelatedServicesAreRunning) { - TPQCDServer server(false); - - const TString datacenterLabel = "fancy_datacenter"; - - server.SetNetDataViaFile("2a0d:d6c0::/29\t" + datacenterLabel); - - server.Run(); - - server.PQClient().InitRoot(); - server.PQClient().InitDCs(); - - auto& actorSystem = server.ActorSystem(); - // check that NetClassifier is up and running - { + + THolder<TTempFileHandle> NetDataFile; +}; + +Y_UNIT_TEST_SUITE(TPQCDTest) { + Y_UNIT_TEST(TestRelatedServicesAreRunning) { + TPQCDServer server(false); + + const TString datacenterLabel = "fancy_datacenter"; + + server.SetNetDataViaFile("2a0d:d6c0::/29\t" + datacenterLabel); + + server.Run(); + + server.PQClient().InitRoot(); + server.PQClient().InitDCs(); + + auto& actorSystem = server.ActorSystem(); + // check that NetClassifier is up and running + { const TActorId sender = actorSystem.AllocateEdgeActor(); - - actorSystem.Send( - new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender, - new NNetClassifier::TEvNetClassifier::TEvSubscribe() - )); - - TAutoPtr<IEventHandle> handle; - const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(handle); - - UNIT_ASSERT(event->NetDataUpdateTimestamp); - UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero()); - - UNIT_ASSERT(event->Classifier); - { - const auto classificationResult = event->Classifier->ClassifyAddress("2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb"); - - UNIT_ASSERT(classificationResult); - UNIT_ASSERT_STRINGS_EQUAL(*classificationResult, datacenterLabel); - } - - { - const auto classificationResult = event->Classifier->ClassifyAddress("2b0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb"); - UNIT_ASSERT(!classificationResult); - } - } - - // check that ClusterTracker is up and running - { + + actorSystem.Send( + new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender, + new NNetClassifier::TEvNetClassifier::TEvSubscribe() + )); + + TAutoPtr<IEventHandle> handle; + const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(handle); + + UNIT_ASSERT(event->NetDataUpdateTimestamp); + UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero()); + + UNIT_ASSERT(event->Classifier); + { + const auto classificationResult = event->Classifier->ClassifyAddress("2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb"); + + UNIT_ASSERT(classificationResult); + UNIT_ASSERT_STRINGS_EQUAL(*classificationResult, datacenterLabel); + } + + { + const auto classificationResult = event->Classifier->ClassifyAddress("2b0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb"); + UNIT_ASSERT(!classificationResult); + } + } + + // check that ClusterTracker is up and running + { const TActorId sender = actorSystem.AllocateEdgeActor(); - - actorSystem.Send( - new IEventHandle(NPQ::NClusterTracker::MakeClusterTrackerID(), sender, - new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe() - )); - - TAutoPtr<IEventHandle> handle; - const auto event = actorSystem.GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(handle); - - UNIT_ASSERT(event->ClustersList); - UNIT_ASSERT(event->ClustersList->Clusters.size()); - - UNIT_ASSERT(event->ClustersListUpdateTimestamp); - UNIT_ASSERT(*event->ClustersListUpdateTimestamp > TInstant::Zero()); - - UNIT_ASSERT_STRINGS_EQUAL(event->ClustersList->Clusters.front().Name, "dc1"); - } - } - - Y_UNIT_TEST(TestDiscoverClusters) { - TPQCDServer server; - - const TString datacenterLabel = "fancy_datacenter"; - - server.SetNetDataViaFile("::1/128\t" + datacenterLabel); - - server.Run(); - - server.PQClient().InitRoot(); - server.PQClient().InitDCs(); - - server.WaitUntilHealthy(); - - TFancyGRpcWrapper wrapper(server.GrpcPort()); - - constexpr i64 dcTestIterations = 4; // it's signed to avoid warnings - for (size_t i = 1; i <= dcTestIterations; ++i) { - wrapper.WaitForExactClustersDataVersion(i); - - ClusterInfo c1Info; - c1Info.set_name("dc1"); + + actorSystem.Send( + new IEventHandle(NPQ::NClusterTracker::MakeClusterTrackerID(), sender, + new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe() + )); + + TAutoPtr<IEventHandle> handle; + const auto event = actorSystem.GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(handle); + + UNIT_ASSERT(event->ClustersList); + UNIT_ASSERT(event->ClustersList->Clusters.size()); + + UNIT_ASSERT(event->ClustersListUpdateTimestamp); + UNIT_ASSERT(*event->ClustersListUpdateTimestamp > TInstant::Zero()); + + UNIT_ASSERT_STRINGS_EQUAL(event->ClustersList->Clusters.front().Name, "dc1"); + } + } + + Y_UNIT_TEST(TestDiscoverClusters) { + TPQCDServer server; + + const TString datacenterLabel = "fancy_datacenter"; + + server.SetNetDataViaFile("::1/128\t" + datacenterLabel); + + server.Run(); + + server.PQClient().InitRoot(); + server.PQClient().InitDCs(); + + server.WaitUntilHealthy(); + + TFancyGRpcWrapper wrapper(server.GrpcPort()); + + constexpr i64 dcTestIterations = 4; // it's signed to avoid warnings + for (size_t i = 1; i <= dcTestIterations; ++i) { + wrapper.WaitForExactClustersDataVersion(i); + + ClusterInfo c1Info; + c1Info.set_name("dc1"); c1Info.set_endpoint("localhost"); - c1Info.set_available(true); - - ClusterInfo c2Info; - c2Info.set_name("dc2"); - c2Info.set_endpoint("dc2.logbroker.yandex.net"); - c2Info.set_available(true); - - // atm it's impossible to disable reading - CheckReadDiscoveryHandlers(wrapper, c1Info, c2Info, i); - - c1Info.set_available(i % 2); - - // only write discovery is affected - CheckWriteDiscoveryHandlers(wrapper, c1Info, c2Info, i); - - server.PQClient().UpdateDC("dc1", true, !(i % 2)); - } - { - // test handle viewer - auto responsePtr = server.HttpRequest("/actors/pqcd"); - const auto& response = *responsePtr->Get(); - - UNIT_ASSERT(!response.Error); - UNIT_ASSERT_STRINGS_EQUAL(response.Response->Status, "200"); - UNIT_ASSERT(response.Response->Body.Contains("dc1")); - UNIT_ASSERT(response.Response->Body.Contains("dc2")); - UNIT_ASSERT(response.Response->Body.Contains("NetClassifier")); - UNIT_ASSERT(response.Response->Body.Contains("Clusters list")); - UNIT_ASSERT(response.Response->Body.Contains("GOOD")); - } - - UNIT_ASSERT_VALUES_EQUAL(server.ActorSystem().GetNodeCount(), 1); - - // counters values might be somehow higher since WaitForExactClustersDataVersion calls number is unpredictable - // read check performs 4 calls, write check performs 2 calls. Also for each iteration WaitForExactClustersDataVersion is called at least once - // therefore the final minimal expected calls number is (4 + 2 + 1) * dcTestIterations - NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); - UNIT_ASSERT_GE(counters.TotalRequestsCount->GetAtomic(), (4 + 2 + 1) * dcTestIterations); - UNIT_ASSERT_GE(counters.SuccessfulRequestsCount->GetAtomic(), (3 + 1 + 1) * dcTestIterations); - - UNIT_ASSERT_VALUES_EQUAL(counters.DroppedRequestsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters.HealthProbe->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters.FailedRequestsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters.BadRequestsCount->GetAtomic(), 2 * dcTestIterations); - - UNIT_ASSERT_VALUES_EQUAL(counters.WriteDiscoveriesCount->GetAtomic(), 4 * dcTestIterations); - UNIT_ASSERT_VALUES_EQUAL(counters.ReadDiscoveriesCount->GetAtomic(), 3 * dcTestIterations); - - auto getTimesPrioritized = [&server](const TString& clusterName) { - return server.ServiceCounters()->GetSubgroup("prioritized_cluster", clusterName)->GetCounter("TimesPrioritizedForWrite", true)->GetAtomic(); - }; - - UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc1"), 1 * dcTestIterations); - UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc2"), 2 * dcTestIterations); - - // all requests are expected to come from a certain dc since the locality of the test - UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier(datacenterLabel), counters.TotalRequestsCount->GetAtomic()); - - auto snapshot = counters.DiscoveryWorkingDurationMs->Snapshot(); - - size_t total = 0; - for (size_t i = 0; i < snapshot->Count(); ++i) { - total += snapshot->Value(i); - } - UNIT_ASSERT_VALUES_EQUAL(total, counters.TotalRequestsCount->GetAtomic() - counters.DroppedRequestsCount->GetAtomic()); - - UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 0); - - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_PREFERENCE]->GetAtomic(), 1 * dcTestIterations); - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 2 * dcTestIterations); - } - - Y_UNIT_TEST(TestPrioritizeLocalDatacenter) { - TPQCDServer server; - - const TString datacenterLabel = "dc2"; - - server.SetNetDataViaFile("::1/128\t" + datacenterLabel); - - server.Run(); - - server.PQClient().InitRoot(); - server.PQClient().InitDCs(); - - server.WaitUntilHealthy(); - - TFancyGRpcWrapper wrapper(server.GrpcPort()); - - DiscoverClustersRequest request; - { - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic"); - writeSessionParams->set_source_id("topic1.log"); - writeSessionParams->set_partition_group(42); - } - - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, 1, 1, 0); - - const auto& writeSessionClusters = result.write_sessions_clusters(0); - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); - - UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), datacenterLabel); - - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_LOCATION); - - NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); - - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 0); - } - - Y_UNIT_TEST(TestUnavailableWithoutNetClassifier) { - TPQCDServer server; - - server.Run(); - - server.PQClient().InitRoot(); - server.PQClient().InitDCs(); - - auto& actorSystem = server.ActorSystem(); - // check that NetClassifier instance is initialized with null - { + c1Info.set_available(true); + + ClusterInfo c2Info; + c2Info.set_name("dc2"); + c2Info.set_endpoint("dc2.logbroker.yandex.net"); + c2Info.set_available(true); + + // atm it's impossible to disable reading + CheckReadDiscoveryHandlers(wrapper, c1Info, c2Info, i); + + c1Info.set_available(i % 2); + + // only write discovery is affected + CheckWriteDiscoveryHandlers(wrapper, c1Info, c2Info, i); + + server.PQClient().UpdateDC("dc1", true, !(i % 2)); + } + { + // test handle viewer + auto responsePtr = server.HttpRequest("/actors/pqcd"); + const auto& response = *responsePtr->Get(); + + UNIT_ASSERT(!response.Error); + UNIT_ASSERT_STRINGS_EQUAL(response.Response->Status, "200"); + UNIT_ASSERT(response.Response->Body.Contains("dc1")); + UNIT_ASSERT(response.Response->Body.Contains("dc2")); + UNIT_ASSERT(response.Response->Body.Contains("NetClassifier")); + UNIT_ASSERT(response.Response->Body.Contains("Clusters list")); + UNIT_ASSERT(response.Response->Body.Contains("GOOD")); + } + + UNIT_ASSERT_VALUES_EQUAL(server.ActorSystem().GetNodeCount(), 1); + + // counters values might be somehow higher since WaitForExactClustersDataVersion calls number is unpredictable + // read check performs 4 calls, write check performs 2 calls. Also for each iteration WaitForExactClustersDataVersion is called at least once + // therefore the final minimal expected calls number is (4 + 2 + 1) * dcTestIterations + NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); + UNIT_ASSERT_GE(counters.TotalRequestsCount->GetAtomic(), (4 + 2 + 1) * dcTestIterations); + UNIT_ASSERT_GE(counters.SuccessfulRequestsCount->GetAtomic(), (3 + 1 + 1) * dcTestIterations); + + UNIT_ASSERT_VALUES_EQUAL(counters.DroppedRequestsCount->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters.HealthProbe->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters.FailedRequestsCount->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters.BadRequestsCount->GetAtomic(), 2 * dcTestIterations); + + UNIT_ASSERT_VALUES_EQUAL(counters.WriteDiscoveriesCount->GetAtomic(), 4 * dcTestIterations); + UNIT_ASSERT_VALUES_EQUAL(counters.ReadDiscoveriesCount->GetAtomic(), 3 * dcTestIterations); + + auto getTimesPrioritized = [&server](const TString& clusterName) { + return server.ServiceCounters()->GetSubgroup("prioritized_cluster", clusterName)->GetCounter("TimesPrioritizedForWrite", true)->GetAtomic(); + }; + + UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc1"), 1 * dcTestIterations); + UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc2"), 2 * dcTestIterations); + + // all requests are expected to come from a certain dc since the locality of the test + UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier(datacenterLabel), counters.TotalRequestsCount->GetAtomic()); + + auto snapshot = counters.DiscoveryWorkingDurationMs->Snapshot(); + + size_t total = 0; + for (size_t i = 0; i < snapshot->Count(); ++i) { + total += snapshot->Value(i); + } + UNIT_ASSERT_VALUES_EQUAL(total, counters.TotalRequestsCount->GetAtomic() - counters.DroppedRequestsCount->GetAtomic()); + + UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 0); + + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_PREFERENCE]->GetAtomic(), 1 * dcTestIterations); + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 2 * dcTestIterations); + } + + Y_UNIT_TEST(TestPrioritizeLocalDatacenter) { + TPQCDServer server; + + const TString datacenterLabel = "dc2"; + + server.SetNetDataViaFile("::1/128\t" + datacenterLabel); + + server.Run(); + + server.PQClient().InitRoot(); + server.PQClient().InitDCs(); + + server.WaitUntilHealthy(); + + TFancyGRpcWrapper wrapper(server.GrpcPort()); + + DiscoverClustersRequest request; + { + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic"); + writeSessionParams->set_source_id("topic1.log"); + writeSessionParams->set_partition_group(42); + } + + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, 1, 1, 0); + + const auto& writeSessionClusters = result.write_sessions_clusters(0); + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); + + UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), datacenterLabel); + + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_LOCATION); + + NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); + + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 0); + } + + Y_UNIT_TEST(TestUnavailableWithoutNetClassifier) { + TPQCDServer server; + + server.Run(); + + server.PQClient().InitRoot(); + server.PQClient().InitDCs(); + + auto& actorSystem = server.ActorSystem(); + // check that NetClassifier instance is initialized with null + { const TActorId sender = actorSystem.AllocateEdgeActor(); - - actorSystem.Send( - new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender, - new NNetClassifier::TEvNetClassifier::TEvSubscribe() - )); - - TAutoPtr<IEventHandle> handle; - const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(sender); - UNIT_ASSERT(!event->Get()->Classifier); - } - - server.EnsureServiceUnavailability(); - } - - Y_UNIT_TEST(TestUnavailableWithoutClustersList) { - TPQCDServer server; - - server.SetNetDataViaFile("::1/128\tdc1"); - - server.Run(); - - server.EnsureServiceUnavailability(); - } - - Y_UNIT_TEST(TestUnavailableWithoutBoth) { - TPQCDServer server; - - server.Run(); - - server.EnsureServiceUnavailability(); - } - - Y_UNIT_TEST(TestCloudClientsAreConsistentlyDistributed) { - TPQCDServer server; - - auto& cloudNets = *server.MutableSettings().PQClusterDiscoveryConfig.MutableCloudNetData(); - auto& subnet = *cloudNets.AddSubnets(); - subnet.SetMask("::1/128"); - subnet.SetLabel("cloudnets"); - - server.SetNetDataViaFile("::1/128\tdc2"); - - server.Run(); - - server.PQClient().InitRoot(); - server.PQClient().InitDCs(); - - server.WaitUntilHealthy(); - - TFancyGRpcWrapper wrapper(server.GrpcPort()); - - DiscoverClustersRequest request; - { - auto* writeSessionParams = request.add_write_sessions(); - writeSessionParams->set_topic("topic12"); - writeSessionParams->set_source_id("topic1.log"); - } - - DiscoverClustersResult result; - - CallPQCDAndAssert(wrapper, request, result, 1, 1, 0); - - const auto& writeSessionClusters = result.write_sessions_clusters(0); - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); - - UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), "dc1"); - - UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); - - NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); - - UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 1); - UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier("dc2"), 1); - - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 1); - } -} - -} // namespace NKikimr::NPQCDTests + + actorSystem.Send( + new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender, + new NNetClassifier::TEvNetClassifier::TEvSubscribe() + )); + + TAutoPtr<IEventHandle> handle; + const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(sender); + UNIT_ASSERT(!event->Get()->Classifier); + } + + server.EnsureServiceUnavailability(); + } + + Y_UNIT_TEST(TestUnavailableWithoutClustersList) { + TPQCDServer server; + + server.SetNetDataViaFile("::1/128\tdc1"); + + server.Run(); + + server.EnsureServiceUnavailability(); + } + + Y_UNIT_TEST(TestUnavailableWithoutBoth) { + TPQCDServer server; + + server.Run(); + + server.EnsureServiceUnavailability(); + } + + Y_UNIT_TEST(TestCloudClientsAreConsistentlyDistributed) { + TPQCDServer server; + + auto& cloudNets = *server.MutableSettings().PQClusterDiscoveryConfig.MutableCloudNetData(); + auto& subnet = *cloudNets.AddSubnets(); + subnet.SetMask("::1/128"); + subnet.SetLabel("cloudnets"); + + server.SetNetDataViaFile("::1/128\tdc2"); + + server.Run(); + + server.PQClient().InitRoot(); + server.PQClient().InitDCs(); + + server.WaitUntilHealthy(); + + TFancyGRpcWrapper wrapper(server.GrpcPort()); + + DiscoverClustersRequest request; + { + auto* writeSessionParams = request.add_write_sessions(); + writeSessionParams->set_topic("topic12"); + writeSessionParams->set_source_id("topic1.log"); + } + + DiscoverClustersResult result; + + CallPQCDAndAssert(wrapper, request, result, 1, 1, 0); + + const auto& writeSessionClusters = result.write_sessions_clusters(0); + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2); + + UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), "dc1"); + + UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION); + + NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr); + + UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier("dc2"), 1); + + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0); + UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 1); + } +} + +} // namespace NKikimr::NPQCDTests diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp index 0c061b9bea..5882832912 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp @@ -1,288 +1,288 @@ -#include "cluster_discovery_worker.h" - +#include "cluster_discovery_worker.h" + #include <ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - -#include <util/digest/numeric.h> -#include <util/generic/hash.h> -#include <util/string/ascii.h> - -#include <algorithm> - - -namespace NKikimr::NPQ::NClusterDiscovery::NWorker { - -using namespace Ydb::PersQueue::ClusterDiscovery; -using NKikimr::NPQ::NClusterDiscovery::NClusterOrdering::OrderByHashAndWeight; - -inline auto& Ctx() { - return TActivationContext::AsActorContext(); -} - -class TClusterDiscoveryWorker : public TActorBootstrapped<TClusterDiscoveryWorker> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::GRPC_REQ; - } - - TClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, - TLabeledAddressClassifier::TConstPtr datacenterClassifier, - TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, - TClustersList::TConstPtr clustersList, - TClusterDiscoveryCounters::TPtr counters) - : Request(ev->Release().Release()) - , DatacenterClassifier(std::move(datacenterClassifier)) - , CloudNetsClassifier(std::move(cloudNetsClassifier)) - , ClustersList(std::move(clustersList)) - , Counters(counters) - {} - - void Bootstrap() { - RequestStartTs = Ctx().Now(); - Process(); - } - - template<typename TSelectClusterFunc> - bool MoveTheBestClusterToFront(std::vector<TClustersList::TCluster>& clusters, const TSelectClusterFunc selectClusterFunc) const { - auto it = std::find_if(begin(clusters), end(clusters), selectClusterFunc); - - if (it == clusters.end()) { - return false; - } else if (it != begin(clusters)) { - std::swap(*begin(clusters), *it); - } - - return true; - } - - bool FillWriteSessionClusters(const TClustersList& clustersList, - const WriteSessionParams& sessionParams, - WriteSessionClusters& sessionClusters) const - { - auto clusters = clustersList.Clusters; // make a copy for reordering - - bool movedFirstPriorityCluster = false; - WriteSessionClusters::SelectionReason primaryClusterSelectionReason = WriteSessionClusters::CONSISTENT_DISTRIBUTION; - - if (sessionParams.preferred_cluster_name()) { - movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters, - [preferred_cluster_name = sessionParams.preferred_cluster_name()](const auto& cluster) { - return AsciiEqualsIgnoreCase(cluster.Name, preferred_cluster_name); - } - ); - - if (!movedFirstPriorityCluster) { - return false; // failed to prioritize the specified cluster - } - - primaryClusterSelectionReason = WriteSessionClusters::CLIENT_PREFERENCE; - } - - if (!movedFirstPriorityCluster && ClientDatacenterName && !IsInfracloudClient) { - movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters, - [datacenter = *ClientDatacenterName](const auto& cluster) { - return AsciiEqualsIgnoreCase(cluster.Datacenter, datacenter); - } - ); - - if (movedFirstPriorityCluster) { - primaryClusterSelectionReason = WriteSessionClusters::CLIENT_LOCATION; - } - } - - ui64 hashValue = THash<TString>()(sessionParams.topic()); - hashValue = CombineHashes(hashValue, THash<TString>()(sessionParams.source_id())); - hashValue = CombineHashes(hashValue, static_cast<ui64>(sessionParams.partition_group())); - - auto reorderingBeginIt = begin(clusters); - if (!clusters.empty() && movedFirstPriorityCluster) { - ++reorderingBeginIt; - } - - OrderByHashAndWeight(reorderingBeginIt, end(clusters), hashValue, [](const auto& cluster){ return cluster.Weight; }); - - sessionClusters.set_primary_cluster_selection_reason(primaryClusterSelectionReason); - ReportPrimaryClusterSelectionReason(primaryClusterSelectionReason); - - if (!clusters.empty()) { - ReportPrioritizedCluster(clusters.front()); - } - - for (const auto& cluster : clusters) { - auto& clusterInfo = *sessionClusters.add_clusters(); - clusterInfo.set_endpoint(cluster.Balancer); - clusterInfo.set_name(cluster.Name); - clusterInfo.set_available(cluster.IsEnabled); - } - - return true; - } - - bool FillReadSessionClusters(const TClustersList& clustersList, - const ReadSessionParams& sessionParams, - ReadSessionClusters& sessionClusters) const - { - if (sessionParams.has_all_original()) { - for (const auto& cluster : clustersList.Clusters) { - auto& clusterInfo = *sessionClusters.add_clusters(); - clusterInfo.set_endpoint(cluster.Balancer); - clusterInfo.set_name(cluster.Name); - clusterInfo.set_available(true); // at the moment we can't logically disable reading - } - } else { - auto it = std::find_if(begin(clustersList.Clusters), end(clustersList.Clusters), - [mirror_to_cluster = sessionParams.mirror_to_cluster()](const auto& cluster) { - return AsciiEqualsIgnoreCase(cluster.Name, mirror_to_cluster); - } - ); - - if (it != end(clustersList.Clusters)) { - auto& clusterInfo = *sessionClusters.add_clusters(); - clusterInfo.set_endpoint(it->Balancer); - clusterInfo.set_name(it->Name); - clusterInfo.set_available(true); - } else { - return false; - } - } - - return true; - } - - TMaybe<TString> TryDetectDatacenter(const TString& address) const { - if (!DatacenterClassifier) { - return Nothing(); - } - - return DatacenterClassifier->ClassifyAddress(address); - } - - void Process() { + +#include <util/digest/numeric.h> +#include <util/generic/hash.h> +#include <util/string/ascii.h> + +#include <algorithm> + + +namespace NKikimr::NPQ::NClusterDiscovery::NWorker { + +using namespace Ydb::PersQueue::ClusterDiscovery; +using NKikimr::NPQ::NClusterDiscovery::NClusterOrdering::OrderByHashAndWeight; + +inline auto& Ctx() { + return TActivationContext::AsActorContext(); +} + +class TClusterDiscoveryWorker : public TActorBootstrapped<TClusterDiscoveryWorker> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::GRPC_REQ; + } + + TClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, + TLabeledAddressClassifier::TConstPtr datacenterClassifier, + TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, + TClustersList::TConstPtr clustersList, + TClusterDiscoveryCounters::TPtr counters) + : Request(ev->Release().Release()) + , DatacenterClassifier(std::move(datacenterClassifier)) + , CloudNetsClassifier(std::move(cloudNetsClassifier)) + , ClustersList(std::move(clustersList)) + , Counters(counters) + {} + + void Bootstrap() { + RequestStartTs = Ctx().Now(); + Process(); + } + + template<typename TSelectClusterFunc> + bool MoveTheBestClusterToFront(std::vector<TClustersList::TCluster>& clusters, const TSelectClusterFunc selectClusterFunc) const { + auto it = std::find_if(begin(clusters), end(clusters), selectClusterFunc); + + if (it == clusters.end()) { + return false; + } else if (it != begin(clusters)) { + std::swap(*begin(clusters), *it); + } + + return true; + } + + bool FillWriteSessionClusters(const TClustersList& clustersList, + const WriteSessionParams& sessionParams, + WriteSessionClusters& sessionClusters) const + { + auto clusters = clustersList.Clusters; // make a copy for reordering + + bool movedFirstPriorityCluster = false; + WriteSessionClusters::SelectionReason primaryClusterSelectionReason = WriteSessionClusters::CONSISTENT_DISTRIBUTION; + + if (sessionParams.preferred_cluster_name()) { + movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters, + [preferred_cluster_name = sessionParams.preferred_cluster_name()](const auto& cluster) { + return AsciiEqualsIgnoreCase(cluster.Name, preferred_cluster_name); + } + ); + + if (!movedFirstPriorityCluster) { + return false; // failed to prioritize the specified cluster + } + + primaryClusterSelectionReason = WriteSessionClusters::CLIENT_PREFERENCE; + } + + if (!movedFirstPriorityCluster && ClientDatacenterName && !IsInfracloudClient) { + movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters, + [datacenter = *ClientDatacenterName](const auto& cluster) { + return AsciiEqualsIgnoreCase(cluster.Datacenter, datacenter); + } + ); + + if (movedFirstPriorityCluster) { + primaryClusterSelectionReason = WriteSessionClusters::CLIENT_LOCATION; + } + } + + ui64 hashValue = THash<TString>()(sessionParams.topic()); + hashValue = CombineHashes(hashValue, THash<TString>()(sessionParams.source_id())); + hashValue = CombineHashes(hashValue, static_cast<ui64>(sessionParams.partition_group())); + + auto reorderingBeginIt = begin(clusters); + if (!clusters.empty() && movedFirstPriorityCluster) { + ++reorderingBeginIt; + } + + OrderByHashAndWeight(reorderingBeginIt, end(clusters), hashValue, [](const auto& cluster){ return cluster.Weight; }); + + sessionClusters.set_primary_cluster_selection_reason(primaryClusterSelectionReason); + ReportPrimaryClusterSelectionReason(primaryClusterSelectionReason); + + if (!clusters.empty()) { + ReportPrioritizedCluster(clusters.front()); + } + + for (const auto& cluster : clusters) { + auto& clusterInfo = *sessionClusters.add_clusters(); + clusterInfo.set_endpoint(cluster.Balancer); + clusterInfo.set_name(cluster.Name); + clusterInfo.set_available(cluster.IsEnabled); + } + + return true; + } + + bool FillReadSessionClusters(const TClustersList& clustersList, + const ReadSessionParams& sessionParams, + ReadSessionClusters& sessionClusters) const + { + if (sessionParams.has_all_original()) { + for (const auto& cluster : clustersList.Clusters) { + auto& clusterInfo = *sessionClusters.add_clusters(); + clusterInfo.set_endpoint(cluster.Balancer); + clusterInfo.set_name(cluster.Name); + clusterInfo.set_available(true); // at the moment we can't logically disable reading + } + } else { + auto it = std::find_if(begin(clustersList.Clusters), end(clustersList.Clusters), + [mirror_to_cluster = sessionParams.mirror_to_cluster()](const auto& cluster) { + return AsciiEqualsIgnoreCase(cluster.Name, mirror_to_cluster); + } + ); + + if (it != end(clustersList.Clusters)) { + auto& clusterInfo = *sessionClusters.add_clusters(); + clusterInfo.set_endpoint(it->Balancer); + clusterInfo.set_name(it->Name); + clusterInfo.set_available(true); + } else { + return false; + } + } + + return true; + } + + TMaybe<TString> TryDetectDatacenter(const TString& address) const { + if (!DatacenterClassifier) { + return Nothing(); + } + + return DatacenterClassifier->ClassifyAddress(address); + } + + void Process() { auto* result = TEvDiscoverPQClustersRequest::AllocateResult<DiscoverClustersResult>(Request); - - auto statusCode = Ydb::StatusIds::INTERNAL_ERROR; - - if (ClustersList) { - const TString address = NAddressClassifier::ExtractAddress(Request->GetPeerName()); - - ClientDatacenterName = TryDetectDatacenter(address); - - if (CloudNetsClassifier && CloudNetsClassifier->ClassifyAddress(address)) { - IsInfracloudClient = true; - } - - statusCode = ProcessWriteSessions(*result); - if (statusCode == Ydb::StatusIds::SUCCESS) { - statusCode = ProcessReadSessions(*result); - } - - result->set_version(ClustersList->Version); - } - - ReportMetrics(statusCode); - - Request->SendResult(*result, statusCode); - - return PassAway(); - } - - Ydb::StatusIds::StatusCode ProcessWriteSessions(DiscoverClustersResult& result) const { - const size_t sessionsCount = Request->GetProtoRequest()->write_sessions_size(); - for (size_t i = 0; i < sessionsCount; ++i) { - if (!FillWriteSessionClusters(*ClustersList, - Request->GetProtoRequest()->write_sessions(i), - *result.add_write_sessions_clusters())) - { - return Ydb::StatusIds::BAD_REQUEST; - } - } - - return Ydb::StatusIds::SUCCESS; - } - - Ydb::StatusIds::StatusCode ProcessReadSessions(DiscoverClustersResult& result) const { - const size_t sessionsCount = Request->GetProtoRequest()->read_sessions_size(); - for (size_t i = 0; i < sessionsCount; ++i) { - if (!FillReadSessionClusters(*ClustersList, - Request->GetProtoRequest()->read_sessions(i), - *result.add_read_sessions_clusters())) - { - return Ydb::StatusIds::BAD_REQUEST; - } - } - - return Ydb::StatusIds::SUCCESS; - } - - // Monitoring - - template<typename TMappedCounters, typename TLabel> - void IncrementLabeledCounter(TMappedCounters& counters, const TLabel& label) const { - auto it = counters.find(label); - - Y_VERIFY(it != counters.end()); - - it->second->Inc(); - } - - void ReportPrioritizedCluster(const TClustersList::TCluster& cluster) const { - IncrementLabeledCounter(Counters->TimesPrioritizedForWrite, cluster.Name); - } - - void ReportPrimaryClusterSelectionReason(const WriteSessionClusters::SelectionReason reason) const { - IncrementLabeledCounter(Counters->PrimaryClusterSelectionReasons, reason); - } - - void ReportMetrics(const Ydb::StatusIds::StatusCode statusCode) const { - if (statusCode == Ydb::StatusIds::SUCCESS) { - Counters->SuccessfulRequestsCount->Inc(); - } else if (statusCode == Ydb::StatusIds::BAD_REQUEST) { - Counters->BadRequestsCount->Inc(); - } else { - Y_VERIFY(statusCode == Ydb::StatusIds::INTERNAL_ERROR); - Counters->FailedRequestsCount->Inc(); - } - - Counters->WriteDiscoveriesCount->Add(Request->GetProtoRequest()->write_sessions_size()); - Counters->ReadDiscoveriesCount->Add(Request->GetProtoRequest()->read_sessions_size()); - - if (ClientDatacenterName) { - IncrementLabeledCounter(Counters->TimesResolvedByClassifier, *ClientDatacenterName); - } - - if (IsInfracloudClient) { - Counters->InfracloudRequestsCount->Inc(); - } - - Counters->DiscoveryWorkingDurationMs->Collect(Ctx().Now().MilliSeconds() - RequestStartTs.MilliSeconds()); - } - -private: - THolder<TEvDiscoverPQClustersRequest> Request; - TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null - TLabeledAddressClassifier::TConstPtr CloudNetsClassifier; // Special classifier instance for cloud networks detection. May be null - TMaybe<TString> ClientDatacenterName; // Detected client datacenter - bool IsInfracloudClient = false; // Client came from infracloud - TClustersList::TConstPtr ClustersList; // Cached list of clusters - TInstant RequestStartTs; - - TClusterDiscoveryCounters::TPtr Counters; -}; - -IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, - TLabeledAddressClassifier::TConstPtr datacenterClassifier, - TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, - TClustersList::TConstPtr clustersList, - TClusterDiscoveryCounters::TPtr counters) -{ - return new TClusterDiscoveryWorker(ev, - std::move(datacenterClassifier), - std::move(cloudNetsClassifier), - std::move(clustersList), - std::move(counters) - ); -} - -} // namespace NKikimr::NPQ::NClusterDiscovery::NWorker + + auto statusCode = Ydb::StatusIds::INTERNAL_ERROR; + + if (ClustersList) { + const TString address = NAddressClassifier::ExtractAddress(Request->GetPeerName()); + + ClientDatacenterName = TryDetectDatacenter(address); + + if (CloudNetsClassifier && CloudNetsClassifier->ClassifyAddress(address)) { + IsInfracloudClient = true; + } + + statusCode = ProcessWriteSessions(*result); + if (statusCode == Ydb::StatusIds::SUCCESS) { + statusCode = ProcessReadSessions(*result); + } + + result->set_version(ClustersList->Version); + } + + ReportMetrics(statusCode); + + Request->SendResult(*result, statusCode); + + return PassAway(); + } + + Ydb::StatusIds::StatusCode ProcessWriteSessions(DiscoverClustersResult& result) const { + const size_t sessionsCount = Request->GetProtoRequest()->write_sessions_size(); + for (size_t i = 0; i < sessionsCount; ++i) { + if (!FillWriteSessionClusters(*ClustersList, + Request->GetProtoRequest()->write_sessions(i), + *result.add_write_sessions_clusters())) + { + return Ydb::StatusIds::BAD_REQUEST; + } + } + + return Ydb::StatusIds::SUCCESS; + } + + Ydb::StatusIds::StatusCode ProcessReadSessions(DiscoverClustersResult& result) const { + const size_t sessionsCount = Request->GetProtoRequest()->read_sessions_size(); + for (size_t i = 0; i < sessionsCount; ++i) { + if (!FillReadSessionClusters(*ClustersList, + Request->GetProtoRequest()->read_sessions(i), + *result.add_read_sessions_clusters())) + { + return Ydb::StatusIds::BAD_REQUEST; + } + } + + return Ydb::StatusIds::SUCCESS; + } + + // Monitoring + + template<typename TMappedCounters, typename TLabel> + void IncrementLabeledCounter(TMappedCounters& counters, const TLabel& label) const { + auto it = counters.find(label); + + Y_VERIFY(it != counters.end()); + + it->second->Inc(); + } + + void ReportPrioritizedCluster(const TClustersList::TCluster& cluster) const { + IncrementLabeledCounter(Counters->TimesPrioritizedForWrite, cluster.Name); + } + + void ReportPrimaryClusterSelectionReason(const WriteSessionClusters::SelectionReason reason) const { + IncrementLabeledCounter(Counters->PrimaryClusterSelectionReasons, reason); + } + + void ReportMetrics(const Ydb::StatusIds::StatusCode statusCode) const { + if (statusCode == Ydb::StatusIds::SUCCESS) { + Counters->SuccessfulRequestsCount->Inc(); + } else if (statusCode == Ydb::StatusIds::BAD_REQUEST) { + Counters->BadRequestsCount->Inc(); + } else { + Y_VERIFY(statusCode == Ydb::StatusIds::INTERNAL_ERROR); + Counters->FailedRequestsCount->Inc(); + } + + Counters->WriteDiscoveriesCount->Add(Request->GetProtoRequest()->write_sessions_size()); + Counters->ReadDiscoveriesCount->Add(Request->GetProtoRequest()->read_sessions_size()); + + if (ClientDatacenterName) { + IncrementLabeledCounter(Counters->TimesResolvedByClassifier, *ClientDatacenterName); + } + + if (IsInfracloudClient) { + Counters->InfracloudRequestsCount->Inc(); + } + + Counters->DiscoveryWorkingDurationMs->Collect(Ctx().Now().MilliSeconds() - RequestStartTs.MilliSeconds()); + } + +private: + THolder<TEvDiscoverPQClustersRequest> Request; + TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null + TLabeledAddressClassifier::TConstPtr CloudNetsClassifier; // Special classifier instance for cloud networks detection. May be null + TMaybe<TString> ClientDatacenterName; // Detected client datacenter + bool IsInfracloudClient = false; // Client came from infracloud + TClustersList::TConstPtr ClustersList; // Cached list of clusters + TInstant RequestStartTs; + + TClusterDiscoveryCounters::TPtr Counters; +}; + +IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, + TLabeledAddressClassifier::TConstPtr datacenterClassifier, + TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, + TClustersList::TConstPtr clustersList, + TClusterDiscoveryCounters::TPtr counters) +{ + return new TClusterDiscoveryWorker(ev, + std::move(datacenterClassifier), + std::move(cloudNetsClassifier), + std::move(clustersList), + std::move(counters) + ); +} + +} // namespace NKikimr::NPQ::NClusterDiscovery::NWorker diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h index c92bb64940..65cfe165f8 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h +++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h @@ -1,25 +1,25 @@ -#pragma once - -#include "counters.h" - +#pragma once + +#include "counters.h" + #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/persqueue/cluster_tracker.h> #include <ydb/core/util/address_classifier.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> - -namespace NKikimr::NPQ::NClusterDiscovery::NWorker { - -using namespace NActors; -using namespace NAddressClassifier; -using namespace NGRpcService; -using namespace NPQ::NClusterTracker; -using namespace NCounters; - -IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, - TLabeledAddressClassifier::TConstPtr datacenterClassifier, - TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, - TClustersList::TConstPtr clustersList, - TClusterDiscoveryCounters::TPtr counters); - -} // namespace NKikimr::NClusterDiscovery::NWorker + +namespace NKikimr::NPQ::NClusterDiscovery::NWorker { + +using namespace NActors; +using namespace NAddressClassifier; +using namespace NGRpcService; +using namespace NPQ::NClusterTracker; +using namespace NCounters; + +IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev, + TLabeledAddressClassifier::TConstPtr datacenterClassifier, + TLabeledAddressClassifier::TConstPtr cloudNetsClassifier, + TClustersList::TConstPtr clustersList, + TClusterDiscoveryCounters::TPtr counters); + +} // namespace NKikimr::NClusterDiscovery::NWorker diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make index c1e137b002..f7863157f3 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make @@ -1,17 +1,17 @@ UNITTEST_FOR(ydb/services/persqueue_cluster_discovery/cluster_ordering) - -OWNER( - radix + +OWNER( + radix g:kikimr g:logbroker -) - -SRCS( - weighed_ordering_ut.cpp -) - -PEERDIR( +) + +SRCS( + weighed_ordering_ut.cpp +) + +PEERDIR( ydb/services/persqueue_cluster_discovery/cluster_ordering -) - -END() +) + +END() diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp index be1823bcff..35c0234095 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp @@ -1 +1 @@ -#include "weighed_ordering.h" +#include "weighed_ordering.h" diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h index a9695507ec..13b045cf5e 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h @@ -1,50 +1,50 @@ -#pragma once - -#include <algorithm> -#include <type_traits> -#include <vector> - -#include <util/system/types.h> +#pragma once + +#include <algorithm> +#include <type_traits> +#include <vector> + +#include <util/system/types.h> #include <util/system/yassert.h> - -namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering { - -template <typename Iterator, typename GetWeightFunc> -Iterator SelectByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) { - if (std::distance(begin, end) < 2) { - return begin; - } - - static_assert(std::is_integral<decltype(getWeight(*begin))>::value); - - std::vector<std::pair<const ui64, const Iterator>> borders; - borders.reserve(std::distance(begin, end)); - - ui64 total = 0; - for (auto it = begin; it != end; ++it) { - total += getWeight(*it); - borders.emplace_back(total, it); - } - + +namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering { + +template <typename Iterator, typename GetWeightFunc> +Iterator SelectByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) { + if (std::distance(begin, end) < 2) { + return begin; + } + + static_assert(std::is_integral<decltype(getWeight(*begin))>::value); + + std::vector<std::pair<const ui64, const Iterator>> borders; + borders.reserve(std::distance(begin, end)); + + ui64 total = 0; + for (auto it = begin; it != end; ++it) { + total += getWeight(*it); + borders.emplace_back(total, it); + } + Y_VERIFY(total); - const ui64 borderToFind = hashValue % total + 1; - - const auto bordersIt = lower_bound(borders.begin(), borders.end(), borderToFind, - [](const auto& borderInfo, const ui64& toFind) { return borderInfo.first < toFind; }); - - return bordersIt->second; -} - -template <typename Iterator, typename GetWeightFunc> -void OrderByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) { - auto currentBegin = begin; - while (std::distance(currentBegin, end) > 1) { - auto selectedIt = SelectByHashAndWeight(currentBegin, end, hashValue, getWeight); - if (currentBegin != selectedIt) { - std::swap(*currentBegin, *selectedIt); - } - ++currentBegin; - } -} - -} // namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering + const ui64 borderToFind = hashValue % total + 1; + + const auto bordersIt = lower_bound(borders.begin(), borders.end(), borderToFind, + [](const auto& borderInfo, const ui64& toFind) { return borderInfo.first < toFind; }); + + return bordersIt->second; +} + +template <typename Iterator, typename GetWeightFunc> +void OrderByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) { + auto currentBegin = begin; + while (std::distance(currentBegin, end) > 1) { + auto selectedIt = SelectByHashAndWeight(currentBegin, end, hashValue, getWeight); + if (currentBegin != selectedIt) { + std::swap(*currentBegin, *selectedIt); + } + ++currentBegin; + } +} + +} // namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp index 068f90e293..b56071421a 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp @@ -1,55 +1,55 @@ #include <ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/generic/hash.h> -#include <util/generic/string.h> - -using namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering; - -Y_UNIT_TEST_SUITE(TWeighedOrderingTest) { - struct TCluster { - TString Name; - ui64 Weight = 0; - - bool operator==(const TCluster& other) const { - return Name == other.Name && Weight == other.Weight; - } - }; - - Y_UNIT_TEST(SimpleSelectionTest) { - std::vector<TCluster> clusters = {{"sas", 1}, {"man", 1}}; - - UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 0, [](const TCluster& c){ return c.Weight; })->Name, "sas"); - UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 1, [](const TCluster& c){ return c.Weight; })->Name, "man"); - UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100500, [](const TCluster& c){ return c.Weight; })->Name, "sas"); - UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; })->Name, "man"); - } - - Y_UNIT_TEST(WeighedSelectionTest) { - std::vector<TCluster> clusters = {{"sas", 10}, {"man", 10000}}; - - THashMap<TString, ui64> counters; - for (ui64 hashValue = 0; hashValue < 10000000; hashValue += 661) { - ++counters[SelectByHashAndWeight(begin(clusters), end(clusters), hashValue, [](const TCluster& c){ return c.Weight; })->Name]; - } - - UNIT_ASSERT_VALUES_EQUAL(counters.size(), 2); - UNIT_ASSERT(counters["man"] > counters["sas"] * 1000); // ensure the imbalance - } - - Y_UNIT_TEST(WeighedOrderingTest) { - std::vector<TCluster> clusters = {{"iva", 1}, {"man", 2}, {"myt", 3}, {"sas", 4}, {"vla", 5}}; - - auto firstOrder = clusters; - OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; }); - auto secondOrder = clusters; - OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; }); - auto thirdOrder = clusters; - - UNIT_ASSERT(firstOrder != secondOrder); - UNIT_ASSERT(secondOrder != thirdOrder); - UNIT_ASSERT(firstOrder != thirdOrder); - } -} + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +using namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering; + +Y_UNIT_TEST_SUITE(TWeighedOrderingTest) { + struct TCluster { + TString Name; + ui64 Weight = 0; + + bool operator==(const TCluster& other) const { + return Name == other.Name && Weight == other.Weight; + } + }; + + Y_UNIT_TEST(SimpleSelectionTest) { + std::vector<TCluster> clusters = {{"sas", 1}, {"man", 1}}; + + UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 0, [](const TCluster& c){ return c.Weight; })->Name, "sas"); + UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 1, [](const TCluster& c){ return c.Weight; })->Name, "man"); + UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100500, [](const TCluster& c){ return c.Weight; })->Name, "sas"); + UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; })->Name, "man"); + } + + Y_UNIT_TEST(WeighedSelectionTest) { + std::vector<TCluster> clusters = {{"sas", 10}, {"man", 10000}}; + + THashMap<TString, ui64> counters; + for (ui64 hashValue = 0; hashValue < 10000000; hashValue += 661) { + ++counters[SelectByHashAndWeight(begin(clusters), end(clusters), hashValue, [](const TCluster& c){ return c.Weight; })->Name]; + } + + UNIT_ASSERT_VALUES_EQUAL(counters.size(), 2); + UNIT_ASSERT(counters["man"] > counters["sas"] * 1000); // ensure the imbalance + } + + Y_UNIT_TEST(WeighedOrderingTest) { + std::vector<TCluster> clusters = {{"iva", 1}, {"man", 2}, {"myt", 3}, {"sas", 4}, {"vla", 5}}; + + auto firstOrder = clusters; + OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; }); + auto secondOrder = clusters; + OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; }); + auto thirdOrder = clusters; + + UNIT_ASSERT(firstOrder != secondOrder); + UNIT_ASSERT(secondOrder != thirdOrder); + UNIT_ASSERT(firstOrder != thirdOrder); + } +} diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make index 6978da1d9b..f01a54c355 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make @@ -1,13 +1,13 @@ -OWNER( - radix +OWNER( + radix g:kikimr g:logbroker -) - -LIBRARY() - -SRCS( - weighed_ordering.cpp -) - -END() +) + +LIBRARY() + +SRCS( + weighed_ordering.cpp +) + +END() diff --git a/ydb/services/persqueue_cluster_discovery/counters.cpp b/ydb/services/persqueue_cluster_discovery/counters.cpp index c1a9bf7589..3cfc4eb863 100644 --- a/ydb/services/persqueue_cluster_discovery/counters.cpp +++ b/ydb/services/persqueue_cluster_discovery/counters.cpp @@ -1,56 +1,56 @@ -#include "counters.h" - -namespace NKikimr::NPQ::NClusterDiscovery::NCounters { - -#define SETUP_SIMPLE_COUNTER(name, derived) \ - name(counters->GetCounter(#name, derived)) - -TClusterDiscoveryCounters::TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters, - NClusterTracker::TClustersList::TConstPtr clustersList, - TLabeledAddressClassifier::TConstPtr addressClassifier) - : SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false) - , SETUP_SIMPLE_COUNTER(ClustersListUpdateLagSeconds, false) - , SETUP_SIMPLE_COUNTER(HealthProbe, false) - , SETUP_SIMPLE_COUNTER(DroppedRequestsCount, true) - , SETUP_SIMPLE_COUNTER(TotalRequestsCount, true) - , SETUP_SIMPLE_COUNTER(SuccessfulRequestsCount, true) - , SETUP_SIMPLE_COUNTER(BadRequestsCount, true) - , SETUP_SIMPLE_COUNTER(FailedRequestsCount, true) - , SETUP_SIMPLE_COUNTER(WriteDiscoveriesCount, true) - , SETUP_SIMPLE_COUNTER(ReadDiscoveriesCount, true) - , SETUP_SIMPLE_COUNTER(InfracloudRequestsCount, true) -{ - DiscoveryWorkingDurationMs = - counters->GetHistogram("DiscoveryWorkingDurationMs", - NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 25, 50, 100, 500})); - - if (clustersList) { - for (const auto& cluster : clustersList->Clusters) { - TimesPrioritizedForWrite[cluster.Name] = - counters->GetSubgroup("prioritized_cluster", cluster.Name)->GetCounter("TimesPrioritizedForWrite", true); - } - } - - if (addressClassifier) { - for (const auto& label : addressClassifier->GetLabels()) { - TimesResolvedByClassifier[label] = - counters->GetSubgroup("resolved_dc", label)->GetCounter("TimesResolvedByClassifier", true); - } - } - - for ( - std::underlying_type<WriteSessionClusters::SelectionReason>::type reason = WriteSessionClusters::SELECTION_REASON_UNSPECIFIED; - reason <= WriteSessionClusters::CONSISTENT_DISTRIBUTION; - ++reason - ) - { - const auto currentReason = static_cast<WriteSessionClusters::SelectionReason>(reason); - PrimaryClusterSelectionReasons[currentReason] = - counters->GetSubgroup("primary_cluster_selection_reason", ToString(currentReason))->GetCounter("TimesMentionedForWrite", true); - } - -} - -#undef SETUP_SIMPLE_COUNTER - -} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters +#include "counters.h" + +namespace NKikimr::NPQ::NClusterDiscovery::NCounters { + +#define SETUP_SIMPLE_COUNTER(name, derived) \ + name(counters->GetCounter(#name, derived)) + +TClusterDiscoveryCounters::TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters, + NClusterTracker::TClustersList::TConstPtr clustersList, + TLabeledAddressClassifier::TConstPtr addressClassifier) + : SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false) + , SETUP_SIMPLE_COUNTER(ClustersListUpdateLagSeconds, false) + , SETUP_SIMPLE_COUNTER(HealthProbe, false) + , SETUP_SIMPLE_COUNTER(DroppedRequestsCount, true) + , SETUP_SIMPLE_COUNTER(TotalRequestsCount, true) + , SETUP_SIMPLE_COUNTER(SuccessfulRequestsCount, true) + , SETUP_SIMPLE_COUNTER(BadRequestsCount, true) + , SETUP_SIMPLE_COUNTER(FailedRequestsCount, true) + , SETUP_SIMPLE_COUNTER(WriteDiscoveriesCount, true) + , SETUP_SIMPLE_COUNTER(ReadDiscoveriesCount, true) + , SETUP_SIMPLE_COUNTER(InfracloudRequestsCount, true) +{ + DiscoveryWorkingDurationMs = + counters->GetHistogram("DiscoveryWorkingDurationMs", + NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 25, 50, 100, 500})); + + if (clustersList) { + for (const auto& cluster : clustersList->Clusters) { + TimesPrioritizedForWrite[cluster.Name] = + counters->GetSubgroup("prioritized_cluster", cluster.Name)->GetCounter("TimesPrioritizedForWrite", true); + } + } + + if (addressClassifier) { + for (const auto& label : addressClassifier->GetLabels()) { + TimesResolvedByClassifier[label] = + counters->GetSubgroup("resolved_dc", label)->GetCounter("TimesResolvedByClassifier", true); + } + } + + for ( + std::underlying_type<WriteSessionClusters::SelectionReason>::type reason = WriteSessionClusters::SELECTION_REASON_UNSPECIFIED; + reason <= WriteSessionClusters::CONSISTENT_DISTRIBUTION; + ++reason + ) + { + const auto currentReason = static_cast<WriteSessionClusters::SelectionReason>(reason); + PrimaryClusterSelectionReasons[currentReason] = + counters->GetSubgroup("primary_cluster_selection_reason", ToString(currentReason))->GetCounter("TimesMentionedForWrite", true); + } + +} + +#undef SETUP_SIMPLE_COUNTER + +} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters diff --git a/ydb/services/persqueue_cluster_discovery/counters.h b/ydb/services/persqueue_cluster_discovery/counters.h index 495b2e6939..fcfbd8db65 100644 --- a/ydb/services/persqueue_cluster_discovery/counters.h +++ b/ydb/services/persqueue_cluster_discovery/counters.h @@ -1,53 +1,53 @@ -#pragma once - +#pragma once + #include <ydb/core/grpc_services/grpc_request_proxy.h> - + #include <ydb/core/persqueue/cluster_tracker.h> #include <ydb/core/util/address_classifier.h> - + #include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/noncopyable.h> - -namespace NKikimr::NPQ::NClusterDiscovery::NCounters { - -using namespace NAddressClassifier; -using namespace NClusterTracker; -using namespace NMonitoring; -using namespace Ydb::PersQueue::ClusterDiscovery; - -struct TClusterDiscoveryCounters : TAtomicRefCount<TClusterDiscoveryCounters>, TNonCopyable { - using TPtr = TIntrusivePtr<TClusterDiscoveryCounters>; - using TCounterPtr = TDynamicCounters::TCounterPtr; - - TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters, - TClustersList::TConstPtr clustersList, - TLabeledAddressClassifier::TConstPtr addressClassifier); - - TCounterPtr NetDataUpdateLagSeconds; // Seconds since last net data update - TCounterPtr ClustersListUpdateLagSeconds; // Seconds since last clusters list update - TCounterPtr HealthProbe; // Health check results timeline - - TCounterPtr DroppedRequestsCount; // GRpc status: UNAVAILABLE - TCounterPtr TotalRequestsCount; // GRpc status: ANY - TCounterPtr SuccessfulRequestsCount; // GRpc status: SUCCESS - TCounterPtr BadRequestsCount; // GRrpc status: BAD_REQUEST - TCounterPtr FailedRequestsCount; // GRpc status: INTERNAL_ERROR - - TCounterPtr WriteDiscoveriesCount; // Write discovery proto elements count in request - TCounterPtr ReadDiscoveriesCount; // Read discovery proto elements count in request - - TCounterPtr InfracloudRequestsCount; // Requests coming from infracloud clients - - THistogramPtr DiscoveryWorkingDurationMs; // NB: dropped requests are not measured - - THashMap<TString, TCounterPtr> TimesPrioritizedForWrite; // Times a cluster with a certain name was prioritized - THashMap<TString, TCounterPtr> TimesResolvedByClassifier; // Times a client's datacenter with a certain name was resolved - - // Primary cluster selection reasons for write sessions - THashMap<WriteSessionClusters::SelectionReason, TCounterPtr> PrimaryClusterSelectionReasons; -}; - -} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters + +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/noncopyable.h> + +namespace NKikimr::NPQ::NClusterDiscovery::NCounters { + +using namespace NAddressClassifier; +using namespace NClusterTracker; +using namespace NMonitoring; +using namespace Ydb::PersQueue::ClusterDiscovery; + +struct TClusterDiscoveryCounters : TAtomicRefCount<TClusterDiscoveryCounters>, TNonCopyable { + using TPtr = TIntrusivePtr<TClusterDiscoveryCounters>; + using TCounterPtr = TDynamicCounters::TCounterPtr; + + TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters, + TClustersList::TConstPtr clustersList, + TLabeledAddressClassifier::TConstPtr addressClassifier); + + TCounterPtr NetDataUpdateLagSeconds; // Seconds since last net data update + TCounterPtr ClustersListUpdateLagSeconds; // Seconds since last clusters list update + TCounterPtr HealthProbe; // Health check results timeline + + TCounterPtr DroppedRequestsCount; // GRpc status: UNAVAILABLE + TCounterPtr TotalRequestsCount; // GRpc status: ANY + TCounterPtr SuccessfulRequestsCount; // GRpc status: SUCCESS + TCounterPtr BadRequestsCount; // GRrpc status: BAD_REQUEST + TCounterPtr FailedRequestsCount; // GRpc status: INTERNAL_ERROR + + TCounterPtr WriteDiscoveriesCount; // Write discovery proto elements count in request + TCounterPtr ReadDiscoveriesCount; // Read discovery proto elements count in request + + TCounterPtr InfracloudRequestsCount; // Requests coming from infracloud clients + + THistogramPtr DiscoveryWorkingDurationMs; // NB: dropped requests are not measured + + THashMap<TString, TCounterPtr> TimesPrioritizedForWrite; // Times a cluster with a certain name was prioritized + THashMap<TString, TCounterPtr> TimesResolvedByClassifier; // Times a client's datacenter with a certain name was resolved + + // Primary cluster selection reasons for write sessions + THashMap<WriteSessionClusters::SelectionReason, TCounterPtr> PrimaryClusterSelectionReasons; +}; + +} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp index 086d162443..0a729b4d09 100644 --- a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp +++ b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp @@ -1,86 +1,86 @@ -#include "grpc_service.h" -#include "cluster_discovery_service.h" - +#include "grpc_service.h" +#include "cluster_discovery_service.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> - + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> - + #include <library/cpp/grpc/server/grpc_request.h> - - -namespace NKikimr::NGRpcService { - + + +namespace NKikimr::NGRpcService { + TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService( NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit ) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) { if (requestsInflightLimit.Defined()) { Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef()); } } - + void TGRpcPQClusterDiscoveryService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; - - if (ActorSystem_->AppData<TAppData>()->PQClusterDiscoveryConfig.GetEnabled()) { - IActor* actor = NPQ::NClusterDiscovery::CreateClusterDiscoveryService(GetServiceCounters(Counters_, "persqueue")->GetSubgroup("subsystem", "cluster_discovery")); + CQ_ = cq; + + if (ActorSystem_->AppData<TAppData>()->PQClusterDiscoveryConfig.GetEnabled()) { + IActor* actor = NPQ::NClusterDiscovery::CreateClusterDiscoveryService(GetServiceCounters(Counters_, "persqueue")->GetSubgroup("subsystem", "cluster_discovery")); TActorId clusterDiscoveryServiceId = ActorSystem_->Register(actor, TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); - ActorSystem_->RegisterLocalService(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID(), clusterDiscoveryServiceId); - + ActorSystem_->RegisterLocalService(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID(), clusterDiscoveryServiceId); + SetupIncomingRequests(std::move(logger)); - } -} - + } +} + void TGRpcPQClusterDiscoveryService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter*) { -} - -bool TGRpcPQClusterDiscoveryService::IncRequest() { +} + +bool TGRpcPQClusterDiscoveryService::IncRequest() { if (Limiter) { return Limiter->Inc(); } return true; -} - -void TGRpcPQClusterDiscoveryService::DecRequest() { +} + +void TGRpcPQClusterDiscoveryService::DecRequest() { if (Limiter) { Limiter->Dec(); } -} - +} + void TGRpcPQClusterDiscoveryService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - auto getCounterBlock = NGRpcService::CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::PersQueue::ClusterDiscovery::IN, Ydb::PersQueue::ClusterDiscovery::OUT, TGRpcPQClusterDiscoveryService>>(this, &Service_, CQ_, \ + auto getCounterBlock = NGRpcService::CreateCounterCb(Counters_, ActorSystem_); + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + MakeIntrusive<TGRpcRequest<Ydb::PersQueue::ClusterDiscovery::IN, Ydb::PersQueue::ClusterDiscovery::OUT, TGRpcPQClusterDiscoveryService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase* ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ACTION; \ - }, &Ydb::PersQueue::V1::ClusterDiscoveryService::AsyncService::Request ## NAME, \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ACTION; \ + }, &Ydb::PersQueue::V1::ClusterDiscoveryService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("pq_cluster_discovery", #NAME))->Run(); - - ADD_REQUEST(DiscoverClusters, DiscoverClustersRequest, DiscoverClustersResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvDiscoverPQClustersRequest(ctx)); - }) -#undef ADD_REQUEST - -} - -void TGRpcPQClusterDiscoveryService::StopService() noexcept { - TGrpcServiceBase::StopService(); -} - -void TGRpcRequestProxy::Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Send(ev->Forward(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID())); -} - -} // namespace NKikimr::NGRpcService + + ADD_REQUEST(DiscoverClusters, DiscoverClustersRequest, DiscoverClustersResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvDiscoverPQClustersRequest(ctx)); + }) +#undef ADD_REQUEST + +} + +void TGRpcPQClusterDiscoveryService::StopService() noexcept { + TGrpcServiceBase::StopService(); +} + +void TGRpcRequestProxy::Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Send(ev->Forward(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID())); +} + +} // namespace NKikimr::NGRpcService diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.h b/ydb/services/persqueue_cluster_discovery/grpc_service.h index 7bc17ba2f5..5d430364b8 100644 --- a/ydb/services/persqueue_cluster_discovery/grpc_service.h +++ b/ydb/services/persqueue_cluster_discovery/grpc_service.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <library/cpp/grpc/server/grpc_server.h> - -namespace NKikimr::NGRpcService { - -class TGRpcPQClusterDiscoveryService + +namespace NKikimr::NGRpcService { + +class TGRpcPQClusterDiscoveryService : public NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService> { -public: +public: TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing()); - + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - bool IncRequest(); - void DecRequest(); - void StopService() noexcept override; - + bool IncRequest(); + void DecRequest(); + void StopService() noexcept override; + using NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService>::GetService; -private: +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; THolder<NGrpc::TGlobalLimiter> Limiter; -}; - -} +}; + +} diff --git a/ydb/services/persqueue_cluster_discovery/ut/ya.make b/ydb/services/persqueue_cluster_discovery/ut/ya.make index 352988d2fe..1ef703a01f 100644 --- a/ydb/services/persqueue_cluster_discovery/ut/ya.make +++ b/ydb/services/persqueue_cluster_discovery/ut/ya.make @@ -1,34 +1,34 @@ UNITTEST_FOR(ydb/services/persqueue_cluster_discovery) - + OWNER( radix g:kikimr g:logbroker ) + +FORK_SUBTESTS() -FORK_SUBTESTS() - -IF (WITH_VALGRIND) - TIMEOUT(1800) - SIZE(LARGE) - TAG(ya:fat) +IF (WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) REQUIREMENTS(ram:32) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -SRCS( - cluster_discovery_service_ut.cpp -) - -PEERDIR( +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + cluster_discovery_service_ut.cpp +) + +PEERDIR( library/cpp/actors/http ydb/core/testlib ydb/public/api/grpc ydb/services/persqueue_cluster_discovery -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/services/persqueue_cluster_discovery/ya.make b/ydb/services/persqueue_cluster_discovery/ya.make index 23917ae376..aea0ea543e 100644 --- a/ydb/services/persqueue_cluster_discovery/ya.make +++ b/ydb/services/persqueue_cluster_discovery/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER( - radix +LIBRARY() + +OWNER( + radix g:kikimr g:logbroker -) - -SRCS( - cluster_discovery_service.cpp - cluster_discovery_worker.cpp - counters.cpp - grpc_service.cpp -) - -PEERDIR( +) + +SRCS( + cluster_discovery_service.cpp + cluster_discovery_worker.cpp + counters.cpp + grpc_service.cpp +) + +PEERDIR( ydb/core/base ydb/core/client/server ydb/core/grpc_services @@ -26,9 +26,9 @@ PEERDIR( ydb/public/api/grpc/draft ydb/public/api/protos ydb/services/persqueue_cluster_discovery/cluster_ordering -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/services/persqueue_v1/grpc_pq_read.cpp b/ydb/services/persqueue_v1/grpc_pq_read.cpp index 6b7e72fdd0..eb7fbd35b1 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read.cpp @@ -4,8 +4,8 @@ #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/tx/scheme_board/cache.h> -#include <algorithm> - +#include <algorithm> + using namespace NActors; using namespace NKikimrClient; @@ -71,24 +71,24 @@ void TPQReadService::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdat DatacenterClassifier = ev->Get()->Classifier; } -void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { - Y_VERIFY(ev->Get()->ClustersList); - - Y_VERIFY(ev->Get()->ClustersList->Clusters.size()); - - const auto& clusters = ev->Get()->ClustersList->Clusters; - - LocalCluster = {}; - - auto it = std::find_if(begin(clusters), end(clusters), [](const auto& cluster) { return cluster.IsLocal; }); - if (it != end(clusters)) { - LocalCluster = it->Name; - } - - Clusters.resize(clusters.size()); - for (size_t i = 0; i < clusters.size(); ++i) { - Clusters[i] = clusters[i].Name; - } +void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) { + Y_VERIFY(ev->Get()->ClustersList); + + Y_VERIFY(ev->Get()->ClustersList->Clusters.size()); + + const auto& clusters = ev->Get()->ClustersList->Clusters; + + LocalCluster = {}; + + auto it = std::find_if(begin(clusters), end(clusters), [](const auto& cluster) { return cluster.IsLocal; }); + if (it != end(clusters)) { + LocalCluster = it->Name; + } + + Clusters.resize(clusters.size()); + for (size_t i = 0; i < clusters.size(); ++i) { + Clusters[i] = clusters[i].Name; + } TopicsHandler->UpdateClusters(Clusters, LocalCluster); } diff --git a/ydb/services/persqueue_v1/grpc_pq_read.h b/ydb/services/persqueue_v1/grpc_pq_read.h index 558063ed13..3348688a0a 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.h +++ b/ydb/services/persqueue_v1/grpc_pq_read.h @@ -46,7 +46,7 @@ private: switch (ev->GetTypeRewrite()) { HFunc(NKikimr::NGRpcService::TEvStreamPQReadRequest, Handle); HFunc(NKikimr::NGRpcService::TEvPQReadInfoRequest, Handle); - hFunc(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate, Handle); + hFunc(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate, Handle); HFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, Handle); HFunc(TEvPQProxy::TEvSessionDead, Handle); } @@ -55,7 +55,7 @@ private: private: void Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr& ev, const TActorContext& ctx); void Handle(NKikimr::NGRpcService::TEvPQReadInfoRequest::TPtr& ev, const TActorContext& ctx); - void Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev); + void Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev); void Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQProxy::TEvSessionDead::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp index 5f17a2f3ab..c18175de4e 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp @@ -66,14 +66,14 @@ void TPQWriteService::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpda void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev, const TActorContext& ctx) { - Y_VERIFY(ev->Get()->ClustersList); - Y_VERIFY(ev->Get()->ClustersList->Clusters.size()); - - const auto& clusters = ev->Get()->ClustersList->Clusters; - - LocalCluster = ""; - Enabled = false; - + Y_VERIFY(ev->Get()->ClustersList); + Y_VERIFY(ev->Get()->ClustersList->Clusters.size()); + + const auto& clusters = ev->Get()->ClustersList->Clusters; + + LocalCluster = ""; + Enabled = false; + // Rebalance load on installation clusters: if preferred cluster is enabled and session is alive long enough close it so client can recreate it in preferred cluster auto remoteClusterEnabledDelay = TDuration::Seconds(AppData(ctx)->PQConfig.GetRemoteClusterEnabledDelaySec()); auto closeClientSessionWithEnabledRemotePreferredClusterDelay = TDuration::Seconds(AppData(ctx)->PQConfig.GetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec()); @@ -86,7 +86,7 @@ void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvCluster Enabled = cluster.IsEnabled; continue; } - + remoteClusters.emplace(cluster.Name); if (!cluster.IsEnabled) { @@ -126,7 +126,7 @@ void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvCluster const auto& workerID = Sessions[session.first]; Send(workerID, new TEvPQProxy::TEvDieCommand(closeReason, PersQueue::ErrorCode::PREFERRED_CLUSTER_MISMATCHED)); } - } + } } } } @@ -180,7 +180,7 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt TString localCluster = AvailableLocalCluster(ctx); if (HaveClusters && localCluster.empty()) { ev->Get()->GetStreamCtx()->Attach(ctx.SelfID); - if (LocalCluster) { + if (LocalCluster) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new grpc connection failed - cluster disabled"); ev->Get()->GetStreamCtx()->WriteAndFinish(FillWriteResponse("cluster disabled", PersQueue::ErrorCode::CLUSTER_DISABLED), grpc::Status::OK); //CANCELLED } else { diff --git a/ydb/services/persqueue_v1/grpc_pq_write.h b/ydb/services/persqueue_v1/grpc_pq_write.h index 73148388b2..787c47e262 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.h +++ b/ydb/services/persqueue_v1/grpc_pq_write.h @@ -10,7 +10,7 @@ #include <library/cpp/actors/core/actorsystem.h> #include <util/generic/hash.h> -#include <util/generic/maybe.h> +#include <util/generic/maybe.h> #include <util/system/mutex.h> namespace NKikimr { @@ -74,7 +74,7 @@ private: TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; ui32 MaxSessions; - TMaybe<TString> LocalCluster; + TMaybe<TString> LocalCluster; bool Enabled; TString SelectSourceIdQuery; TString UpdateSourceIdQuery; diff --git a/ydb/tests/functional/sqs/sqs_requests_client.py b/ydb/tests/functional/sqs/sqs_requests_client.py index 6b7545503f..e7823ea22e 100644 --- a/ydb/tests/functional/sqs/sqs_requests_client.py +++ b/ydb/tests/functional/sqs/sqs_requests_client.py @@ -28,23 +28,23 @@ def to_bytes(v): return v.encode('utf-8') -def auth_string(user): +def auth_string(user): return "AWS4-HMAC-SHA256 Credential={user}/{date}/yandex/sqs/aws4_request".format( - user=user, date=DEFAULT_DATE + user=user, date=DEFAULT_DATE ) -def auth_headers(user, security_token=None, iam_token=None): - headers = { - 'X-Amz-Date': '{}T104419Z'.format(DEFAULT_DATE), - 'Authorization': auth_string(user) +def auth_headers(user, security_token=None, iam_token=None): + headers = { + 'X-Amz-Date': '{}T104419Z'.format(DEFAULT_DATE), + 'Authorization': auth_string(user) } - if security_token is not None: - headers['X-Amz-Security-Token'] = security_token - if iam_token is not None: - headers['X-YaCloud-SubjectToken'] = iam_token - - return headers + if security_token is not None: + headers['X-Amz-Security-Token'] = security_token + if iam_token is not None: + headers['X-YaCloud-SubjectToken'] = iam_token + + return headers SQS_ATTRIBUTE_TYPES = {'String': 'StringValue', 'Number': 'StringValue', 'Binary': 'BinaryValue'} @@ -79,38 +79,38 @@ class SqsChangeMessageVisibilityParams(object): class SqsHttpApi(object): - def __init__(self, server, port, user, raise_on_error=False, timeout=REQUEST_TIMEOUT, security_token=None, force_private=False, iam_token=None, folder_id=None): - self.__auth_headers = auth_headers(user, security_token, iam_token) + def __init__(self, server, port, user, raise_on_error=False, timeout=REQUEST_TIMEOUT, security_token=None, force_private=False, iam_token=None, folder_id=None): + self.__auth_headers = auth_headers(user, security_token, iam_token) if not server.startswith('http'): server = 'http://' + server self.__request = requests.Request( 'POST', "{}:{}".format(server, port), - headers=auth_headers(user, security_token, iam_token) + headers=auth_headers(user, security_token, iam_token) ) self.__private_request = requests.Request( 'POST', "{}:{}/private".format(server, port), - headers=auth_headers(user, security_token, iam_token) + headers=auth_headers(user, security_token, iam_token) ) self.__session = requests.Session() self.__raise_on_error = raise_on_error self.__user = user self.__timeout = timeout - self.__security_token = security_token - assert(isinstance(force_private, bool)) - self.__force_private = force_private - self.__folder_id = folder_id + self.__security_token = security_token + assert(isinstance(force_private, bool)) + self.__force_private = force_private + self.__folder_id = folder_id def execute_request(self, action, extract_result_method, default=None, private=False, **params): if params is None: params = {} params['Action'] = action - - if self.__folder_id is not None: - params['folderId'] = self.__folder_id - - if self.__force_private or private: + + if self.__folder_id is not None: + params['folderId'] = self.__folder_id + + if self.__force_private or private: request = self.__private_request else: request = self.__request @@ -141,7 +141,7 @@ class SqsHttpApi(object): response.status_code, response.reason, response.text )) # Assert that no internal info will be given to user - assert response.text.find('.cpp:') == -1, 'No internal info should be given to user' + assert response.text.find('.cpp:') == -1, 'No internal info should be given to user' if self.__raise_on_error: raise RuntimeError( "Request {} failed with status {} and text {}".format( @@ -165,13 +165,13 @@ class SqsHttpApi(object): UserName=username ) - def delete_user(self, username): - return self.execute_request( - action='DeleteUser', - extract_result_method=lambda x: x['DeleteUserResponse']['ResponseMetadata'], - UserName=username - ) - + def delete_user(self, username): + return self.execute_request( + action='DeleteUser', + extract_result_method=lambda x: x['DeleteUserResponse']['ResponseMetadata'], + UserName=username + ) + def list_users(self): return self.execute_request( action='ListUsers', @@ -191,13 +191,13 @@ class SqsHttpApi(object): extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(), ) - def private_count_queues(self): - return self.execute_request( - action='CountQueues', - private=True, - extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'], - ) - + def private_count_queues(self): + return self.execute_request( + action='CountQueues', + private=True, + extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'], + ) + def create_queue(self, queue_name, is_fifo=False, attributes=None): # if is_fifo and not queue_name.endswith('.fifo'): # return None @@ -265,13 +265,13 @@ class SqsHttpApi(object): QueueName=queue_name ) - def list_dead_letter_source_queues(self, queue_url): - return self.execute_request( - action='ListDeadLetterSourceQueues', - extract_result_method=lambda x: wrap_in_list(x['ListDeadLetterSourceQueuesResponse']['ListDeadLetterSourceQueuesResult']['QueueUrl']), - QueueUrl=queue_url - ) - + def list_dead_letter_source_queues(self, queue_url): + return self.execute_request( + action='ListDeadLetterSourceQueues', + extract_result_method=lambda x: wrap_in_list(x['ListDeadLetterSourceQueuesResponse']['ListDeadLetterSourceQueuesResult']['QueueUrl']), + QueueUrl=queue_url + ) + def get_queue_attributes(self, queue_url, attributes=['All']): params = {} for i in six.moves.range(len(attributes)): @@ -283,9 +283,9 @@ class SqsHttpApi(object): **params ) result = {} - if attrib_list is None: - return result - + if attrib_list is None: + return result + for attr in wrap_in_list(attrib_list): result[attr['Name']] = attr['Value'] return result @@ -313,30 +313,30 @@ class SqsHttpApi(object): return result_list - def modify_permissions(self, action, subject, path, permissions): - args = { - 'Path': path, - 'Subject': subject - } - for i, permission in enumerate(permissions): - args['Permission.' + str(i)] = permission - - return self.execute_request( - action=action + 'Permissions', - extract_result_method=lambda x: x['ModifyPermissionsResponse']['ResponseMetadata']['RequestId'], - **args - ) - - def list_permissions(self, path): - args = { - 'Path': path, - } - return self.execute_request( - action='ListPermissions', - extract_result_method=lambda x: x['ListPermissionsResponse']['YaListPermissionsResult'], - **args - ) - + def modify_permissions(self, action, subject, path, permissions): + args = { + 'Path': path, + 'Subject': subject + } + for i, permission in enumerate(permissions): + args['Permission.' + str(i)] = permission + + return self.execute_request( + action=action + 'Permissions', + extract_result_method=lambda x: x['ModifyPermissionsResponse']['ResponseMetadata']['RequestId'], + **args + ) + + def list_permissions(self, path): + args = { + 'Path': path, + } + return self.execute_request( + action='ListPermissions', + extract_result_method=lambda x: x['ListPermissionsResponse']['YaListPermissionsResult'], + **args + ) + def set_queue_attributes(self, queue_url, attributes): params = {} i = 1 diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py index 05d5303f64..83c07bf585 100644 --- a/ydb/tests/functional/sqs/sqs_test_base.py +++ b/ydb/tests/functional/sqs/sqs_test_base.py @@ -11,7 +11,7 @@ import socket from hamcrest import assert_that, equal_to, not_none, none, greater_than, less_than_or_equal_to, any_of, not_ import ydb.tests.library.common.yatest_common as yatest_common - + from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels @@ -35,12 +35,12 @@ IS_FIFO_PARAMS = { 'ids': ['fifo', 'std'], } -POLLING_PARAMS = { - 'argnames': 'polling_wait_timeout', - 'argvalues': [0, 1], - 'ids': ['short_polling', 'long_polling'], -} - +POLLING_PARAMS = { + 'argnames': 'polling_wait_timeout', + 'argvalues': [0, 1], + 'ids': ['short_polling', 'long_polling'], +} + STOP_NODE_PARAMS = { 'argnames': 'stop_node', 'argvalues': [True, False], @@ -182,15 +182,15 @@ class KikimrSqsTestBase(object): max_queue_name_length = 80 - len('.fifo') if len(self.queue_name) > max_queue_name_length: self.queue_name = self.queue_name[:max_queue_name_length] - self._msg_body_template = self._username + '-{}' - self._setup_user(self._username) + self._msg_body_template = self._username + '-{}' + self._setup_user(self._username) self._sqs_apis = [] for node in self.cluster_nodes: self._sqs_apis.append( SqsHttpApi( 'localhost', node.sqs_port, - self._username, + self._username, raise_on_error=True, timeout=None ) @@ -312,7 +312,7 @@ class KikimrSqsTestBase(object): 'user', '-u', 'metauser', '-n', _username, - ] + self._sqs_server_opts + ] + self._sqs_server_opts while retries_count: logging.debug("Running {}".format(' '.join(cmd))) try: @@ -325,17 +325,17 @@ class KikimrSqsTestBase(object): return raise RuntimeError("Failed to create SQS user") - def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None): + def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None): api = SqsHttpApi(self.cluster.nodes[1].host, self.cluster_nodes[0].sqs_port, user_name, - raise_on_error=raise_on_error, - timeout=None, - security_token=security_token, - force_private=force_private, - iam_token=iam_token, - folder_id=folder_id - ) + raise_on_error=raise_on_error, + timeout=None, + security_token=security_token, + force_private=force_private, + iam_token=iam_token, + folder_id=folder_id + ) return api def _make_kikimr_driver(self, node_index=0): @@ -386,8 +386,8 @@ class KikimrSqsTestBase(object): if attributes is None: attributes = dict() logging.debug('Create queue. Attributes: {}. Use http: {}. Is fifo: {}'.format(attributes, use_http, is_fifo)) - assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation' - assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode' + assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation' + assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode' while retries: retries -= 1 try: @@ -397,11 +397,11 @@ class KikimrSqsTestBase(object): cmd = [ get_sqs_client_path(), 'create', # create queue command - '-u', self._username, - '--shards', str(shards) if shards is not None else '2', + '-u', self._username, + '--shards', str(shards) if shards is not None else '2', '--partitions', '1', '--queue-name', queue_name, - ] + self._sqs_server_opts + ] + self._sqs_server_opts execute = yatest_common.execute(cmd) self.queue_url = execute.std_out self.queue_url = self.queue_url.strip() @@ -626,29 +626,29 @@ class KikimrSqsTestBase(object): logging.debug('Connection error: trying to enable tablets on node {} in 1 second'.format(node_index)) time.sleep(1) # wait node to start - def _smart_make_table_path(self, user_name, queue_name, queue_version, shard, table_name): + def _smart_make_table_path(self, user_name, queue_name, queue_version, shard, table_name): table_path = '{}/{}'.format(self.sqs_root, user_name) - if queue_name is not None: - table_path += '/{}'.format(queue_name) - if queue_version is not None and queue_version != 0: - table_path += '/v{}'.format(queue_version) - if shard is not None: - table_path += '/{}'.format(shard) - - return table_path + '/{}'.format(table_name) - - def _get_queue_version_number(self, user_name, queue_name): + if queue_name is not None: + table_path += '/{}'.format(queue_name) + if queue_version is not None and queue_version != 0: + table_path += '/v{}'.format(queue_version) + if shard is not None: + table_path += '/{}'.format(shard) + + return table_path + '/{}'.format(table_name) + + def _get_queue_version_number(self, user_name, queue_name): table_path = '{}/.Queues'.format(self.sqs_root) data_result_sets = self._execute_yql_query('SELECT Version FROM `{}` WHERE Account=\'{}\' AND QueueName=\'{}\''.format(table_path, user_name, queue_name)) - assert_that(len(data_result_sets), equal_to(1)) - assert_that(len(data_result_sets[0].rows), equal_to(1)) - - version = data_result_sets[0].rows[0]['Version'] - if version is None: - return 0 - else: - return int(version) - + assert_that(len(data_result_sets), equal_to(1)) + assert_that(len(data_result_sets[0].rows), equal_to(1)) + + version = data_result_sets[0].rows[0]['Version'] + if version is None: + return 0 + else: + return int(version) + def _get_queue_master_tablet_id(self, user_name_param=None, queue_name_param=None): user_name = user_name_param if user_name_param else self._username queue_name = queue_name_param if queue_name_param else self.queue_name @@ -696,12 +696,12 @@ class KikimrSqsTestBase(object): queue_name = self.queue_name is_fifo = queue_name.endswith('.fifo') - queue_version = self._get_queue_version_number(self._username, queue_name) - + queue_version = self._get_queue_version_number(self._username, queue_name) + if is_fifo: - self._check_fifo_queue_is_empty(queue_name, queue_version) + self._check_fifo_queue_is_empty(queue_name, queue_version) else: - self._check_std_queue_is_empty(queue_name, queue_version) + self._check_std_queue_is_empty(queue_name, queue_version) def _get_table_lines_count(self, table_path): data_result_sets = self._execute_yql_query('SELECT COUNT(*) AS count FROM `{}`;'.format(table_path)) @@ -710,15 +710,15 @@ class KikimrSqsTestBase(object): logging.debug('Received count result for table {}: {}'.format(table_path, data_result_sets[0].rows[0])) return data_result_sets[0].rows[0]['count'] - def _check_std_queue_is_empty(self, queue_name, queue_version): + def _check_std_queue_is_empty(self, queue_name, queue_version): shards = self._get_queue_shards_count(self._username, queue_name, queue_version) assert_that(shards, not_(equal_to(0))) for shard in range(shards): - self._check_std_queue_shard_is_empty(queue_name, queue_version, shard) + self._check_std_queue_shard_is_empty(queue_name, queue_version, shard) - def _check_std_queue_shard_is_empty(self, queue_name, queue_version, shard): + def _check_std_queue_shard_is_empty(self, queue_name, queue_version, shard): def get_table_path(table_name): - return self._smart_make_table_path(self._username, queue_name, queue_version, shard, table_name) + return self._smart_make_table_path(self._username, queue_name, queue_version, shard, table_name) def get_lines_count(table_name): return self._get_table_lines_count(get_table_path(table_name)) @@ -728,9 +728,9 @@ class KikimrSqsTestBase(object): assert_that(get_lines_count('Messages'), equal_to(0)) assert_that(get_lines_count('SentTimestampIdx'), equal_to(0)) - def _check_fifo_queue_is_empty(self, queue_name, queue_version): + def _check_fifo_queue_is_empty(self, queue_name, queue_version): def get_table_path(table_name): - return self._smart_make_table_path(self._username, queue_name, queue_version, None, table_name) + return self._smart_make_table_path(self._username, queue_name, queue_version, None, table_name) def get_lines_count(table_name): return self._get_table_lines_count(get_table_path(table_name)) diff --git a/ydb/tests/functional/sqs/test_account_actions.py b/ydb/tests/functional/sqs/test_account_actions.py index 60ab1e4d10..1b7e733534 100644 --- a/ydb/tests/functional/sqs/test_account_actions.py +++ b/ydb/tests/functional/sqs/test_account_actions.py @@ -4,29 +4,29 @@ from hamcrest import assert_that, not_none, has_item, is_not from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation - + class AccountActionsTest(KikimrSqsTestBase): - def test_manage_account(self): - user_name = 'pupkin' - create_user_result = self._sqs_api.create_user(user_name) - assert_that( - create_user_result, - not_none() - ) - user_list = self._sqs_api.list_users() - assert_that( - user_list, has_item(user_name) - ) - - delete_user_result = self._sqs_api.delete_user(user_name) - assert_that( - delete_user_result, - not_none() - ) - user_list = self._sqs_api.list_users() - assert_that( - user_list, is_not(has_item(user_name)) - ) + def test_manage_account(self): + user_name = 'pupkin' + create_user_result = self._sqs_api.create_user(user_name) + assert_that( + create_user_result, + not_none() + ) + user_list = self._sqs_api.list_users() + assert_that( + user_list, has_item(user_name) + ) + + delete_user_result = self._sqs_api.delete_user(user_name) + assert_that( + delete_user_result, + not_none() + ) + user_list = self._sqs_api.list_users() + assert_that( + user_list, is_not(has_item(user_name)) + ) class TestAccountActionsWithTenant(get_test_with_sqs_tenant_installation(AccountActionsTest)): diff --git a/ydb/tests/functional/sqs/test_acl.py b/ydb/tests/functional/sqs/test_acl.py index e5221f0f43..658fcebcb6 100644 --- a/ydb/tests/functional/sqs/test_acl.py +++ b/ydb/tests/functional/sqs/test_acl.py @@ -7,167 +7,167 @@ import pytest from hamcrest import assert_that, equal_to, none, is_not, is_, raises import ydb.tests.library.common.yatest_common as yatest_common - + from sqs_test_base import KikimrSqsTestBase, get_sqs_client_path, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, to_bytes class SqsACLTest(KikimrSqsTestBase): - def _modify_permissions(self, resource, action, permissions_string, clear_acl_flag=False): - cmd = [ - get_sqs_client_path(), - 'permissions', - '--resource', resource, - '--{}'.format(action), - '{}'.format(permissions_string) - ] - if clear_acl_flag: - cmd.append('--clear-acl') - - cmd += self._sqs_server_opts - - retries_count = 1 - while retries_count: + def _modify_permissions(self, resource, action, permissions_string, clear_acl_flag=False): + cmd = [ + get_sqs_client_path(), + 'permissions', + '--resource', resource, + '--{}'.format(action), + '{}'.format(permissions_string) + ] + if clear_acl_flag: + cmd.append('--clear-acl') + + cmd += self._sqs_server_opts + + retries_count = 1 + while retries_count: logging.debug("Running {}".format(' '.join(cmd))) - try: - yatest_common.execute(cmd) - except yatest_common.ExecutionError as ex: - logging.debug("Modify permissions failed: {}. Retrying".format(ex)) - retries_count -= 1 - time.sleep(3) - else: - return - raise RuntimeError("Failed to modify permissions") - - def _list_permissions(self, path, retries_count=1): - cmd = [ - get_sqs_client_path(), - 'list-permissions', - '--path', path, - ] + self._sqs_server_opts - - while retries_count: + try: + yatest_common.execute(cmd) + except yatest_common.ExecutionError as ex: + logging.debug("Modify permissions failed: {}. Retrying".format(ex)) + retries_count -= 1 + time.sleep(3) + else: + return + raise RuntimeError("Failed to modify permissions") + + def _list_permissions(self, path, retries_count=1): + cmd = [ + get_sqs_client_path(), + 'list-permissions', + '--path', path, + ] + self._sqs_server_opts + + while retries_count: logging.debug("Running {}".format(' '.join(cmd))) - try: - execute = yatest_common.execute(cmd) - except yatest_common.ExecutionError as ex: - logging.debug("List permissions failed: {}. Retrying".format(ex)) - retries_count -= 1 - time.sleep(3) - else: - return execute.std_out - raise RuntimeError("Failed to list permissions") - - def __generate_expected_simplified_permissions_response(self, sid=None, permission_names=[]): - resp = 'AccountPermissions {' - if not permission_names: - resp += '}' - else: - resp += 'Permissions {{ Subject: \"{}\"'.format(sid) - for permission_name in permission_names: - resp += 'PermissionNames: \"{}\"'.format(permission_name) - resp += '}} EffectivePermissions {{ Subject: \"{}\"'.format(sid) - for permission_name in permission_names: - resp += 'PermissionNames: \"{}\"'.format(permission_name) - resp += '}}' - - return resp - - def __clean_and_compare(self, s1, s2): + try: + execute = yatest_common.execute(cmd) + except yatest_common.ExecutionError as ex: + logging.debug("List permissions failed: {}. Retrying".format(ex)) + retries_count -= 1 + time.sleep(3) + else: + return execute.std_out + raise RuntimeError("Failed to list permissions") + + def __generate_expected_simplified_permissions_response(self, sid=None, permission_names=[]): + resp = 'AccountPermissions {' + if not permission_names: + resp += '}' + else: + resp += 'Permissions {{ Subject: \"{}\"'.format(sid) + for permission_name in permission_names: + resp += 'PermissionNames: \"{}\"'.format(permission_name) + resp += '}} EffectivePermissions {{ Subject: \"{}\"'.format(sid) + for permission_name in permission_names: + resp += 'PermissionNames: \"{}\"'.format(permission_name) + resp += '}}' + + return resp + + def __clean_and_compare(self, s1, s2): s1 = to_bytes(s1)[to_bytes(s1).find(to_bytes('AccountPermissions')):].replace(to_bytes('\n'), to_bytes('')).replace(to_bytes(' '), to_bytes('')) s2 = to_bytes(s2)[to_bytes(s2).find(to_bytes('AccountPermissions')):].replace(to_bytes('\n'), to_bytes('')).replace(to_bytes(' '), to_bytes('')) - - assert_that(s1, equal_to(s2)) - - def test_modify_permissions(self): - queue_url = self._create_queue_and_assert(self.queue_name, False, True) - self._send_message_and_assert(queue_url, 'data') - - alkonavt_sid = 'alkonavt@builtin' - berkanavt_sid = 'berkanavt@builtin' - - create_queue_permission = 'CreateQueue' - send_message_permission = 'SendMessage' - - # no permissions expected - description = self._list_permissions(self._username) - expected_description = self.__generate_expected_simplified_permissions_response() - self.__clean_and_compare(description, expected_description) - - # two permissions expected - self._modify_permissions( - self._username, - 'grant', - alkonavt_sid + ':' + ','.join([create_queue_permission, send_message_permission]) - ) - description = self._list_permissions(self._username) - expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [create_queue_permission, send_message_permission]) - self.__clean_and_compare(description, expected_description) - - # single permission expected - self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission) - description = self._list_permissions(self._username) - expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [send_message_permission]) - self.__clean_and_compare(description, expected_description) - - receive_message_permission = 'ReceiveMessage' - - # other single permission expected - self._modify_permissions(self._username, 'set', alkonavt_sid + ':' + receive_message_permission) - description = self._list_permissions(self._username) - expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [receive_message_permission]) - self.__clean_and_compare(description, expected_description) - - # clear all permissions - self._modify_permissions(self._username, 'set', berkanavt_sid + ':' + receive_message_permission) - self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission, True) - description = self._list_permissions(self._username) - expected_description = self.__generate_expected_simplified_permissions_response() - self.__clean_and_compare(description, expected_description) - - def test_apply_permissions(self): - queue_url = self._create_queue_and_assert(self.queue_name, False, True) - self._send_message_and_assert(queue_url, 'data') - - berkanavt_sid = 'berkanavt@builtin' - - self._sqs_api = self._create_api_for_user(self._username, raise_on_error=False, security_token=berkanavt_sid) - - def __send_message_with_retries(_queue_url, data, result_predicate): - retries = 3 - - while retries > 0: - retries = retries - 1 - try: - result = self._sqs_api.send_message(_queue_url, data) - assert_that(result, result_predicate) - return result - except: - if retries == 0: - raise - time.sleep(0.1) - - __send_message_with_retries(queue_url, 'megadata', is_(none())) # no access - - self._modify_permissions( - self._username, - 'grant', - berkanavt_sid + ':' + ','.join(['ModifyPermissions']) - ) - - result = self._sqs_api.modify_permissions('Grant', berkanavt_sid, self._username, ['SendMessage', 'DescribePath']) - assert_that(result, is_not(none())) - - __send_message_with_retries(queue_url, 'utradata', is_not(none())) # has access - - result = self._sqs_api.modify_permissions('Revoke', berkanavt_sid, self._username, ['SendMessage', 'AlterQueue']) - assert_that(result, is_not(none())) - - __send_message_with_retries(queue_url, 'superdata', is_(none())) # no access again. that's a pity - - result = self._sqs_api.list_permissions(self._username) - assert('Account' in str(result)) - assert(berkanavt_sid in str(result)) - assert('Permissions' in str(result)) + + assert_that(s1, equal_to(s2)) + + def test_modify_permissions(self): + queue_url = self._create_queue_and_assert(self.queue_name, False, True) + self._send_message_and_assert(queue_url, 'data') + + alkonavt_sid = 'alkonavt@builtin' + berkanavt_sid = 'berkanavt@builtin' + + create_queue_permission = 'CreateQueue' + send_message_permission = 'SendMessage' + + # no permissions expected + description = self._list_permissions(self._username) + expected_description = self.__generate_expected_simplified_permissions_response() + self.__clean_and_compare(description, expected_description) + + # two permissions expected + self._modify_permissions( + self._username, + 'grant', + alkonavt_sid + ':' + ','.join([create_queue_permission, send_message_permission]) + ) + description = self._list_permissions(self._username) + expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [create_queue_permission, send_message_permission]) + self.__clean_and_compare(description, expected_description) + + # single permission expected + self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission) + description = self._list_permissions(self._username) + expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [send_message_permission]) + self.__clean_and_compare(description, expected_description) + + receive_message_permission = 'ReceiveMessage' + + # other single permission expected + self._modify_permissions(self._username, 'set', alkonavt_sid + ':' + receive_message_permission) + description = self._list_permissions(self._username) + expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [receive_message_permission]) + self.__clean_and_compare(description, expected_description) + + # clear all permissions + self._modify_permissions(self._username, 'set', berkanavt_sid + ':' + receive_message_permission) + self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission, True) + description = self._list_permissions(self._username) + expected_description = self.__generate_expected_simplified_permissions_response() + self.__clean_and_compare(description, expected_description) + + def test_apply_permissions(self): + queue_url = self._create_queue_and_assert(self.queue_name, False, True) + self._send_message_and_assert(queue_url, 'data') + + berkanavt_sid = 'berkanavt@builtin' + + self._sqs_api = self._create_api_for_user(self._username, raise_on_error=False, security_token=berkanavt_sid) + + def __send_message_with_retries(_queue_url, data, result_predicate): + retries = 3 + + while retries > 0: + retries = retries - 1 + try: + result = self._sqs_api.send_message(_queue_url, data) + assert_that(result, result_predicate) + return result + except: + if retries == 0: + raise + time.sleep(0.1) + + __send_message_with_retries(queue_url, 'megadata', is_(none())) # no access + + self._modify_permissions( + self._username, + 'grant', + berkanavt_sid + ':' + ','.join(['ModifyPermissions']) + ) + + result = self._sqs_api.modify_permissions('Grant', berkanavt_sid, self._username, ['SendMessage', 'DescribePath']) + assert_that(result, is_not(none())) + + __send_message_with_retries(queue_url, 'utradata', is_not(none())) # has access + + result = self._sqs_api.modify_permissions('Revoke', berkanavt_sid, self._username, ['SendMessage', 'AlterQueue']) + assert_that(result, is_not(none())) + + __send_message_with_retries(queue_url, 'superdata', is_(none())) # no access again. that's a pity + + result = self._sqs_api.list_permissions(self._username) + assert('Account' in str(result)) + assert(berkanavt_sid in str(result)) + assert('Permissions' in str(result)) class SqsWithForceAuthorizationTest(KikimrSqsTestBase): diff --git a/ydb/tests/functional/sqs/test_fifo_messaging.py b/ydb/tests/functional/sqs/test_fifo_messaging.py index 6410b21966..14d96ed93e 100644 --- a/ydb/tests/functional/sqs/test_fifo_messaging.py +++ b/ydb/tests/functional/sqs/test_fifo_messaging.py @@ -11,35 +11,35 @@ from sqs_matchers import ReadResponseMatcher, extract_message_ids from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, VISIBILITY_CHANGE_METHOD_PARAMS -class SqsFifoMicroBatchTest(KikimrSqsTestBase): - @classmethod - def _setup_config_generator(cls): - config_generator = super(SqsFifoMicroBatchTest, cls)._setup_config_generator() +class SqsFifoMicroBatchTest(KikimrSqsTestBase): + @classmethod + def _setup_config_generator(cls): + config_generator = super(SqsFifoMicroBatchTest, cls)._setup_config_generator() config_generator.yaml_config['sqs_config']['group_selection_batch_size'] = 2 - return config_generator - - def test_micro_batch_read(self): - self.queue_name = self.queue_name + '.fifo' - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) - seq_no = 0 - max_number_of_messages = 3 - for i in range(max_number_of_messages): - self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=str(seq_no), group_id=str(seq_no)) - seq_no += 1 - - for i in range(2): - msgs = self._sqs_api.receive_message(created_queue_url, max_number_of_messages=max_number_of_messages, visibility_timeout=0, receive_request_attempt_id='test') - assert_that(len(set([msgs[i]['MessageId'] for i in range(len(msgs))])), equal_to(len(msgs))) - - -class TestSqsFifoMicroBatchesWithTenant(get_test_with_sqs_tenant_installation(SqsFifoMicroBatchTest)): - pass - - -class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(SqsFifoMicroBatchTest)): - pass - - + return config_generator + + def test_micro_batch_read(self): + self.queue_name = self.queue_name + '.fifo' + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) + seq_no = 0 + max_number_of_messages = 3 + for i in range(max_number_of_messages): + self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=str(seq_no), group_id=str(seq_no)) + seq_no += 1 + + for i in range(2): + msgs = self._sqs_api.receive_message(created_queue_url, max_number_of_messages=max_number_of_messages, visibility_timeout=0, receive_request_attempt_id='test') + assert_that(len(set([msgs[i]['MessageId'] for i in range(len(msgs))])), equal_to(len(msgs))) + + +class TestSqsFifoMicroBatchesWithTenant(get_test_with_sqs_tenant_installation(SqsFifoMicroBatchTest)): + pass + + +class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(SqsFifoMicroBatchTest)): + pass + + class SqsFifoMessagingTest(KikimrSqsTestBase): def setup_method(self, method=None): super(SqsFifoMessagingTest, self).setup_method(method) diff --git a/ydb/tests/functional/sqs/test_garbage_collection.py b/ydb/tests/functional/sqs/test_garbage_collection.py index 992c8a6513..919ad29705 100644 --- a/ydb/tests/functional/sqs/test_garbage_collection.py +++ b/ydb/tests/functional/sqs/test_garbage_collection.py @@ -9,7 +9,7 @@ import pytest from hamcrest import assert_that, equal_to, less_than_or_equal_to from sqs_requests_client import SqsHttpApi - + from sqs_matchers import ReadResponseMatcher from sqs_test_base import to_bytes diff --git a/ydb/tests/functional/sqs/test_generic_messaging.py b/ydb/tests/functional/sqs/test_generic_messaging.py index 3f6b2371ba..1546a2b0e7 100644 --- a/ydb/tests/functional/sqs/test_generic_messaging.py +++ b/ydb/tests/functional/sqs/test_generic_messaging.py @@ -9,7 +9,7 @@ import pytest from hamcrest import assert_that, equal_to, not_none, greater_than, has_item, has_items, raises, is_not, not_, empty, instance_of from sqs_requests_client import SqsMessageAttribute, SqsSendMessageParams, SqsChangeMessageVisibilityParams - + from sqs_matchers import ReadResponseMatcher, extract_message_ids from sqs_test_base import to_bytes from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS @@ -644,10 +644,10 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): assert_that( self._sqs_api.delete_message(self.queue_url, handle), not_none() ) - # check double deletion - assert_that( - self._sqs_api.delete_message(self.queue_url, handle), not_none() - ) + # check double deletion + assert_that( + self._sqs_api.delete_message(self.queue_url, handle), not_none() + ) self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids(self.message_ids) @@ -849,13 +849,13 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): if is_fifo: self.queue_name = self.queue_name + '.fifo' queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) - - # assert empty response when no attribute names are provided - attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=[]) - assert_that(attributes, equal_to({})) - - # check common case - attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=['All']) + + # assert empty response when no attribute names are provided + attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=[]) + assert_that(attributes, equal_to({})) + + # check common case + attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=['All']) if is_fifo: assert_that(attributes, has_item('FifoQueue')) assert_that(attributes, has_item('ContentBasedDeduplication')) @@ -876,7 +876,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): )) assert_that(attributes['ReceiveMessageWaitTimeSeconds'], equal_to('0')) if is_fifo: - assert_that(attributes['ContentBasedDeduplication'], equal_to('false')) + assert_that(attributes['ContentBasedDeduplication'], equal_to('false')) self._sqs_api.set_queue_attributes(queue_url, {'ReceiveMessageWaitTimeSeconds': '10', 'MaximumMessageSize': '111111'}) attributes = self._sqs_api.get_queue_attributes(queue_url) @@ -886,7 +886,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): if is_fifo: self._sqs_api.set_queue_attributes(queue_url, {'ContentBasedDeduplication': 'true'}) attributes = self._sqs_api.get_queue_attributes(queue_url) - assert_that(attributes['ContentBasedDeduplication'], equal_to('true')) + assert_that(attributes['ContentBasedDeduplication'], equal_to('true')) def test_set_very_big_visibility_timeout(self): queue_url = self._create_queue_and_assert(self.queue_name) @@ -896,7 +896,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): assert_that( call_with_very_big_visibility_timeout, - raises(RuntimeError, pattern='InvalidAttributeValue') + raises(RuntimeError, pattern='InvalidAttributeValue') ) def test_wrong_attribute_name(self): diff --git a/ydb/tests/functional/sqs/test_multinode_cluster.py b/ydb/tests/functional/sqs/test_multinode_cluster.py index 047ae56198..4a27391ef2 100644 --- a/ydb/tests/functional/sqs/test_multinode_cluster.py +++ b/ydb/tests/functional/sqs/test_multinode_cluster.py @@ -8,7 +8,7 @@ import pytest from hamcrest import assert_that, equal_to, not_none, raises, not_ from ydb.tests.library.common.types import Erasure - + from sqs_matchers import ReadResponseMatcher from sqs_test_base import KikimrSqsTestBase, STOP_NODE_PARAMS, IS_FIFO_PARAMS diff --git a/ydb/tests/functional/sqs/test_polling.py b/ydb/tests/functional/sqs/test_polling.py index dba0556e7d..7f972cb169 100644 --- a/ydb/tests/functional/sqs/test_polling.py +++ b/ydb/tests/functional/sqs/test_polling.py @@ -4,36 +4,36 @@ import pytest from hamcrest import assert_that, equal_to from sqs_matchers import ReadResponseMatcher - + from sqs_test_base import KikimrSqsTestBase, POLLING_PARAMS, IS_FIFO_PARAMS -class TestSqsPolling(KikimrSqsTestBase): - @classmethod - def _setup_config_generator(cls): - config_generator = super(TestSqsPolling, cls)._setup_config_generator() +class TestSqsPolling(KikimrSqsTestBase): + @classmethod + def _setup_config_generator(cls): + config_generator = super(TestSqsPolling, cls)._setup_config_generator() config_generator.yaml_config['sqs_config']['check_all_shards_in_receive_message'] = False - return config_generator - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - @pytest.mark.parametrize(**POLLING_PARAMS) - def test_receive_message_with_polling(self, is_fifo, polling_wait_timeout): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1) - empty_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1) - - self.seq_no += 1 - self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) - - read_result = self._read_messages_and_assert( - created_queue_url, messages_count=1, visibility_timeout=1000, wait_timeout=polling_wait_timeout, + return config_generator + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + @pytest.mark.parametrize(**POLLING_PARAMS) + def test_receive_message_with_polling(self, is_fifo, polling_wait_timeout): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1) + empty_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1) + + self.seq_no += 1 + self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) + + read_result = self._read_messages_and_assert( + created_queue_url, messages_count=1, visibility_timeout=1000, wait_timeout=polling_wait_timeout, matcher=ReadResponseMatcher().with_n_messages(1) - ) - assert_that(read_result[0]['Body'], equal_to('test_send_message')) - - # check that there's no infinite loop for empty queue - read_result = self._read_messages_and_assert( - empty_queue_url, messages_count=3, visibility_timeout=1000, wait_timeout=polling_wait_timeout - ) - assert_that(len(read_result), equal_to(0)) + ) + assert_that(read_result[0]['Body'], equal_to('test_send_message')) + + # check that there's no infinite loop for empty queue + read_result = self._read_messages_and_assert( + empty_queue_url, messages_count=3, visibility_timeout=1000, wait_timeout=polling_wait_timeout + ) + assert_that(len(read_result), equal_to(0)) diff --git a/ydb/tests/functional/sqs/test_queue_attributes_validation.py b/ydb/tests/functional/sqs/test_queue_attributes_validation.py index 27ec79c9ee..e7e2ea8723 100644 --- a/ydb/tests/functional/sqs/test_queue_attributes_validation.py +++ b/ydb/tests/functional/sqs/test_queue_attributes_validation.py @@ -4,183 +4,183 @@ import pytest from hamcrest import assert_that, equal_to from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS + - -class TestQueueAttributesInCompatibilityMode(KikimrSqsTestBase): - @classmethod - def _setup_config_generator(cls): - config_generator = super(TestQueueAttributesInCompatibilityMode, cls)._setup_config_generator() +class TestQueueAttributesInCompatibilityMode(KikimrSqsTestBase): + @classmethod + def _setup_config_generator(cls): + config_generator = super(TestQueueAttributesInCompatibilityMode, cls)._setup_config_generator() config_generator.yaml_config['sqs_config']['enable_queue_attributes_validation'] = False - return config_generator - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_set_queue_attributes_no_validation(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': '1000000'}) - assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to(str(256 * 1024))) - - self._sqs_api.set_queue_attributes(queue_url, {'MaximumMessageSize': '1'}) - assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024')) - - # create queue again, this should yield no error - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) - # previously set attributes should be right there - assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024')) - try: + return config_generator + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + def test_set_queue_attributes_no_validation(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': '1000000'}) + assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to(str(256 * 1024))) + + self._sqs_api.set_queue_attributes(queue_url, {'MaximumMessageSize': '1'}) + assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024')) + + # create queue again, this should yield no error + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) + # previously set attributes should be right there + assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024')) + try: queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': 'troll'}, retries=1) - except: - pass - else: - raise Exception('only parseable attributes are valid') - - -class TestQueueAttributesValidation(KikimrSqsTestBase): - @classmethod - def _setup_config_generator(cls): - config_generator = super(TestQueueAttributesValidation, cls)._setup_config_generator() - return config_generator - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_set_queue_attributes(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) - - attributes = self._sqs_api.get_queue_attributes(queue_url) - queue_attributes_list = ['VisibilityTimeout', 'MaximumMessageSize', 'DelaySeconds', 'MessageRetentionPeriod', 'ReceiveMessageWaitTimeSeconds'] - attributes_to_set_and_check = {} - for attr in queue_attributes_list: - val = int(attributes[attr]) - # change attributes slightly - attributes_to_set_and_check[attr] = str(val - 1 if val > 0 else val + 1) - assert(attributes_to_set_and_check[attr] != attributes[attr]) - - content_based_dedup_key = 'ContentBasedDeduplication' - if is_fifo: - val = attributes[content_based_dedup_key] - attributes_to_set_and_check[content_based_dedup_key] = 'true' if val == 'false' else 'false' - assert(attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key]) - - self._sqs_api.set_queue_attributes(queue_url, attributes_to_set_and_check) - # match set attributes - attributes = self._sqs_api.get_queue_attributes(queue_url) - for attr in attributes_to_set_and_check: - assert_that(attributes_to_set_and_check[attr], equal_to(attributes[attr])) - - # okay, now we'll try to break it - for attr in attributes_to_set_and_check: - try: - self._sqs_api.set_queue_attributes(queue_url, {attr: '2.5'}) + except: + pass + else: + raise Exception('only parseable attributes are valid') + + +class TestQueueAttributesValidation(KikimrSqsTestBase): + @classmethod + def _setup_config_generator(cls): + config_generator = super(TestQueueAttributesValidation, cls)._setup_config_generator() + return config_generator + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + def test_set_queue_attributes(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) + + attributes = self._sqs_api.get_queue_attributes(queue_url) + queue_attributes_list = ['VisibilityTimeout', 'MaximumMessageSize', 'DelaySeconds', 'MessageRetentionPeriod', 'ReceiveMessageWaitTimeSeconds'] + attributes_to_set_and_check = {} + for attr in queue_attributes_list: + val = int(attributes[attr]) + # change attributes slightly + attributes_to_set_and_check[attr] = str(val - 1 if val > 0 else val + 1) + assert(attributes_to_set_and_check[attr] != attributes[attr]) + + content_based_dedup_key = 'ContentBasedDeduplication' + if is_fifo: + val = attributes[content_based_dedup_key] + attributes_to_set_and_check[content_based_dedup_key] = 'true' if val == 'false' else 'false' + assert(attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key]) + + self._sqs_api.set_queue_attributes(queue_url, attributes_to_set_and_check) + # match set attributes + attributes = self._sqs_api.get_queue_attributes(queue_url) + for attr in attributes_to_set_and_check: + assert_that(attributes_to_set_and_check[attr], equal_to(attributes[attr])) + + # okay, now we'll try to break it + for attr in attributes_to_set_and_check: + try: + self._sqs_api.set_queue_attributes(queue_url, {attr: '2.5'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert(attr in str(e)) - else: - assert_that(False) # expected InvalidAttributeValue exception - - try: - self._sqs_api.set_queue_attributes(queue_url, {attr: '2147483647'}) + assert('InvalidAttributeValue' in str(e)) + assert(attr in str(e)) + else: + assert_that(False) # expected InvalidAttributeValue exception + + try: + self._sqs_api.set_queue_attributes(queue_url, {attr: '2147483647'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert(attr in str(e)) - else: - assert_that(False) # expected InvalidAttributeValue exception - - try: - self._sqs_api.set_queue_attributes(queue_url, {'trololo': 'ololo'}) + assert('InvalidAttributeValue' in str(e)) + assert(attr in str(e)) + else: + assert_that(False) # expected InvalidAttributeValue exception + + try: + self._sqs_api.set_queue_attributes(queue_url, {'trololo': 'ololo'}) except Exception as e: - assert('InvalidAttributeName' in str(e)) - else: - assert_that(False) # expected InvalidAttributeName exception - - # check setting FifoQueue attribute - if is_fifo: - self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'true'}) # ok, do nothing - try: - self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'omg'}) + assert('InvalidAttributeName' in str(e)) + else: + assert_that(False) # expected InvalidAttributeName exception + + # check setting FifoQueue attribute + if is_fifo: + self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'true'}) # ok, do nothing + try: + self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'omg'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert('FifoQueue' in str(e)) - else: - assert_that(False) # expected InvalidAttributeValue exception - - # special case: queue type mismatch - try: - self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) + assert('InvalidAttributeValue' in str(e)) + assert('FifoQueue' in str(e)) + else: + assert_that(False) # expected InvalidAttributeValue exception + + # special case: queue type mismatch + try: + self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert('Modifying queue type is not supported' in str(e)) - else: - assert_that(False) # expected InvalidAttributeValue exception - else: - try: - self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) + assert('InvalidAttributeValue' in str(e)) + assert('Modifying queue type is not supported' in str(e)) + else: + assert_that(False) # expected InvalidAttributeValue exception + else: + try: + self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) except Exception as e: - assert('InvalidAttributeName' in str(e)) - else: - assert_that(False) # expected InvalidAttributeName exception - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_create_queue_with_default_attributes(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - - # check default queue creation - queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo) - - expected_attributes = { - 'DelaySeconds': '0', - 'MaximumMessageSize': '262144', - 'MessageRetentionPeriod': '345600', - 'ReceiveMessageWaitTimeSeconds': '0', - 'VisibilityTimeout': '30' - } - - if is_fifo: - expected_attributes['ContentBasedDeduplication'] = 'false' - expected_attributes['FifoQueue'] = 'true' - - attributes = self._sqs_api.get_queue_attributes(queue_url) - for attr in expected_attributes: - assert_that(attributes[attr], equal_to(expected_attributes[attr])) - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_create_queue_with_custom_attributes(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - - custom_attributes = { - 'DelaySeconds': '1', - 'MaximumMessageSize': '262143', - 'MessageRetentionPeriod': '345599', - 'ReceiveMessageWaitTimeSeconds': '1', - 'VisibilityTimeout': '29' - } - - if is_fifo: - custom_attributes['ContentBasedDeduplication'] = 'true' - - queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo, attributes=custom_attributes) - - attributes = self._sqs_api.get_queue_attributes(queue_url) - for attr in custom_attributes: - assert_that(attributes[attr], equal_to(custom_attributes[attr])) - - # get arn by default + assert('InvalidAttributeName' in str(e)) + else: + assert_that(False) # expected InvalidAttributeName exception + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + def test_create_queue_with_default_attributes(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + + # check default queue creation + queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo) + + expected_attributes = { + 'DelaySeconds': '0', + 'MaximumMessageSize': '262144', + 'MessageRetentionPeriod': '345600', + 'ReceiveMessageWaitTimeSeconds': '0', + 'VisibilityTimeout': '30' + } + + if is_fifo: + expected_attributes['ContentBasedDeduplication'] = 'false' + expected_attributes['FifoQueue'] = 'true' + + attributes = self._sqs_api.get_queue_attributes(queue_url) + for attr in expected_attributes: + assert_that(attributes[attr], equal_to(expected_attributes[attr])) + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + def test_create_queue_with_custom_attributes(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + + custom_attributes = { + 'DelaySeconds': '1', + 'MaximumMessageSize': '262143', + 'MessageRetentionPeriod': '345599', + 'ReceiveMessageWaitTimeSeconds': '1', + 'VisibilityTimeout': '29' + } + + if is_fifo: + custom_attributes['ContentBasedDeduplication'] = 'true' + + queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo, attributes=custom_attributes) + + attributes = self._sqs_api.get_queue_attributes(queue_url) + for attr in custom_attributes: + assert_that(attributes[attr], equal_to(custom_attributes[attr])) + + # get arn by default assert_that(attributes['QueueArn'], equal_to('yrn:ya:sqs:ru-central1:' + self._username + ':' + self.queue_name)) - - # okay, now we'll try to break it - for attr in custom_attributes: - try: - queue_url = self._sqs_api.create_queue('new_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2147483647'}) + + # okay, now we'll try to break it + for attr in custom_attributes: + try: + queue_url = self._sqs_api.create_queue('new_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2147483647'}) except Exception as e: - assert(attr in str(e)) - else: - assert_that(False) # expected some exception - - try: - queue_url = self._sqs_api.create_queue('new2_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2.5'}) + assert(attr in str(e)) + else: + assert_that(False) # expected some exception + + try: + queue_url = self._sqs_api.create_queue('new2_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2.5'}) except Exception as e: - assert(attr in str(e)) - else: - assert_that(False) # expected some exception + assert(attr in str(e)) + else: + assert_that(False) # expected some exception diff --git a/ydb/tests/functional/sqs/test_queues_managing.py b/ydb/tests/functional/sqs/test_queues_managing.py index bcfd91ca67..58bf407ebf 100644 --- a/ydb/tests/functional/sqs/test_queues_managing.py +++ b/ydb/tests/functional/sqs/test_queues_managing.py @@ -7,7 +7,7 @@ import pytest from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of from sqs_matchers import ReadResponseMatcher - + from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS from sqs_test_base import to_bytes from ydb import issues as ydb_issues @@ -269,19 +269,19 @@ class QueuesManagingTest(KikimrSqsTestBase): created_queue_url, 10, ReadResponseMatcher().with_message_ids([msg_id, ]) ) - def test_ya_count_queues(self): - assert_that(self._sqs_api.private_count_queues(), equal_to('0')) - q_url = self._create_queue_and_assert('new_q') - self._create_queue_and_assert('new_q_2') - + def test_ya_count_queues(self): + assert_that(self._sqs_api.private_count_queues(), equal_to('0')) + q_url = self._create_queue_and_assert('new_q') + self._create_queue_and_assert('new_q_2') + time.sleep(2.1) - assert_that(self._sqs_api.private_count_queues(), equal_to('2')) - - self._sqs_api.delete_queue(q_url) - + assert_that(self._sqs_api.private_count_queues(), equal_to('2')) + + self._sqs_api.delete_queue(q_url) + time.sleep(2.1) - assert_that(self._sqs_api.private_count_queues(), equal_to('1')) - + assert_that(self._sqs_api.private_count_queues(), equal_to('1')) + def test_queues_count_over_limit(self): urls = [] for i in range(10): diff --git a/ydb/tests/functional/sqs/test_recompiles_requests.py b/ydb/tests/functional/sqs/test_recompiles_requests.py index 367003323f..3182bff9ad 100644 --- a/ydb/tests/functional/sqs/test_recompiles_requests.py +++ b/ydb/tests/functional/sqs/test_recompiles_requests.py @@ -4,7 +4,7 @@ import pytest from hamcrest import assert_that, not_none from ydb.tests.library.common.types import Erasure - + from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make index bbb0a57bb8..f71e3a9b0f 100644 --- a/ydb/tests/functional/sqs/ya.make +++ b/ydb/tests/functional/sqs/ya.make @@ -52,8 +52,8 @@ PEERDIR( ydb/tests/library ydb/tests/library/sqs contrib/python/xmltodict - contrib/python/boto3 - contrib/python/botocore + contrib/python/boto3 + contrib/python/botocore ) FORK_SUBTESTS() |